Model Compare

"use client"

import { DefaultChatTransport, type LanguageModelUsage, type UIMessage, type UIMessageChunk } from "ai"

import type { ModelInfo } from "../models/models"
import type { PromptTokenPricing } from "../prompt-input/prompt-controls"
import {
  ModelCompare,
  type CompareColumn,
  type CompareRequest,
  type CompareRequestColumn,
  type CompareStream,
} from "@/components/blocks/model-compare/model-compare"

type ChatMessage = UIMessage<{ usage?: LanguageModelUsage }>

const transport = new DefaultChatTransport<ChatMessage>({ api: "/api/chat" })

function toUIMessages(column: CompareRequestColumn): ChatMessage[] {
  return column.messages.map((message, index) => ({
    id: `${column.column}-${index}`,
    role: message.role,
    parts: [{ type: "text", text: message.content }],
  }))
}

async function* streamColumn(request: CompareRequest, column: CompareRequestColumn): CompareStream {
  const stream = await transport.sendMessages({
    trigger: "submit-message",
    chatId: `${request.turn}-${column.column}`,
    messageId: undefined,
    messages: toUIMessages(column),
    abortSignal: column.signal,
    body: {
      model: column.model,
      system: column.settings.system,
      temperature: column.settings.temperature,
      maxOutputTokens: column.settings.maxTokens,
    },
  })
  const reader = stream.getReader()
  try {
    while (true) {
      const { done, value } = await reader.read()
      if (done) return
      const chunk: UIMessageChunk = value
      if (chunk.type === "text-delta") yield chunk.delta
      if (chunk.type === "error") throw new Error(chunk.errorText)
      if (chunk.type === "finish" || chunk.type === "message-metadata") {
        const usage = (chunk.messageMetadata as ChatMessage["metadata"])?.usage
        if (usage) yield { type: "usage", inputTokens: usage.inputTokens, outputTokens: usage.outputTokens }
      }
    }
  } finally {
    reader.releaseLock()
  }
}

export function Playground({
  models,
  prices,
  columns,
}: {
  models: ModelInfo[]
  prices: Record<string, PromptTokenPricing>
  columns: CompareColumn[]
}) {
  return (
    <ModelCompare
      models={models}
      prices={prices}
      columns={columns}
      onSend={(request) =>
        Object.fromEntries(request.columns.map((column) => [column.column, streamColumn(request, column)]))
      }
    />
  )
}

"use client"

import * as React from "react"

import type { ModelInfo } from "../models/models"
import type { PromptTokenPricing } from "../prompt-input/prompt-controls"
import {
  ModelCompare,
  readTextStream,
  type CompareColumn,
  type CompareRequest,
  type CompareVote,
} from "@/components/blocks/model-compare/model-compare"

function send(request: CompareRequest) {
  return Object.fromEntries(
    request.columns.map(({ column, model, settings, messages, signal }) => [
      column,
      (async function* () {
        const response = await fetch("/api/compare", {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ model, settings, messages }),
          signal,
        })
        yield* readTextStream(response)
      })(),
    ])
  )
}

async function saveVote(vote: CompareVote) {
  const response = await fetch("/api/preferences", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      turn: vote.turn,
      verdict: vote.verdict,
      winner: vote.winner?.model ?? null,
      models: vote.columns.map((column) => column.model),
      reason: vote.reason ?? null,
    }),
  })
  if (!response.ok) throw new Error("Your vote wasn’t saved. Try again.")
}

export function BlindEvaluation({
  models,
  prices,
  pairs,
}: {
  models: ModelInfo[]
  prices: Record<string, PromptTokenPricing>
  pairs: [string, string][]
}) {
  const [round, setRound] = React.useState(0)
  const pair = pairs[round % pairs.length]
  const columns: CompareColumn[] = React.useMemo(
    () => [
      { id: "left", model: pair[0] },
      { id: "right", model: pair[1] },
    ],
    [pair]
  )

  return (
    <ModelCompare
      key={round}
      blind
      models={models}
      prices={prices}
      columns={columns}
      onSend={send}
      onVote={async (vote) => {
        await saveVote(vote)
        setTimeout(() => setRound((value) => value + 1), 2500)
      }}
    />
  )
}

"use client"

import * as React from "react"

import type { ModelInfo } from "../models/models"
import type { PromptTokenPricing } from "../prompt-input/prompt-controls"
import { ModelCompare, type CompareColumn, type CompareTurn } from "@/components/blocks/model-compare/model-compare"

type SharedComparison = {
  columns: CompareColumn[]
  turns: CompareTurn[]
}

export function SharedComparison({
  comparison,
  models,
  prices,
}: {
  comparison: SharedComparison
  models: ModelInfo[]
  prices: Record<string, PromptTokenPricing>
}) {
  return (
    <ModelCompare
      readOnly
      models={models}
      prices={prices}
      columns={comparison.columns}
      turns={comparison.turns}
    />
  )
}

export function SaveForSharing({
  models,
  prices,
  columns,
  onSave,
}: {
  models: ModelInfo[]
  prices: Record<string, PromptTokenPricing>
  columns: CompareColumn[]
  onSave: (comparison: SharedComparison) => void
}) {
  const current = React.useRef(columns)
  return (
    <ModelCompare
      models={models}
      prices={prices}
      columns={columns}
      onColumnsChange={(next) => {
        current.current = next
      }}
      onTurnsChange={(turns) => {
        if (turns.every((turn) => Object.values(turn.answers).every((answer) => answer.status === "done"))) {
          onSave({ columns: current.current, turns })
        }
      }}
    />
  )
}