diff --git a/.github/workflows/refresh-model-capabilities.yml b/.github/workflows/refresh-model-capabilities.yml new file mode 100644 index 000000000..5d2d053fa --- /dev/null +++ b/.github/workflows/refresh-model-capabilities.yml @@ -0,0 +1,43 @@ +name: Refresh Model Capabilities + +on: + schedule: + - cron: "17 4 * * 1" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + refresh: + runs-on: ubuntu-latest + if: github.repository == 'code-yeongyu/oh-my-openagent' + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + env: + BUN_INSTALL_ALLOW_SCRIPTS: "@ast-grep/napi" + + - name: Refresh bundled model capabilities snapshot + run: bun run build:model-capabilities + + - name: Create refresh pull request + uses: peter-evans/create-pull-request@v7 + with: + commit-message: "chore: refresh model capabilities snapshot" + title: "chore: refresh model capabilities snapshot" + body: | + Automated refresh of `src/generated/model-capabilities.generated.json` from `https://models.dev/api.json`. + + This keeps the bundled capability snapshot aligned with upstream model metadata without relying on manual refreshes. + branch: automation/refresh-model-capabilities + delete-branch: true + labels: | + maintenance diff --git a/src/cli/doctor/checks/model-resolution-details.ts b/src/cli/doctor/checks/model-resolution-details.ts index e96655476..3443e92b1 100644 --- a/src/cli/doctor/checks/model-resolution-details.ts +++ b/src/cli/doctor/checks/model-resolution-details.ts @@ -4,6 +4,10 @@ import { getOpenCodeCacheDir } from "../../../shared" import type { AvailableModelsInfo, ModelResolutionInfo, OmoConfig } from "./model-resolution-types" import { formatModelWithVariant, getCategoryEffectiveVariant, getEffectiveVariant } from "./model-resolution-variant" +function formatCapabilityResolutionLabel(mode: string | undefined): string { + return mode ?? "unknown" +} + export function buildModelResolutionDetails(options: { info: ModelResolutionInfo available: AvailableModelsInfo @@ -37,7 +41,7 @@ export function buildModelResolutionDetails(options: { agent.effectiveModel, getEffectiveVariant(agent.name, agent.requirement, options.config) ) - details.push(` ${marker} ${agent.name}: ${display}`) + details.push(` ${marker} ${agent.name}: ${display} [capabilities: ${formatCapabilityResolutionLabel(agent.capabilityDiagnostics?.resolutionMode)}]`) } details.push("") details.push("Categories:") @@ -47,7 +51,7 @@ export function buildModelResolutionDetails(options: { category.effectiveModel, getCategoryEffectiveVariant(category.name, category.requirement, options.config) ) - details.push(` ${marker} ${category.name}: ${display}`) + details.push(` ${marker} ${category.name}: ${display} [capabilities: ${formatCapabilityResolutionLabel(category.capabilityDiagnostics?.resolutionMode)}]`) } details.push("") details.push("● = user override, ○ = provider fallback") diff --git a/src/cli/doctor/checks/model-resolution-types.ts b/src/cli/doctor/checks/model-resolution-types.ts index c0396d958..2e77fddd1 100644 --- a/src/cli/doctor/checks/model-resolution-types.ts +++ b/src/cli/doctor/checks/model-resolution-types.ts @@ -1,3 +1,4 @@ +import type { ModelCapabilitiesDiagnostics } from "../../../shared/model-capabilities" import type { ModelRequirement } from "../../../shared/model-requirements" export interface AgentResolutionInfo { @@ -7,6 +8,7 @@ export interface AgentResolutionInfo { userVariant?: string effectiveModel: string effectiveResolution: string + capabilityDiagnostics?: ModelCapabilitiesDiagnostics } export interface CategoryResolutionInfo { @@ -16,6 +18,7 @@ export interface CategoryResolutionInfo { userVariant?: string effectiveModel: string effectiveResolution: string + capabilityDiagnostics?: ModelCapabilitiesDiagnostics } export interface ModelResolutionInfo { diff --git a/src/cli/doctor/checks/model-resolution.test.ts b/src/cli/doctor/checks/model-resolution.test.ts index 902c92cfe..696e8c4d4 100644 --- a/src/cli/doctor/checks/model-resolution.test.ts +++ b/src/cli/doctor/checks/model-resolution.test.ts @@ -129,6 +129,19 @@ describe("model-resolution check", () => { expect(visual!.userOverride).toBe("google/gemini-3-flash-preview") expect(visual!.userVariant).toBe("high") }) + + it("attaches snapshot-backed capability diagnostics for built-in models", async () => { + const { getModelResolutionInfoWithOverrides } = await import("./model-resolution") + + const info = getModelResolutionInfoWithOverrides({}) + const sisyphus = info.agents.find((a) => a.name === "sisyphus") + + expect(sisyphus).toBeDefined() + expect(sisyphus!.capabilityDiagnostics).toMatchObject({ + resolutionMode: "snapshot-backed", + snapshot: { source: "bundled-snapshot" }, + }) + }) }) describe("checkModelResolution", () => { @@ -162,6 +175,23 @@ describe("model-resolution check", () => { expect(result.details!.some((d) => d.includes("Categories:"))).toBe(true) // Should have legend expect(result.details!.some((d) => d.includes("user override"))).toBe(true) + expect(result.details!.some((d) => d.includes("capabilities: snapshot-backed"))).toBe(true) + }) + + it("collects warnings when configured models rely on compatibility fallback", async () => { + const { collectCapabilityResolutionIssues, getModelResolutionInfoWithOverrides } = await import("./model-resolution") + + const info = getModelResolutionInfoWithOverrides({ + agents: { + oracle: { model: "custom/unknown-llm" }, + }, + }) + + const issues = collectCapabilityResolutionIssues(info) + + expect(issues).toHaveLength(1) + expect(issues[0]?.title).toContain("compatibility fallback") + expect(issues[0]?.description).toContain("oracle=custom/unknown-llm") }) }) diff --git a/src/cli/doctor/checks/model-resolution.ts b/src/cli/doctor/checks/model-resolution.ts index c9cc0c0b0..706b18f4b 100644 --- a/src/cli/doctor/checks/model-resolution.ts +++ b/src/cli/doctor/checks/model-resolution.ts @@ -1,4 +1,5 @@ import { AGENT_MODEL_REQUIREMENTS, CATEGORY_MODEL_REQUIREMENTS } from "../../../shared/model-requirements" +import { getModelCapabilities } from "../../../shared/model-capabilities" import { CHECK_IDS, CHECK_NAMES } from "../constants" import type { CheckResult, DoctorIssue } from "../types" import { loadAvailableModelsFromCache } from "./model-resolution-cache" @@ -7,16 +8,36 @@ import { buildModelResolutionDetails } from "./model-resolution-details" import { buildEffectiveResolution, getEffectiveModel } from "./model-resolution-effective-model" import type { AgentResolutionInfo, CategoryResolutionInfo, ModelResolutionInfo, OmoConfig } from "./model-resolution-types" -export function getModelResolutionInfo(): ModelResolutionInfo { - const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map(([name, requirement]) => ({ - name, - requirement, - effectiveModel: getEffectiveModel(requirement), - effectiveResolution: buildEffectiveResolution(requirement), - })) +function parseProviderModel(value: string): { providerID: string; modelID: string } | null { + const slashIndex = value.indexOf("/") + if (slashIndex <= 0 || slashIndex === value.length - 1) { + return null + } - const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map( - ([name, requirement]) => ({ + return { + providerID: value.slice(0, slashIndex), + modelID: value.slice(slashIndex + 1), + } +} + +function attachCapabilityDiagnostics(entry: T): T { + const parsed = parseProviderModel(entry.effectiveModel) + if (!parsed) { + return entry + } + + return { + ...entry, + capabilityDiagnostics: getModelCapabilities({ + providerID: parsed.providerID, + modelID: parsed.modelID, + }).diagnostics, + } +} + +export function getModelResolutionInfo(): ModelResolutionInfo { + const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map(([name, requirement]) => + attachCapabilityDiagnostics({ name, requirement, effectiveModel: getEffectiveModel(requirement), @@ -24,6 +45,16 @@ export function getModelResolutionInfo(): ModelResolutionInfo { }) ) + const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map( + ([name, requirement]) => + attachCapabilityDiagnostics({ + name, + requirement, + effectiveModel: getEffectiveModel(requirement), + effectiveResolution: buildEffectiveResolution(requirement), + }) + ) + return { agents, categories } } @@ -31,34 +62,60 @@ export function getModelResolutionInfoWithOverrides(config: OmoConfig): ModelRes const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map(([name, requirement]) => { const userOverride = config.agents?.[name]?.model const userVariant = config.agents?.[name]?.variant - return { + return attachCapabilityDiagnostics({ name, requirement, userOverride, userVariant, effectiveModel: getEffectiveModel(requirement, userOverride), effectiveResolution: buildEffectiveResolution(requirement, userOverride), - } + }) }) const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map( ([name, requirement]) => { const userOverride = config.categories?.[name]?.model const userVariant = config.categories?.[name]?.variant - return { + return attachCapabilityDiagnostics({ name, requirement, userOverride, userVariant, effectiveModel: getEffectiveModel(requirement, userOverride), effectiveResolution: buildEffectiveResolution(requirement, userOverride), - } + }) } ) return { agents, categories } } +export function collectCapabilityResolutionIssues(info: ModelResolutionInfo): DoctorIssue[] { + const issues: DoctorIssue[] = [] + const allEntries = [...info.agents, ...info.categories] + const fallbackEntries = allEntries.filter((entry) => { + const mode = entry.capabilityDiagnostics?.resolutionMode + return mode === "alias-backed" || mode === "heuristic-backed" || mode === "unknown" + }) + + if (fallbackEntries.length === 0) { + return issues + } + + const summary = fallbackEntries + .map((entry) => `${entry.name}=${entry.effectiveModel} (${entry.capabilityDiagnostics?.resolutionMode ?? "unknown"})`) + .join(", ") + + issues.push({ + title: "Configured models rely on compatibility fallback", + description: summary, + severity: "warning", + affects: fallbackEntries.map((entry) => entry.name), + }) + + return issues +} + export async function checkModels(): Promise { const config = loadOmoConfig() ?? {} const info = getModelResolutionInfoWithOverrides(config) @@ -75,6 +132,8 @@ export async function checkModels(): Promise { }) } + issues.push(...collectCapabilityResolutionIssues(info)) + const overrideCount = info.agents.filter((agent) => Boolean(agent.userOverride)).length + info.categories.filter((category) => Boolean(category.userOverride)).length diff --git a/src/shared/model-capabilities.test.ts b/src/shared/model-capabilities.test.ts index a145aab45..afad4ba0a 100644 --- a/src/shared/model-capabilities.test.ts +++ b/src/shared/model-capabilities.test.ts @@ -2,8 +2,10 @@ import { describe, expect, test } from "bun:test" import { getModelCapabilities, + getBundledModelCapabilitiesSnapshot, type ModelCapabilitiesSnapshot, } from "./model-capabilities" +import { AGENT_MODEL_REQUIREMENTS, CATEGORY_MODEL_REQUIREMENTS } from "./model-requirements" describe("getModelCapabilities", () => { const bundledSnapshot: ModelCapabilitiesSnapshot = { @@ -79,6 +81,12 @@ describe("getModelCapabilities", () => { maxOutputTokens: 128_000, toolCall: true, }) + expect(result.diagnostics).toMatchObject({ + resolutionMode: "snapshot-backed", + canonicalization: { source: "canonical" }, + snapshot: { source: "bundled-snapshot" }, + variants: { source: "runtime" }, + }) }) test("reads structured runtime capabilities from the SDK v2 shape", () => { @@ -113,6 +121,12 @@ describe("getModelCapabilities", () => { output: ["text"], }, }) + expect(result.diagnostics).toMatchObject({ + resolutionMode: "snapshot-backed", + reasoning: { source: "runtime" }, + supportsThinking: { source: "runtime" }, + toolCall: { source: "runtime" }, + }) }) test("respects root-level thinking flags when providers do not nest them under capabilities", () => { @@ -129,6 +143,9 @@ describe("getModelCapabilities", () => { canonicalModelID: "gpt-5.4", supportsThinking: true, }) + expect(result.diagnostics).toMatchObject({ + supportsThinking: { source: "runtime" }, + }) }) test("accepts runtime variant arrays without corrupting them into numeric keys", () => { @@ -158,6 +175,14 @@ describe("getModelCapabilities", () => { supportsTemperature: true, maxOutputTokens: 128_000, }) + expect(result.diagnostics).toMatchObject({ + resolutionMode: "alias-backed", + canonicalization: { + source: "pattern-alias", + ruleID: "anthropic-thinking-suffix", + }, + snapshot: { source: "bundled-snapshot" }, + }) }) test("maps local gemini aliases to canonical models.dev entries", () => { @@ -174,6 +199,14 @@ describe("getModelCapabilities", () => { supportsTemperature: true, maxOutputTokens: 65_000, }) + expect(result.diagnostics).toMatchObject({ + resolutionMode: "alias-backed", + canonicalization: { + source: "exact-alias", + ruleID: "gemini-3.1-pro-tier-alias", + }, + snapshot: { source: "bundled-snapshot" }, + }) }) test("prefers runtime models.dev cache over bundled snapshot", () => { @@ -203,6 +236,11 @@ describe("getModelCapabilities", () => { maxOutputTokens: 64_000, supportsTemperature: false, }) + expect(result.diagnostics).toMatchObject({ + snapshot: { source: "runtime-snapshot" }, + maxOutputTokens: { source: "runtime-snapshot" }, + supportsTemperature: { source: "runtime-snapshot" }, + }) }) test("falls back to heuristic family rules when no snapshot entry exists", () => { @@ -218,6 +256,12 @@ describe("getModelCapabilities", () => { variants: ["low", "medium", "high"], reasoningEfforts: ["none", "minimal", "low", "medium", "high"], }) + expect(result.diagnostics).toMatchObject({ + resolutionMode: "heuristic-backed", + snapshot: { source: "none" }, + family: { source: "heuristic" }, + reasoningEfforts: { source: "heuristic" }, + }) }) test("detects prefixed o-series model IDs through the heuristic fallback", () => { @@ -233,5 +277,34 @@ describe("getModelCapabilities", () => { variants: ["low", "medium", "high"], reasoningEfforts: ["none", "minimal", "low", "medium", "high"], }) + expect(result.diagnostics).toMatchObject({ + resolutionMode: "heuristic-backed", + snapshot: { source: "none" }, + family: { source: "heuristic" }, + }) + }) + + test("keeps every built-in OmO requirement model snapshot-backed", () => { + const bundledSnapshot = getBundledModelCapabilitiesSnapshot() + const requirementModels = new Set() + + for (const requirement of Object.values(AGENT_MODEL_REQUIREMENTS)) { + for (const entry of requirement.fallbackChain) requirementModels.add(entry.model) + } + + for (const requirement of Object.values(CATEGORY_MODEL_REQUIREMENTS)) { + for (const entry of requirement.fallbackChain) requirementModels.add(entry.model) + } + + for (const modelID of requirementModels) { + const result = getModelCapabilities({ + providerID: "test-provider", + modelID, + bundledSnapshot, + }) + + expect(result.diagnostics.resolutionMode).toBe("snapshot-backed") + expect(result.diagnostics.snapshot.source).toBe("bundled-snapshot") + } }) }) diff --git a/src/shared/model-capabilities.ts b/src/shared/model-capabilities.ts index 25835f5d5..0a9749243 100644 --- a/src/shared/model-capabilities.ts +++ b/src/shared/model-capabilities.ts @@ -1,5 +1,6 @@ import bundledModelCapabilitiesSnapshotJson from "../generated/model-capabilities.generated.json" import { findProviderModelMetadata, type ModelMetadata } from "./connected-providers-cache" +import { resolveModelIDAlias } from "./model-capability-aliases" import { detectHeuristicModelFamily } from "./model-capability-heuristics" export type ModelCapabilitiesSnapshotEntry = { @@ -41,6 +42,7 @@ export type ModelCapabilities = { input?: string[] output?: string[] } + diagnostics: ModelCapabilitiesDiagnostics } type GetModelCapabilitiesInput = { @@ -52,7 +54,6 @@ type GetModelCapabilitiesInput = { } type ModelCapabilityOverride = { - canonicalModelID?: string variants?: string[] reasoningEfforts?: string[] supportsThinking?: boolean @@ -60,17 +61,40 @@ type ModelCapabilityOverride = { supportsTopP?: boolean } -const MODEL_ID_OVERRIDES: Record = { - "claude-opus-4-6-thinking": { canonicalModelID: "claude-opus-4-6" }, - "claude-sonnet-4-6-thinking": { canonicalModelID: "claude-sonnet-4-6" }, - "claude-opus-4-5-thinking": { canonicalModelID: "claude-opus-4-5-20251101" }, - "gpt-5.3-codex-spark": { canonicalModelID: "gpt-5.3-codex" }, - "gemini-3.1-pro-high": { canonicalModelID: "gemini-3.1-pro-preview" }, - "gemini-3.1-pro-low": { canonicalModelID: "gemini-3.1-pro-preview" }, - "gemini-3-pro-high": { canonicalModelID: "gemini-3-pro-preview" }, - "gemini-3-pro-low": { canonicalModelID: "gemini-3-pro-preview" }, +type DiagnosticSource = + | "none" + | "runtime" + | "runtime-snapshot" + | "bundled-snapshot" + | "override" + | "heuristic" + | "canonical" + | "exact-alias" + | "pattern-alias" + +export type ModelCapabilitiesDiagnostics = { + resolutionMode: "snapshot-backed" | "alias-backed" | "heuristic-backed" | "unknown" + canonicalization: { + source: "canonical" | "exact-alias" | "pattern-alias" + ruleID?: string + } + snapshot: { + source: "runtime-snapshot" | "bundled-snapshot" | "none" + } + family: { source: "snapshot" | "heuristic" | "none" } + variants: { source: Exclude } + reasoningEfforts: { source: Exclude } + reasoning: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } + supportsThinking: { source: "runtime" | "override" | "heuristic" | "runtime-snapshot" | "bundled-snapshot" | "none" } + supportsTemperature: { source: "runtime" | "override" | "runtime-snapshot" | "bundled-snapshot" | "none" } + supportsTopP: { source: "runtime" | "override" | "none" } + maxOutputTokens: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } + toolCall: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } + modalities: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } } +const MODEL_ID_OVERRIDES: Record = {} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value) } @@ -149,20 +173,6 @@ function normalizeSnapshot(snapshot: ModelCapabilitiesSnapshot | typeof bundledM return snapshot as ModelCapabilitiesSnapshot } -function getCanonicalModelID(modelID: string): string { - const normalizedModelID = normalizeLookupModelID(modelID) - const override = MODEL_ID_OVERRIDES[normalizedModelID] - if (override?.canonicalModelID) { - return override.canonicalModelID - } - - if (normalizedModelID.startsWith("claude-") && normalizedModelID.endsWith("-thinking")) { - return normalizedModelID.replace(/-thinking$/i, "") - } - - return normalizedModelID -} - function getOverride(modelID: string): ModelCapabilityOverride | undefined { return MODEL_ID_OVERRIDES[normalizeLookupModelID(modelID)] } @@ -307,8 +317,9 @@ export function getBundledModelCapabilitiesSnapshot(): ModelCapabilitiesSnapshot } export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCapabilities { - const requestedModelID = normalizeLookupModelID(input.modelID) - const canonicalModelID = getCanonicalModelID(input.modelID) + const canonicalization = resolveModelIDAlias(input.modelID) + const requestedModelID = canonicalization.requestedModelID + const canonicalModelID = canonicalization.canonicalModelID const override = getOverride(input.modelID) const runtimeModel = readRuntimeModel( input.runtimeModel ?? findProviderModelMetadata(input.providerID, input.modelID), @@ -318,6 +329,88 @@ export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCap const snapshotEntry = runtimeSnapshot?.models?.[canonicalModelID] ?? bundledSnapshot.models[canonicalModelID] const heuristicFamily = detectHeuristicModelFamily(canonicalModelID) const runtimeVariants = readRuntimeModelVariants(runtimeModel) + const snapshotSource: ModelCapabilitiesDiagnostics["snapshot"]["source"] = + runtimeSnapshot?.models?.[canonicalModelID] + ? "runtime-snapshot" + : bundledSnapshot.models[canonicalModelID] + ? "bundled-snapshot" + : "none" + const familySource: ModelCapabilitiesDiagnostics["family"]["source"] = + snapshotEntry?.family + ? "snapshot" + : heuristicFamily?.family + ? "heuristic" + : "none" + const variantsSource: ModelCapabilitiesDiagnostics["variants"]["source"] = + runtimeVariants + ? "runtime" + : override?.variants + ? "override" + : heuristicFamily?.variants + ? "heuristic" + : "none" + const reasoningEffortsSource: ModelCapabilitiesDiagnostics["reasoningEfforts"]["source"] = + override?.reasoningEfforts + ? "override" + : heuristicFamily?.reasoningEfforts + ? "heuristic" + : "none" + const reasoningSource: ModelCapabilitiesDiagnostics["reasoning"]["source"] = + readRuntimeModelReasoningSupport(runtimeModel) !== undefined + ? "runtime" + : snapshotEntry?.reasoning !== undefined + ? snapshotSource + : "none" + const supportsThinkingSource: ModelCapabilitiesDiagnostics["supportsThinking"]["source"] = + override?.supportsThinking !== undefined + ? "override" + : heuristicFamily?.supportsThinking !== undefined + ? "heuristic" + : readRuntimeModelThinkingSupport(runtimeModel) !== undefined + ? "runtime" + : snapshotEntry?.reasoning !== undefined + ? snapshotSource + : "none" + const supportsTemperatureSource: ModelCapabilitiesDiagnostics["supportsTemperature"]["source"] = + readRuntimeModelTemperatureSupport(runtimeModel) !== undefined + ? "runtime" + : override?.supportsTemperature !== undefined + ? "override" + : snapshotEntry?.temperature !== undefined + ? snapshotSource + : "none" + const supportsTopPSource: ModelCapabilitiesDiagnostics["supportsTopP"]["source"] = + readRuntimeModelTopPSupport(runtimeModel) !== undefined + ? "runtime" + : override?.supportsTopP !== undefined + ? "override" + : "none" + const maxOutputTokensSource: ModelCapabilitiesDiagnostics["maxOutputTokens"]["source"] = + readRuntimeModelLimitOutput(runtimeModel) !== undefined + ? "runtime" + : snapshotEntry?.limit?.output !== undefined + ? snapshotSource + : "none" + const toolCallSource: ModelCapabilitiesDiagnostics["toolCall"]["source"] = + readRuntimeModelToolCallSupport(runtimeModel) !== undefined + ? "runtime" + : snapshotEntry?.toolCall !== undefined + ? snapshotSource + : "none" + const modalitiesSource: ModelCapabilitiesDiagnostics["modalities"]["source"] = + readRuntimeModelModalities(runtimeModel) !== undefined + ? "runtime" + : snapshotEntry?.modalities !== undefined + ? snapshotSource + : "none" + const resolutionMode: ModelCapabilitiesDiagnostics["resolutionMode"] = + snapshotSource !== "none" && canonicalization.source === "canonical" + ? "snapshot-backed" + : snapshotSource !== "none" + ? "alias-backed" + : familySource === "heuristic" || variantsSource === "heuristic" || reasoningEffortsSource === "heuristic" + ? "heuristic-backed" + : "unknown" return { requestedModelID, @@ -347,5 +440,23 @@ export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCap modalities: readRuntimeModelModalities(runtimeModel) ?? snapshotEntry?.modalities, + diagnostics: { + resolutionMode, + canonicalization: { + source: canonicalization.source, + ...(canonicalization.ruleID ? { ruleID: canonicalization.ruleID } : {}), + }, + snapshot: { source: snapshotSource }, + family: { source: familySource }, + variants: { source: variantsSource }, + reasoningEfforts: { source: reasoningEffortsSource }, + reasoning: { source: reasoningSource }, + supportsThinking: { source: supportsThinkingSource }, + supportsTemperature: { source: supportsTemperatureSource }, + supportsTopP: { source: supportsTopPSource }, + maxOutputTokens: { source: maxOutputTokensSource }, + toolCall: { source: toolCallSource }, + modalities: { source: modalitiesSource }, + }, } } diff --git a/src/shared/model-capability-aliases.test.ts b/src/shared/model-capability-aliases.test.ts new file mode 100644 index 000000000..31be1852a --- /dev/null +++ b/src/shared/model-capability-aliases.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from "bun:test" + +import { resolveModelIDAlias } from "./model-capability-aliases" + +describe("model-capability-aliases", () => { + test("keeps canonical model IDs unchanged", () => { + const result = resolveModelIDAlias("gpt-5.4") + + expect(result).toEqual({ + requestedModelID: "gpt-5.4", + canonicalModelID: "gpt-5.4", + source: "canonical", + }) + }) + + test("normalizes exact local tier aliases to canonical models.dev IDs", () => { + const result = resolveModelIDAlias("gemini-3.1-pro-high") + + expect(result).toEqual({ + requestedModelID: "gemini-3.1-pro-high", + canonicalModelID: "gemini-3.1-pro-preview", + source: "exact-alias", + ruleID: "gemini-3.1-pro-tier-alias", + }) + }) + + test("normalizes decorated thinking aliases through a named pattern rule", () => { + const result = resolveModelIDAlias("claude-opus-4-6-thinking") + + expect(result).toEqual({ + requestedModelID: "claude-opus-4-6-thinking", + canonicalModelID: "claude-opus-4-6", + source: "pattern-alias", + ruleID: "anthropic-thinking-suffix", + }) + }) +}) diff --git a/src/shared/model-capability-aliases.ts b/src/shared/model-capability-aliases.ts new file mode 100644 index 000000000..92454ad5d --- /dev/null +++ b/src/shared/model-capability-aliases.ts @@ -0,0 +1,84 @@ +type ExactAliasRule = { + ruleID: string + canonicalModelID: string +} + +type PatternAliasRule = { + ruleID: string + match: (normalizedModelID: string) => boolean + canonicalize: (normalizedModelID: string) => string +} + +export type ModelIDAliasResolution = { + requestedModelID: string + canonicalModelID: string + source: "canonical" | "exact-alias" | "pattern-alias" + ruleID?: string +} + +const EXACT_ALIAS_RULES: Record = { + "gpt-5.3-codex-spark": { + ruleID: "gpt-5.3-codex-spark-alias", + canonicalModelID: "gpt-5.3-codex", + }, + "gemini-3.1-pro-high": { + ruleID: "gemini-3.1-pro-tier-alias", + canonicalModelID: "gemini-3.1-pro-preview", + }, + "gemini-3.1-pro-low": { + ruleID: "gemini-3.1-pro-tier-alias", + canonicalModelID: "gemini-3.1-pro-preview", + }, + "gemini-3-pro-high": { + ruleID: "gemini-3-pro-tier-alias", + canonicalModelID: "gemini-3-pro-preview", + }, + "gemini-3-pro-low": { + ruleID: "gemini-3-pro-tier-alias", + canonicalModelID: "gemini-3-pro-preview", + }, +} + +const PATTERN_ALIAS_RULES: ReadonlyArray = [ + { + ruleID: "anthropic-thinking-suffix", + match: (normalizedModelID) => normalizedModelID.startsWith("claude-") && normalizedModelID.endsWith("-thinking"), + canonicalize: (normalizedModelID) => normalizedModelID.replace(/-thinking$/i, ""), + }, +] + +function normalizeLookupModelID(modelID: string): string { + return modelID.trim().toLowerCase() +} + +export function resolveModelIDAlias(modelID: string): ModelIDAliasResolution { + const normalizedModelID = normalizeLookupModelID(modelID) + const exactRule = EXACT_ALIAS_RULES[normalizedModelID] + if (exactRule) { + return { + requestedModelID: normalizedModelID, + canonicalModelID: exactRule.canonicalModelID, + source: "exact-alias", + ruleID: exactRule.ruleID, + } + } + + for (const rule of PATTERN_ALIAS_RULES) { + if (!rule.match(normalizedModelID)) { + continue + } + + return { + requestedModelID: normalizedModelID, + canonicalModelID: rule.canonicalize(normalizedModelID), + source: "pattern-alias", + ruleID: rule.ruleID, + } + } + + return { + requestedModelID: normalizedModelID, + canonicalModelID: normalizedModelID, + source: "canonical", + } +}