fix: add fallback resolution warnings for unavailable models

This commit is contained in:
YeonGyu-Kim
2026-02-17 10:29:48 +09:00
parent 72fa2c7e65
commit ca06ce134f
3 changed files with 146 additions and 11 deletions

View File

@@ -2,29 +2,64 @@ import { readConnectedProvidersCache } from "./connected-providers-cache"
import { log } from "./logger"
import { fuzzyMatchModel } from "./model-name-matcher"
export function isAnyFallbackModelAvailable(
fallbackChain: Array<{ providers: string[]; model: string }>,
type FallbackEntry = { providers: string[]; model: string }
type ResolvedFallbackModel = {
provider: string
model: string
}
export function resolveFirstAvailableFallback(
fallbackChain: FallbackEntry[],
availableModels: Set<string>,
): boolean {
if (availableModels.size > 0) {
for (const entry of fallbackChain) {
const hasAvailableProvider = entry.providers.some((provider) => {
return fuzzyMatchModel(entry.model, availableModels, [provider]) !== null
): ResolvedFallbackModel | null {
for (const entry of fallbackChain) {
for (const provider of entry.providers) {
const matchedModel = fuzzyMatchModel(entry.model, availableModels, [provider])
log("[resolveFirstAvailableFallback] attempt", {
provider,
requestedModel: entry.model,
resolvedModel: matchedModel,
})
if (hasAvailableProvider) {
return true
if (matchedModel !== null) {
log("[resolveFirstAvailableFallback] resolved", {
provider,
requestedModel: entry.model,
resolvedModel: matchedModel,
})
return { provider, model: matchedModel }
}
}
}
log("[resolveFirstAvailableFallback] WARNING: no fallback model resolved", {
chain: fallbackChain.map((entry) => ({
model: entry.model,
providers: entry.providers,
})),
availableCount: availableModels.size,
})
return null
}
export function isAnyFallbackModelAvailable(
fallbackChain: FallbackEntry[],
availableModels: Set<string>,
): boolean {
if (resolveFirstAvailableFallback(fallbackChain, availableModels) !== null) {
return true
}
const connectedProviders = readConnectedProvidersCache()
if (connectedProviders) {
const connectedSet = new Set(connectedProviders)
for (const entry of fallbackChain) {
if (entry.providers.some((p) => connectedSet.has(p))) {
log(
"[isAnyFallbackModelAvailable] model not in available set, but provider is connected",
{ model: entry.model, availableCount: availableModels.size },
"[isAnyFallbackModelAvailable] WARNING: No fuzzy match found for any model in fallback chain, but provider is connected. Agent may fail at runtime.",
{ chain: fallbackChain.map((entryItem) => entryItem.model), availableCount: availableModels.size },
)
return true
}

View File

@@ -9,6 +9,14 @@ let fetchAvailableModels: (client?: unknown, options?: { connectedProviders?: st
let fuzzyMatchModel: (target: string, available: Set<string>, providers?: string[]) => string | null
let isModelAvailable: (targetModel: string, availableModels: Set<string>) => boolean
let getConnectedProviders: (client: unknown) => Promise<string[]>
let isAnyFallbackModelAvailable: (
fallbackChain: Array<{ providers: string[]; model: string }>,
availableModels: Set<string>,
) => boolean
let resolveFirstAvailableFallback: (
fallbackChain: Array<{ providers: string[]; model: string }>,
availableModels: Set<string>,
) => { provider: string; model: string } | null
beforeAll(async () => {
;({
@@ -18,6 +26,10 @@ beforeAll(async () => {
isModelAvailable,
getConnectedProviders,
} = await import("./model-availability"))
;({
isAnyFallbackModelAvailable,
resolveFirstAvailableFallback,
} = await import("./fallback-model-availability"))
})
describe("fetchAvailableModels", () => {
@@ -863,3 +875,90 @@ describe("isModelAvailable", () => {
expect(result).toBe(false)
})
})
describe("fallback model availability", () => {
let tempDir: string
let originalXdgCache: string | undefined
beforeEach(() => {
// given
tempDir = mkdtempSync(join(tmpdir(), "opencode-test-"))
originalXdgCache = process.env.XDG_CACHE_HOME
process.env.XDG_CACHE_HOME = tempDir
})
afterEach(() => {
if (originalXdgCache !== undefined) {
process.env.XDG_CACHE_HOME = originalXdgCache
} else {
delete process.env.XDG_CACHE_HOME
}
rmSync(tempDir, { recursive: true, force: true })
})
function writeConnectedProvidersCache(connected: string[]): void {
const cacheDir = join(tempDir, "oh-my-opencode")
require("fs").mkdirSync(cacheDir, { recursive: true })
writeFileSync(
join(cacheDir, "connected-providers.json"),
JSON.stringify({ connected, updatedAt: new Date().toISOString() }),
)
}
it("returns null for completely unknown model", () => {
// given
const available = new Set(["openai/gpt-5.2", "anthropic/claude-opus-4-6"])
// when
const result = fuzzyMatchModel("non-existent-model-family", available)
// then
expect(result).toBeNull()
})
it("returns true when models do not match but provider is connected", () => {
// given
const fallbackChain = [{ providers: ["openai"], model: "gpt-5.2" }]
const availableModels = new Set(["anthropic/claude-opus-4-6"])
writeConnectedProvidersCache(["openai"])
// when
const result = isAnyFallbackModelAvailable(fallbackChain, availableModels)
// then
expect(result).toBe(true)
})
it("returns first resolved fallback model from chain", () => {
// given
const fallbackChain = [
{ providers: ["openai"], model: "gpt-5.2" },
{ providers: ["anthropic"], model: "claude-opus-4-6" },
]
const availableModels = new Set([
"anthropic/claude-opus-4-6",
"openai/gpt-5.2-preview",
])
// when
const result = resolveFirstAvailableFallback(fallbackChain, availableModels)
// then
expect(result).toEqual({ provider: "openai", model: "openai/gpt-5.2-preview" })
})
it("returns null when no fallback model resolves", () => {
// given
const fallbackChain = [
{ providers: ["openai"], model: "gpt-5.2" },
{ providers: ["anthropic"], model: "claude-opus-4-6" },
]
const availableModels = new Set(["google/gemini-3-pro"])
// when
const result = resolveFirstAvailableFallback(fallbackChain, availableModels)
// then
expect(result).toBeNull()
})
})

View File

@@ -69,6 +69,7 @@ export function fuzzyMatchModel(
log("[fuzzyMatchModel] substring matches", { targetNormalized, matchCount: matches.length, matches })
if (matches.length === 0) {
log("[fuzzyMatchModel] WARNING: no match found", { target, availableCount: available.size, providers })
return null
}