Compare commits
7 Commits
fix/issue-
...
fix/issue-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1946a6397 | ||
|
|
6edbc9638a | ||
|
|
af32f2e258 | ||
|
|
e7e6bd0608 | ||
|
|
7f86103666 | ||
|
|
2c6ba98920 | ||
|
|
80dee4d2c9 |
@@ -1,96 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test"
|
||||
|
||||
import * as loggerModule from "../../shared/logger"
|
||||
import * as spawnWithWindowsHideModule from "../../shared/spawn-with-windows-hide"
|
||||
import { resetConfigContext } from "./config-context"
|
||||
import { runBunInstallWithDetails } from "./bun-install"
|
||||
|
||||
function createProc(
|
||||
exitCode: number,
|
||||
output?: { stdout?: string; stderr?: string }
|
||||
): ReturnType<typeof spawnWithWindowsHideModule.spawnWithWindowsHide> {
|
||||
return {
|
||||
exited: Promise.resolve(exitCode),
|
||||
exitCode,
|
||||
stdout: output?.stdout ? new Blob([output.stdout]).stream() : undefined,
|
||||
stderr: output?.stderr ? new Blob([output.stderr]).stream() : undefined,
|
||||
kill: () => {},
|
||||
} satisfies ReturnType<typeof spawnWithWindowsHideModule.spawnWithWindowsHide>
|
||||
}
|
||||
|
||||
describe("runBunInstallWithDetails", () => {
|
||||
beforeEach(() => {
|
||||
process.env.OPENCODE_CONFIG_DIR = "/test/opencode"
|
||||
resetConfigContext()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
resetConfigContext()
|
||||
delete process.env.OPENCODE_CONFIG_DIR
|
||||
})
|
||||
|
||||
it("inherits install output by default", async () => {
|
||||
// given
|
||||
const spawnSpy = spyOn(spawnWithWindowsHideModule, "spawnWithWindowsHide").mockReturnValue(createProc(0))
|
||||
|
||||
try {
|
||||
// when
|
||||
const result = await runBunInstallWithDetails()
|
||||
|
||||
// then
|
||||
expect(result).toEqual({ success: true })
|
||||
const [_, options] = spawnSpy.mock.calls[0] as Parameters<typeof spawnWithWindowsHideModule.spawnWithWindowsHide>
|
||||
expect(options.stdout).toBe("inherit")
|
||||
expect(options.stderr).toBe("inherit")
|
||||
} finally {
|
||||
spawnSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it("pipes install output when requested", async () => {
|
||||
// given
|
||||
const spawnSpy = spyOn(spawnWithWindowsHideModule, "spawnWithWindowsHide").mockReturnValue(createProc(0))
|
||||
|
||||
try {
|
||||
// when
|
||||
const result = await runBunInstallWithDetails({ outputMode: "pipe" })
|
||||
|
||||
// then
|
||||
expect(result).toEqual({ success: true })
|
||||
const [_, options] = spawnSpy.mock.calls[0] as Parameters<typeof spawnWithWindowsHideModule.spawnWithWindowsHide>
|
||||
expect(options.stdout).toBe("pipe")
|
||||
expect(options.stderr).toBe("pipe")
|
||||
} finally {
|
||||
spawnSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it("logs captured output when piped install fails", async () => {
|
||||
// given
|
||||
const spawnSpy = spyOn(spawnWithWindowsHideModule, "spawnWithWindowsHide").mockReturnValue(
|
||||
createProc(1, {
|
||||
stdout: "resolved 10 packages",
|
||||
stderr: "network error",
|
||||
})
|
||||
)
|
||||
const logSpy = spyOn(loggerModule, "log").mockImplementation(() => {})
|
||||
|
||||
try {
|
||||
// when
|
||||
const result = await runBunInstallWithDetails({ outputMode: "pipe" })
|
||||
|
||||
// then
|
||||
expect(result).toEqual({
|
||||
success: false,
|
||||
error: "bun install failed with exit code 1",
|
||||
})
|
||||
expect(logSpy).toHaveBeenCalledWith("[bun-install] Captured output from failed bun install", {
|
||||
stdout: "resolved 10 packages",
|
||||
stderr: "network error",
|
||||
})
|
||||
} finally {
|
||||
logSpy.mockRestore()
|
||||
spawnSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,30 +1,9 @@
|
||||
import { getConfigDir } from "./config-context"
|
||||
import { log } from "../../shared/logger"
|
||||
import { spawnWithWindowsHide } from "../../shared/spawn-with-windows-hide"
|
||||
|
||||
const BUN_INSTALL_TIMEOUT_SECONDS = 60
|
||||
const BUN_INSTALL_TIMEOUT_MS = BUN_INSTALL_TIMEOUT_SECONDS * 1000
|
||||
|
||||
type BunInstallOutputMode = "inherit" | "pipe"
|
||||
|
||||
interface RunBunInstallOptions {
|
||||
outputMode?: BunInstallOutputMode
|
||||
}
|
||||
|
||||
interface BunInstallOutput {
|
||||
stdout: string
|
||||
stderr: string
|
||||
}
|
||||
|
||||
declare function setTimeout(callback: () => void, delay?: number): number
|
||||
declare function clearTimeout(timeout: number): void
|
||||
|
||||
type ProcessOutputStream = ReturnType<typeof spawnWithWindowsHide>["stdout"]
|
||||
|
||||
declare const Bun: {
|
||||
readableStreamToText(stream: NonNullable<ProcessOutputStream>): Promise<string>
|
||||
}
|
||||
|
||||
export interface BunInstallResult {
|
||||
success: boolean
|
||||
timedOut?: boolean
|
||||
@@ -36,54 +15,21 @@ export async function runBunInstall(): Promise<boolean> {
|
||||
return result.success
|
||||
}
|
||||
|
||||
function readProcessOutput(stream: ProcessOutputStream): Promise<string> {
|
||||
if (!stream) {
|
||||
return Promise.resolve("")
|
||||
}
|
||||
|
||||
return Bun.readableStreamToText(stream)
|
||||
}
|
||||
|
||||
function logCapturedOutputOnFailure(outputMode: BunInstallOutputMode, output: BunInstallOutput): void {
|
||||
if (outputMode !== "pipe") {
|
||||
return
|
||||
}
|
||||
|
||||
const stdout = output.stdout.trim()
|
||||
const stderr = output.stderr.trim()
|
||||
if (!stdout && !stderr) {
|
||||
return
|
||||
}
|
||||
|
||||
log("[bun-install] Captured output from failed bun install", {
|
||||
stdout,
|
||||
stderr,
|
||||
})
|
||||
}
|
||||
|
||||
export async function runBunInstallWithDetails(options?: RunBunInstallOptions): Promise<BunInstallResult> {
|
||||
const outputMode = options?.outputMode ?? "inherit"
|
||||
|
||||
export async function runBunInstallWithDetails(): Promise<BunInstallResult> {
|
||||
try {
|
||||
const proc = spawnWithWindowsHide(["bun", "install"], {
|
||||
cwd: getConfigDir(),
|
||||
stdout: outputMode,
|
||||
stderr: outputMode,
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
})
|
||||
|
||||
const outputPromise = Promise.all([readProcessOutput(proc.stdout), readProcessOutput(proc.stderr)]).then(
|
||||
([stdout, stderr]) => ({ stdout, stderr })
|
||||
)
|
||||
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined
|
||||
let timeoutId: ReturnType<typeof setTimeout>
|
||||
const timeoutPromise = new Promise<"timeout">((resolve) => {
|
||||
timeoutId = setTimeout(() => resolve("timeout"), BUN_INSTALL_TIMEOUT_MS)
|
||||
})
|
||||
const exitPromise = proc.exited.then(() => "completed" as const)
|
||||
const result = await Promise.race([exitPromise, timeoutPromise])
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId)
|
||||
}
|
||||
clearTimeout(timeoutId!)
|
||||
|
||||
if (result === "timeout") {
|
||||
try {
|
||||
@@ -91,10 +37,6 @@ export async function runBunInstallWithDetails(options?: RunBunInstallOptions):
|
||||
} catch {
|
||||
/* intentionally empty - process may have already exited */
|
||||
}
|
||||
|
||||
await proc.exited
|
||||
logCapturedOutputOnFailure(outputMode, await outputPromise)
|
||||
|
||||
return {
|
||||
success: false,
|
||||
timedOut: true,
|
||||
@@ -102,11 +44,7 @@ export async function runBunInstallWithDetails(options?: RunBunInstallOptions):
|
||||
}
|
||||
}
|
||||
|
||||
const output = await outputPromise
|
||||
|
||||
if (proc.exitCode !== 0) {
|
||||
logCapturedOutputOnFailure(outputMode, output)
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `bun install failed with exit code ${proc.exitCode}`,
|
||||
|
||||
@@ -25,7 +25,7 @@ const mockGetCachedVersion = mock((): string | null => "3.4.0")
|
||||
const mockGetLatestVersion = mock(async (): Promise<string | null> => "3.5.0")
|
||||
const mockExtractChannel = mock(() => "latest")
|
||||
const mockInvalidatePackage = mock(() => {})
|
||||
const mockRunBunInstallWithDetails = mock(async () => ({ success: true }))
|
||||
const mockRunBunInstall = mock(async () => true)
|
||||
const mockShowUpdateAvailableToast = mock(
|
||||
async (_ctx: PluginInput, _latestVersion: string, _getToastMessage: ToastMessageGetter): Promise<void> => {}
|
||||
)
|
||||
@@ -41,7 +41,7 @@ mock.module("../checker", () => ({
|
||||
}))
|
||||
mock.module("../version-channel", () => ({ extractChannel: mockExtractChannel }))
|
||||
mock.module("../cache", () => ({ invalidatePackage: mockInvalidatePackage }))
|
||||
mock.module("../../../cli/config-manager", () => ({ runBunInstallWithDetails: mockRunBunInstallWithDetails }))
|
||||
mock.module("../../../cli/config-manager", () => ({ runBunInstall: mockRunBunInstall }))
|
||||
mock.module("./update-toasts", () => ({
|
||||
showUpdateAvailableToast: mockShowUpdateAvailableToast,
|
||||
showAutoUpdatedToast: mockShowAutoUpdatedToast,
|
||||
@@ -62,7 +62,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
mockGetLatestVersion.mockReset()
|
||||
mockExtractChannel.mockReset()
|
||||
mockInvalidatePackage.mockReset()
|
||||
mockRunBunInstallWithDetails.mockReset()
|
||||
mockRunBunInstall.mockReset()
|
||||
mockShowUpdateAvailableToast.mockReset()
|
||||
mockShowAutoUpdatedToast.mockReset()
|
||||
|
||||
@@ -70,7 +70,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
mockGetCachedVersion.mockReturnValue("3.4.0")
|
||||
mockGetLatestVersion.mockResolvedValue("3.5.0")
|
||||
mockExtractChannel.mockReturnValue("latest")
|
||||
mockRunBunInstallWithDetails.mockResolvedValue({ success: true })
|
||||
mockRunBunInstall.mockResolvedValue(true)
|
||||
})
|
||||
|
||||
describe("#given no plugin entry found", () => {
|
||||
@@ -83,7 +83,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
expect(mockFindPluginEntry).toHaveBeenCalledTimes(1)
|
||||
expect(mockShowUpdateAvailableToast).not.toHaveBeenCalled()
|
||||
expect(mockShowAutoUpdatedToast).not.toHaveBeenCalled()
|
||||
expect(mockRunBunInstallWithDetails).not.toHaveBeenCalled()
|
||||
expect(mockRunBunInstall).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -110,7 +110,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
await runBackgroundUpdateCheck(mockCtx, true, getToastMessage)
|
||||
//#then
|
||||
expect(mockGetLatestVersion).toHaveBeenCalledWith("latest")
|
||||
expect(mockRunBunInstallWithDetails).not.toHaveBeenCalled()
|
||||
expect(mockRunBunInstall).not.toHaveBeenCalled()
|
||||
expect(mockShowUpdateAvailableToast).not.toHaveBeenCalled()
|
||||
expect(mockShowAutoUpdatedToast).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -125,7 +125,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
await runBackgroundUpdateCheck(mockCtx, true, getToastMessage)
|
||||
//#then
|
||||
expect(mockGetLatestVersion).toHaveBeenCalledTimes(1)
|
||||
expect(mockRunBunInstallWithDetails).not.toHaveBeenCalled()
|
||||
expect(mockRunBunInstall).not.toHaveBeenCalled()
|
||||
expect(mockShowUpdateAvailableToast).not.toHaveBeenCalled()
|
||||
expect(mockShowAutoUpdatedToast).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -139,7 +139,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
await runBackgroundUpdateCheck(mockCtx, autoUpdate, getToastMessage)
|
||||
//#then
|
||||
expect(mockShowUpdateAvailableToast).toHaveBeenCalledWith(mockCtx, "3.5.0", getToastMessage)
|
||||
expect(mockRunBunInstallWithDetails).not.toHaveBeenCalled()
|
||||
expect(mockRunBunInstall).not.toHaveBeenCalled()
|
||||
expect(mockShowAutoUpdatedToast).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -152,7 +152,7 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
await runBackgroundUpdateCheck(mockCtx, true, getToastMessage)
|
||||
//#then
|
||||
expect(mockShowUpdateAvailableToast).toHaveBeenCalledTimes(1)
|
||||
expect(mockRunBunInstallWithDetails).not.toHaveBeenCalled()
|
||||
expect(mockRunBunInstall).not.toHaveBeenCalled()
|
||||
expect(mockShowAutoUpdatedToast).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -182,13 +182,12 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
describe("#given unpinned with auto-update and install succeeds", () => {
|
||||
it("invalidates cache, installs, and shows auto-updated toast", async () => {
|
||||
//#given
|
||||
mockRunBunInstallWithDetails.mockResolvedValue({ success: true })
|
||||
mockRunBunInstall.mockResolvedValue(true)
|
||||
//#when
|
||||
await runBackgroundUpdateCheck(mockCtx, true, getToastMessage)
|
||||
//#then
|
||||
expect(mockInvalidatePackage).toHaveBeenCalledTimes(1)
|
||||
expect(mockRunBunInstallWithDetails).toHaveBeenCalledTimes(1)
|
||||
expect(mockRunBunInstallWithDetails).toHaveBeenCalledWith({ outputMode: "pipe" })
|
||||
expect(mockRunBunInstall).toHaveBeenCalledTimes(1)
|
||||
expect(mockShowAutoUpdatedToast).toHaveBeenCalledWith(mockCtx, "3.4.0", "3.5.0")
|
||||
expect(mockShowUpdateAvailableToast).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -197,12 +196,11 @@ describe("runBackgroundUpdateCheck", () => {
|
||||
describe("#given unpinned with auto-update and install fails", () => {
|
||||
it("falls back to notification-only toast", async () => {
|
||||
//#given
|
||||
mockRunBunInstallWithDetails.mockResolvedValue({ success: false, error: "install failed" })
|
||||
mockRunBunInstall.mockResolvedValue(false)
|
||||
//#when
|
||||
await runBackgroundUpdateCheck(mockCtx, true, getToastMessage)
|
||||
//#then
|
||||
expect(mockRunBunInstallWithDetails).toHaveBeenCalledTimes(1)
|
||||
expect(mockRunBunInstallWithDetails).toHaveBeenCalledWith({ outputMode: "pipe" })
|
||||
expect(mockRunBunInstall).toHaveBeenCalledTimes(1)
|
||||
expect(mockShowUpdateAvailableToast).toHaveBeenCalledWith(mockCtx, "3.5.0", getToastMessage)
|
||||
expect(mockShowAutoUpdatedToast).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { PluginInput } from "@opencode-ai/plugin"
|
||||
import { runBunInstallWithDetails } from "../../../cli/config-manager"
|
||||
import { runBunInstall } from "../../../cli/config-manager"
|
||||
import { log } from "../../../shared/logger"
|
||||
import { invalidatePackage } from "../cache"
|
||||
import { PACKAGE_NAME } from "../constants"
|
||||
@@ -13,12 +13,7 @@ function getPinnedVersionToastMessage(latestVersion: string): string {
|
||||
|
||||
async function runBunInstallSafe(): Promise<boolean> {
|
||||
try {
|
||||
const result = await runBunInstallWithDetails({ outputMode: "pipe" })
|
||||
if (!result.success && result.error) {
|
||||
log("[auto-update-checker] bun install failed:", result.error)
|
||||
}
|
||||
|
||||
return result.success
|
||||
return await runBunInstall()
|
||||
} catch (err) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||
log("[auto-update-checker] bun install error:", errorMessage)
|
||||
|
||||
@@ -26,6 +26,11 @@ export const RETRYABLE_ERROR_PATTERNS = [
|
||||
/rate.?limit/i,
|
||||
/too.?many.?requests/i,
|
||||
/quota.?exceeded/i,
|
||||
/quota\s+will\s+reset\s+after/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/cooldown/i,
|
||||
/exhausted\s+your\s+capacity/i,
|
||||
/usage\s+limit\s+has\s+been\s+reached/i,
|
||||
/service.?unavailable/i,
|
||||
/overloaded/i,
|
||||
|
||||
@@ -6,9 +6,11 @@ import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableErro
|
||||
import { createFallbackState, prepareFallback } from "./fallback-state"
|
||||
import { getFallbackModelsForSession } from "./fallback-models"
|
||||
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||
import { createSessionStatusHandler } from "./session-status-handler"
|
||||
|
||||
export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
|
||||
const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
|
||||
const sessionStatusHandler = createSessionStatusHandler(deps, helpers)
|
||||
|
||||
const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
|
||||
const sessionInfo = props?.info as { id?: string; model?: string } | undefined
|
||||
@@ -33,6 +35,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
|
||||
sessionRetryInFlight.delete(sessionID)
|
||||
sessionAwaitingFallbackResult.delete(sessionID)
|
||||
helpers.clearSessionFallbackTimeout(sessionID)
|
||||
sessionStatusHandler.clearRetryKey(sessionID)
|
||||
SessionCategoryRegistry.remove(sessionID)
|
||||
}
|
||||
}
|
||||
@@ -191,6 +194,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
|
||||
if (event.type === "session.deleted") { handleSessionDeleted(props); return }
|
||||
if (event.type === "session.stop") { await handleSessionStop(props); return }
|
||||
if (event.type === "session.idle") { handleSessionIdle(props); return }
|
||||
if (event.type === "session.status") { await sessionStatusHandler.handleSessionStatus(props); return }
|
||||
if (event.type === "session.error") { await handleSessionError(props); return }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -387,6 +387,133 @@ describe("runtime-fallback", () => {
|
||||
expect(fallbackLog?.data).toMatchObject({ from: "openai/gpt-5.3-codex", to: "anthropic/claude-opus-4-6" })
|
||||
})
|
||||
|
||||
test("should trigger fallback on session.status auto-retry signal", async () => {
|
||||
const promptCalls: unknown[] = []
|
||||
const hook = createRuntimeFallbackHook(
|
||||
createMockPluginInput({
|
||||
session: {
|
||||
messages: async () => ({
|
||||
data: [
|
||||
{
|
||||
info: { role: "user" },
|
||||
parts: [{ type: "text", text: "continue" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
promptAsync: async (args: unknown) => {
|
||||
promptCalls.push(args)
|
||||
return {}
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
config: createMockConfig({ notify_on_fallback: false }),
|
||||
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.4"]),
|
||||
}
|
||||
)
|
||||
|
||||
const sessionID = "test-session-status-auto-retry"
|
||||
SessionCategoryRegistry.register(sessionID, "test")
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.created",
|
||||
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
|
||||
},
|
||||
})
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.status",
|
||||
properties: {
|
||||
sessionID,
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 1,
|
||||
next: 476,
|
||||
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal in session.status"))
|
||||
expect(signalLog).toBeDefined()
|
||||
|
||||
const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
|
||||
expect(fallbackLog).toBeDefined()
|
||||
expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.4" })
|
||||
expect(promptCalls).toHaveLength(1)
|
||||
})
|
||||
|
||||
test("should deduplicate session.status countdown updates for the same retry attempt", async () => {
|
||||
const promptCalls: unknown[] = []
|
||||
const hook = createRuntimeFallbackHook(
|
||||
createMockPluginInput({
|
||||
session: {
|
||||
messages: async () => ({
|
||||
data: [
|
||||
{
|
||||
info: { role: "user" },
|
||||
parts: [{ type: "text", text: "continue" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
promptAsync: async (args: unknown) => {
|
||||
promptCalls.push(args)
|
||||
return {}
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
config: createMockConfig({ notify_on_fallback: false }),
|
||||
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.4"]),
|
||||
}
|
||||
)
|
||||
|
||||
const sessionID = "test-session-status-countdown-dedup"
|
||||
SessionCategoryRegistry.register(sessionID, "test")
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.created",
|
||||
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
|
||||
},
|
||||
})
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.status",
|
||||
properties: {
|
||||
sessionID,
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 1,
|
||||
next: 476,
|
||||
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.status",
|
||||
properties: {
|
||||
sessionID,
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 1,
|
||||
next: 475,
|
||||
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(promptCalls).toHaveLength(1)
|
||||
})
|
||||
|
||||
test("should NOT trigger fallback on auto-retry signal when timeout_seconds is 0", async () => {
|
||||
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
|
||||
config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 0 }),
|
||||
|
||||
160
src/hooks/runtime-fallback/session-status-handler.ts
Normal file
160
src/hooks/runtime-fallback/session-status-handler.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import type { HookDeps } from "./types"
|
||||
import type { AutoRetryHelpers } from "./auto-retry"
|
||||
import { HOOK_NAME } from "./constants"
|
||||
import { log } from "../../shared/logger"
|
||||
import { isRetryableError } from "./error-classifier"
|
||||
import { createFallbackState, prepareFallback } from "./fallback-state"
|
||||
import { getFallbackModelsForSession } from "./fallback-models"
|
||||
import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../../shared/retry-status-utils"
|
||||
|
||||
type SessionStatus = {
|
||||
type?: string
|
||||
message?: string
|
||||
attempt?: number
|
||||
}
|
||||
|
||||
function resolveInitialModel(
|
||||
props: Record<string, unknown> | undefined,
|
||||
retryMessage: string,
|
||||
resolvedAgent: string | undefined,
|
||||
pluginConfig: HookDeps["pluginConfig"],
|
||||
): string | undefined {
|
||||
const eventModel = typeof props?.model === "string" ? props.model : undefined
|
||||
if (eventModel) {
|
||||
return eventModel
|
||||
}
|
||||
|
||||
const retryModel = extractRetryStatusModel(retryMessage)
|
||||
if (retryModel) {
|
||||
return retryModel
|
||||
}
|
||||
|
||||
const agentConfig = resolvedAgent
|
||||
? pluginConfig?.agents?.[resolvedAgent as keyof typeof pluginConfig.agents]
|
||||
: undefined
|
||||
|
||||
return typeof agentConfig?.model === "string" ? agentConfig.model : undefined
|
||||
}
|
||||
|
||||
export function createSessionStatusHandler(deps: HookDeps, helpers: AutoRetryHelpers): {
|
||||
clearRetryKey: (sessionID: string) => void
|
||||
handleSessionStatus: (props: Record<string, unknown> | undefined) => Promise<void>
|
||||
} {
|
||||
const {
|
||||
config,
|
||||
pluginConfig,
|
||||
sessionStates,
|
||||
sessionLastAccess,
|
||||
sessionRetryInFlight,
|
||||
sessionAwaitingFallbackResult,
|
||||
} = deps
|
||||
const sessionStatusRetryKeys = new Map<string, string>()
|
||||
|
||||
const clearRetryKey = (sessionID: string): void => {
|
||||
sessionStatusRetryKeys.delete(sessionID)
|
||||
}
|
||||
|
||||
const handleSessionStatus = async (props: Record<string, unknown> | undefined): Promise<void> => {
|
||||
const sessionID = props?.sessionID as string | undefined
|
||||
const status = props?.status as SessionStatus | undefined
|
||||
const agent = props?.agent as string | undefined
|
||||
const timeoutEnabled = config.timeout_seconds > 0
|
||||
|
||||
if (!sessionID || status?.type !== "retry" || !timeoutEnabled) {
|
||||
return
|
||||
}
|
||||
|
||||
const retryMessage = typeof status.message === "string" ? status.message : ""
|
||||
if (!retryMessage || !isRetryableError({ message: retryMessage }, config.retry_on_errors)) {
|
||||
return
|
||||
}
|
||||
|
||||
const currentState = sessionStates.get(sessionID)
|
||||
const retryAttempt = extractRetryAttempt(status.attempt, retryMessage)
|
||||
const retryModel =
|
||||
(typeof props?.model === "string" ? props.model : undefined) ??
|
||||
extractRetryStatusModel(retryMessage) ??
|
||||
currentState?.currentModel ??
|
||||
"unknown-model"
|
||||
const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`
|
||||
|
||||
if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
|
||||
return
|
||||
}
|
||||
sessionStatusRetryKeys.set(sessionID, retryKey)
|
||||
|
||||
if (sessionRetryInFlight.has(sessionID)) {
|
||||
log(`[${HOOK_NAME}] Overriding in-flight retry due to provider session.status retry signal`, {
|
||||
sessionID,
|
||||
retryModel,
|
||||
})
|
||||
await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
|
||||
sessionRetryInFlight.delete(sessionID)
|
||||
}
|
||||
|
||||
sessionAwaitingFallbackResult.delete(sessionID)
|
||||
|
||||
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
|
||||
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
|
||||
|
||||
if (fallbackModels.length === 0) {
|
||||
log(`[${HOOK_NAME}] No fallback models configured`, { sessionID, agent: resolvedAgent ?? agent })
|
||||
return
|
||||
}
|
||||
|
||||
let state = currentState
|
||||
if (!state) {
|
||||
const initialModel = resolveInitialModel(props, retryMessage, resolvedAgent, pluginConfig)
|
||||
if (!initialModel) {
|
||||
log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
|
||||
return
|
||||
}
|
||||
|
||||
state = createFallbackState(initialModel)
|
||||
sessionStates.set(sessionID, state)
|
||||
}
|
||||
|
||||
sessionLastAccess.set(sessionID, Date.now())
|
||||
|
||||
if (state.pendingFallbackModel) {
|
||||
log(`[${HOOK_NAME}] Clearing pending fallback due to provider session.status retry signal`, {
|
||||
sessionID,
|
||||
pendingFallbackModel: state.pendingFallbackModel,
|
||||
})
|
||||
state.pendingFallbackModel = undefined
|
||||
}
|
||||
|
||||
log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
|
||||
sessionID,
|
||||
model: state.currentModel,
|
||||
retryAttempt,
|
||||
})
|
||||
|
||||
const result = prepareFallback(sessionID, state, fallbackModels, config)
|
||||
|
||||
if (result.success && config.notify_on_fallback) {
|
||||
await deps.ctx.client.tui
|
||||
.showToast({
|
||||
body: {
|
||||
title: "Model Fallback",
|
||||
message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
|
||||
variant: "warning",
|
||||
duration: 5000,
|
||||
},
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
|
||||
if (result.success && result.newModel) {
|
||||
await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
|
||||
return
|
||||
}
|
||||
|
||||
log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
|
||||
}
|
||||
|
||||
return {
|
||||
clearRetryKey,
|
||||
handleSessionStatus,
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { _resetForTesting, setMainSession } from "../features/claude-code-sessio
|
||||
import { createModelFallbackHook, clearPendingModelFallback } from "../hooks/model-fallback/hook"
|
||||
|
||||
describe("createEventHandler - model fallback", () => {
|
||||
const createHandler = (args?: { hooks?: any }) => {
|
||||
const createHandler = (args?: { hooks?: any; pluginConfig?: any }) => {
|
||||
const abortCalls: string[] = []
|
||||
const promptCalls: string[] = []
|
||||
|
||||
@@ -26,7 +26,7 @@ describe("createEventHandler - model fallback", () => {
|
||||
},
|
||||
},
|
||||
} as any,
|
||||
pluginConfig: {} as any,
|
||||
pluginConfig: (args?.pluginConfig ?? {}) as any,
|
||||
firstMessageVariantGate: {
|
||||
markSessionCreated: () => {},
|
||||
clear: () => {},
|
||||
@@ -213,6 +213,121 @@ describe("createEventHandler - model fallback", () => {
|
||||
expect(output.message["variant"]).toBe("max")
|
||||
})
|
||||
|
||||
test("deduplicates session.status countdown updates for the same retry attempt", async () => {
|
||||
//#given
|
||||
const sessionID = "ses_status_retry_dedup"
|
||||
setMainSession(sessionID)
|
||||
clearPendingModelFallback(sessionID)
|
||||
const modelFallback = createModelFallbackHook()
|
||||
const { handler, abortCalls, promptCalls } = createHandler({ hooks: { modelFallback } })
|
||||
|
||||
await handler({
|
||||
event: {
|
||||
type: "message.updated",
|
||||
properties: {
|
||||
info: {
|
||||
id: "msg_user_status_dedup",
|
||||
sessionID,
|
||||
role: "user",
|
||||
modelID: "claude-opus-4-6-thinking",
|
||||
providerID: "anthropic",
|
||||
agent: "Sisyphus (Ultraworker)",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
//#when
|
||||
await handler({
|
||||
event: {
|
||||
type: "session.status",
|
||||
properties: {
|
||||
sessionID,
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 1,
|
||||
message:
|
||||
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}} [retrying in 27s attempt #1]",
|
||||
next: 27,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await handler({
|
||||
event: {
|
||||
type: "session.status",
|
||||
properties: {
|
||||
sessionID,
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 1,
|
||||
message:
|
||||
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}} [retrying in 26s attempt #1]",
|
||||
next: 26,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
//#then
|
||||
expect(abortCalls).toEqual([sessionID])
|
||||
expect(promptCalls).toEqual([sessionID])
|
||||
})
|
||||
|
||||
test("does not trigger model fallback from session.status when runtime fallback is enabled", async () => {
|
||||
//#given
|
||||
const sessionID = "ses_status_retry_runtime_enabled"
|
||||
setMainSession(sessionID)
|
||||
clearPendingModelFallback(sessionID)
|
||||
const modelFallback = createModelFallbackHook()
|
||||
const runtimeFallback = {
|
||||
event: async () => {},
|
||||
"chat.message": async () => {},
|
||||
}
|
||||
const { handler, abortCalls, promptCalls } = createHandler({
|
||||
hooks: { modelFallback, runtimeFallback },
|
||||
pluginConfig: { runtime_fallback: { enabled: true } },
|
||||
})
|
||||
|
||||
await handler({
|
||||
event: {
|
||||
type: "message.updated",
|
||||
properties: {
|
||||
info: {
|
||||
id: "msg_user_status_runtime_enabled",
|
||||
sessionID,
|
||||
role: "user",
|
||||
modelID: "claude-opus-4-6-thinking",
|
||||
providerID: "anthropic",
|
||||
agent: "Sisyphus (Ultraworker)",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
//#when
|
||||
await handler({
|
||||
event: {
|
||||
type: "session.status",
|
||||
properties: {
|
||||
sessionID,
|
||||
status: {
|
||||
type: "retry",
|
||||
attempt: 1,
|
||||
message:
|
||||
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}} [retrying in 27s attempt #1]",
|
||||
next: 27,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
//#then
|
||||
expect(abortCalls).toEqual([])
|
||||
expect(promptCalls).toEqual([])
|
||||
})
|
||||
|
||||
test("advances main-session fallback chain across repeated session.error retries end-to-end", async () => {
|
||||
//#given
|
||||
const abortCalls: string[] = []
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
import { resetMessageCursor } from "../shared";
|
||||
import { log } from "../shared/logger";
|
||||
import { shouldRetryError } from "../shared/model-error-classifier";
|
||||
import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../shared/retry-status-utils";
|
||||
import { clearSessionModel, setSessionModel } from "../shared/session-model-state";
|
||||
import { deleteSessionTools } from "../shared/session-tools-store";
|
||||
import { lspManager } from "../tools";
|
||||
@@ -342,10 +343,15 @@ export function createEventHandler(args: {
|
||||
const sessionID = props?.sessionID as string | undefined;
|
||||
const status = props?.status as { type?: string; attempt?: number; message?: string; next?: number } | undefined;
|
||||
|
||||
if (sessionID && status?.type === "retry" && isModelFallbackEnabled) {
|
||||
if (sessionID && status?.type === "retry" && !isRuntimeFallbackEnabled && isModelFallbackEnabled) {
|
||||
try {
|
||||
const retryMessage = typeof status.message === "string" ? status.message : "";
|
||||
const retryKey = `${status.attempt ?? "?"}:${status.next ?? "?"}:${retryMessage}`;
|
||||
const retryAttempt = extractRetryAttempt(status.attempt, retryMessage);
|
||||
const retryModel =
|
||||
extractRetryStatusModel(retryMessage) ??
|
||||
lastKnownModelBySession.get(sessionID)?.modelID ??
|
||||
"unknown-model";
|
||||
const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`;
|
||||
if (lastHandledRetryStatusKey.get(sessionID) === retryKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
41
src/shared/retry-status-utils.test.ts
Normal file
41
src/shared/retry-status-utils.test.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "./retry-status-utils"
|
||||
|
||||
describe("retry-status-utils", () => {
|
||||
test("extracts retry attempt from explicit status attempt", () => {
|
||||
//#given
|
||||
const attempt = 6
|
||||
|
||||
//#when
|
||||
const result = extractRetryAttempt(attempt, "The usage limit has been reached [retrying in 27s attempt #6]")
|
||||
|
||||
//#then
|
||||
expect(result).toBe(6)
|
||||
})
|
||||
|
||||
test("extracts retry model from cooldown status text", () => {
|
||||
//#given
|
||||
const message = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]"
|
||||
|
||||
//#when
|
||||
const result = extractRetryStatusModel(message)
|
||||
|
||||
//#then
|
||||
expect(result).toBe("claude-opus-4-6")
|
||||
})
|
||||
|
||||
test("normalizes countdown jitter to a stable cooldown class", () => {
|
||||
//#given
|
||||
const firstMessage = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]"
|
||||
const secondMessage = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]"
|
||||
|
||||
//#when
|
||||
const firstResult = normalizeRetryStatusMessage(firstMessage)
|
||||
const secondResult = normalizeRetryStatusMessage(secondMessage)
|
||||
|
||||
//#then
|
||||
expect(firstResult).toBe("cooldown")
|
||||
expect(secondResult).toBe("cooldown")
|
||||
})
|
||||
})
|
||||
51
src/shared/retry-status-utils.ts
Normal file
51
src/shared/retry-status-utils.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
const RETRY_COUNTDOWN_PATTERN = /\[\s*retrying\s+in[^\]]*\]/gi
|
||||
|
||||
function collapseWhitespace(value: string): string {
|
||||
return value.toLowerCase().replace(/\s+/g, " ").trim()
|
||||
}
|
||||
|
||||
export function extractRetryAttempt(attempt: number | undefined, message: string): number | "?" {
|
||||
if (typeof attempt === "number" && Number.isFinite(attempt)) {
|
||||
return attempt
|
||||
}
|
||||
|
||||
const parsedAttempt = message.match(/attempt\s*#\s*(\d+)/i)?.[1]
|
||||
return parsedAttempt ? Number.parseInt(parsedAttempt, 10) : "?"
|
||||
}
|
||||
|
||||
export function extractRetryStatusModel(message: string): string | undefined {
|
||||
return message.match(/model\s+([a-z0-9._/-]+)(?=\s+(?:are|is)\b)/i)?.[1]?.toLowerCase()
|
||||
}
|
||||
|
||||
export function normalizeRetryStatusMessage(message: string): string {
|
||||
const normalizedMessage = collapseWhitespace(message.replace(RETRY_COUNTDOWN_PATTERN, " "))
|
||||
if (!normalizedMessage) {
|
||||
return "retry"
|
||||
}
|
||||
|
||||
if (/all\s+credentials\s+for\s+model|cool(?:ing)?\s+down|cooldown|exhausted\s+your\s+capacity/.test(normalizedMessage)) {
|
||||
return "cooldown"
|
||||
}
|
||||
|
||||
if (/too\s+many\s+requests/.test(normalizedMessage)) {
|
||||
return "too-many-requests"
|
||||
}
|
||||
|
||||
if (/quota\s+will\s+reset\s+after|quota\s*exceeded/.test(normalizedMessage)) {
|
||||
return "quota"
|
||||
}
|
||||
|
||||
if (/usage\s+limit\s+has\s+been\s+reached|limit\s+reached/.test(normalizedMessage)) {
|
||||
return "usage-limit"
|
||||
}
|
||||
|
||||
if (/rate\s+limit/.test(normalizedMessage)) {
|
||||
return "rate-limit"
|
||||
}
|
||||
|
||||
if (/service.?unavailable|temporarily.?unavailable|overloaded/.test(normalizedMessage)) {
|
||||
return "service-unavailable"
|
||||
}
|
||||
|
||||
return normalizedMessage
|
||||
}
|
||||
Reference in New Issue
Block a user