Compare commits

..

7 Commits

Author SHA1 Message Date
YeonGyu-Kim
b1946a6397 test(plugin): cover session.status dedupe and runtime fallback gating
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:15:13 +09:00
YeonGyu-Kim
6edbc9638a fix(plugin): gate model-fallback session.status retries
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:15:06 +09:00
YeonGyu-Kim
af32f2e258 test(runtime-fallback): cover session.status retry fallback
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:14:58 +09:00
YeonGyu-Kim
e7e6bd0608 fix(runtime-fallback): route session.status events
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:14:53 +09:00
YeonGyu-Kim
7f86103666 fix(runtime-fallback): add session.status retry handler
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:14:47 +09:00
YeonGyu-Kim
2c6ba98920 fix(runtime-fallback): match cooldown retry status messages
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:14:39 +09:00
YeonGyu-Kim
80dee4d2c9 fix(shared): add stable retry status dedupe helpers
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-03-08 02:14:33 +09:00
12 changed files with 590 additions and 193 deletions

View File

@@ -224,12 +224,6 @@ function stubNotifyParentSession(manager: BackgroundManager): void {
;(manager as unknown as { notifyParentSession: () => Promise<void> }).notifyParentSession = async () => {}
}
async function flushBackgroundNotifications(): Promise<void> {
for (let i = 0; i < 6; i++) {
await Promise.resolve()
}
}
function createToastRemoveTaskTracker(): { removeTaskCalls: string[]; resetToastManager: () => void } {
_resetTaskToastManagerForTesting()
const toastManager = initTaskToastManager({
@@ -1312,20 +1306,11 @@ describe("BackgroundManager.tryCompleteTask", () => {
expect(abortedSessionIDs).toEqual(["session-1"])
})
test("should clean pendingByParent even when promptAsync notification fails", async () => {
test("should clean pendingByParent even when notifyParentSession throws", async () => {
// given
const client = {
session: {
prompt: async () => ({}),
promptAsync: async () => {
throw new Error("notify failed")
},
abort: async () => ({}),
messages: async () => ({ data: [] }),
},
;(manager as unknown as { notifyParentSession: () => Promise<void> }).notifyParentSession = async () => {
throw new Error("notify failed")
}
manager.shutdown()
manager = new BackgroundManager({ client, directory: tmpdir() } as unknown as PluginInput)
const task: BackgroundTask = {
id: "task-pending-cleanup",
@@ -1439,7 +1424,7 @@ describe("BackgroundManager.tryCompleteTask", () => {
// then
expect(rejectedCount).toBe(0)
expect(promptBodies.length).toBe(2)
expect(promptBodies.filter((body) => body.noReply === false)).toHaveLength(1)
expect(promptBodies.some((b) => b.noReply === false)).toBe(true)
})
})
@@ -1947,6 +1932,7 @@ describe("BackgroundManager - Non-blocking Queue Integration", () => {
test("should cancel running task and release concurrency", async () => {
// given
const manager = createBackgroundManager()
stubNotifyParentSession(manager)
const concurrencyManager = getConcurrencyManager(manager)
const concurrencyKey = "test-provider/test-model"
@@ -2904,7 +2890,7 @@ describe("BackgroundManager.shutdown session abort", () => {
})
describe("BackgroundManager.handleEvent - session.deleted cascade", () => {
test("should cancel descendant tasks and keep them until delayed cleanup", async () => {
test("should cancel descendant tasks when parent session is deleted", () => {
// given
const manager = createBackgroundManager()
const parentSessionID = "session-parent"
@@ -2951,26 +2937,21 @@ describe("BackgroundManager.handleEvent - session.deleted cascade", () => {
properties: { info: { id: parentSessionID } },
})
await flushBackgroundNotifications()
// then
expect(taskMap.has(childTask.id)).toBe(true)
expect(taskMap.has(siblingTask.id)).toBe(true)
expect(taskMap.has(grandchildTask.id)).toBe(true)
expect(taskMap.has(childTask.id)).toBe(false)
expect(taskMap.has(siblingTask.id)).toBe(false)
expect(taskMap.has(grandchildTask.id)).toBe(false)
expect(taskMap.has(unrelatedTask.id)).toBe(true)
expect(childTask.status).toBe("cancelled")
expect(siblingTask.status).toBe("cancelled")
expect(grandchildTask.status).toBe("cancelled")
expect(pendingByParent.get(parentSessionID)).toBeUndefined()
expect(pendingByParent.get("session-child")).toBeUndefined()
expect(getCompletionTimers(manager).has(childTask.id)).toBe(true)
expect(getCompletionTimers(manager).has(siblingTask.id)).toBe(true)
expect(getCompletionTimers(manager).has(grandchildTask.id)).toBe(true)
manager.shutdown()
})
test("should remove cancelled tasks from toast manager while preserving delayed cleanup", async () => {
test("should remove tasks from toast manager when session is deleted", () => {
//#given
const { removeTaskCalls, resetToastManager } = createToastRemoveTaskTracker()
const manager = createBackgroundManager()
@@ -2999,13 +2980,9 @@ describe("BackgroundManager.handleEvent - session.deleted cascade", () => {
properties: { info: { id: parentSessionID } },
})
await flushBackgroundNotifications()
//#then
expect(removeTaskCalls).toContain(childTask.id)
expect(removeTaskCalls).toContain(grandchildTask.id)
expect(getCompletionTimers(manager).has(childTask.id)).toBe(true)
expect(getCompletionTimers(manager).has(grandchildTask.id)).toBe(true)
manager.shutdown()
resetToastManager()
@@ -3068,7 +3045,7 @@ describe("BackgroundManager.handleEvent - session.error", () => {
return task
}
test("sets task to error, releases concurrency, and keeps it until delayed cleanup", async () => {
test("sets task to error, releases concurrency, and cleans up", async () => {
//#given
const manager = createBackgroundManager()
const concurrencyManager = getConcurrencyManager(manager)
@@ -3101,21 +3078,18 @@ describe("BackgroundManager.handleEvent - session.error", () => {
},
})
await flushBackgroundNotifications()
//#then
expect(task.status).toBe("error")
expect(task.error).toBe("Model not found: kimi-for-coding/k2p5.")
expect(task.completedAt).toBeInstanceOf(Date)
expect(concurrencyManager.getCount(concurrencyKey)).toBe(0)
expect(getTaskMap(manager).has(task.id)).toBe(true)
expect(getTaskMap(manager).has(task.id)).toBe(false)
expect(getPendingByParent(manager).get(task.parentSessionID)).toBeUndefined()
expect(getCompletionTimers(manager).has(task.id)).toBe(true)
manager.shutdown()
})
test("should remove errored task from toast manager while preserving delayed cleanup", async () => {
test("removes errored task from toast manager", () => {
//#given
const { removeTaskCalls, resetToastManager } = createToastRemoveTaskTracker()
const manager = createBackgroundManager()
@@ -3137,11 +3111,8 @@ describe("BackgroundManager.handleEvent - session.error", () => {
},
})
await flushBackgroundNotifications()
//#then
expect(removeTaskCalls).toContain(task.id)
expect(getCompletionTimers(manager).has(task.id)).toBe(true)
manager.shutdown()
resetToastManager()
@@ -3422,7 +3393,7 @@ describe("BackgroundManager.pruneStaleTasksAndNotifications - removes pruned tas
manager.shutdown()
})
test("removes stale task from toast manager", async () => {
test("removes stale task from toast manager", () => {
//#given
const { removeTaskCalls, resetToastManager } = createToastRemoveTaskTracker()
const manager = createBackgroundManager()
@@ -3437,7 +3408,6 @@ describe("BackgroundManager.pruneStaleTasksAndNotifications - removes pruned tas
//#when
pruneStaleTasksAndNotificationsForTest(manager)
await flushBackgroundNotifications()
//#then
expect(removeTaskCalls).toContain(staleTask.id)
@@ -3445,53 +3415,6 @@ describe("BackgroundManager.pruneStaleTasksAndNotifications - removes pruned tas
manager.shutdown()
resetToastManager()
})
test("keeps stale task until notification cleanup after notifying parent", async () => {
//#given
const notifications: string[] = []
const { removeTaskCalls, resetToastManager } = createToastRemoveTaskTracker()
const client = {
session: {
prompt: async () => ({}),
promptAsync: async (args: { path: { id: string }; body: Record<string, unknown> & { noReply?: boolean; parts?: unknown[] } }) => {
const firstPart = args.body.parts?.[0]
if (firstPart && typeof firstPart === "object" && "text" in firstPart && typeof firstPart.text === "string") {
notifications.push(firstPart.text)
}
return {}
},
abort: async () => ({}),
messages: async () => ({ data: [] }),
},
}
const manager = new BackgroundManager({ client, directory: tmpdir() } as unknown as PluginInput)
const staleTask = createMockTask({
id: "task-stale-notify-cleanup",
sessionID: "session-stale-notify-cleanup",
parentSessionID: "parent-stale-notify-cleanup",
status: "running",
startedAt: new Date(Date.now() - 31 * 60 * 1000),
})
getTaskMap(manager).set(staleTask.id, staleTask)
getPendingByParent(manager).set(staleTask.parentSessionID, new Set([staleTask.id]))
//#when
pruneStaleTasksAndNotificationsForTest(manager)
await flushBackgroundNotifications()
//#then
const retainedTask = getTaskMap(manager).get(staleTask.id)
expect(retainedTask?.status).toBe("error")
expect(getTaskMap(manager).has(staleTask.id)).toBe(true)
expect(notifications).toHaveLength(1)
expect(notifications[0]).toContain("[ALL BACKGROUND TASKS COMPLETE]")
expect(notifications[0]).toContain(staleTask.description)
expect(getCompletionTimers(manager).has(staleTask.id)).toBe(true)
expect(removeTaskCalls).toContain(staleTask.id)
manager.shutdown()
resetToastManager()
})
})
describe("BackgroundManager.completionTimers - Memory Leak Fix", () => {
@@ -3595,7 +3518,7 @@ describe("BackgroundManager.completionTimers - Memory Leak Fix", () => {
expect(completionTimers.size).toBe(0)
})
test("should preserve cleanup timer when terminal task session is deleted", () => {
test("should cancel timer when task is deleted via session.deleted", () => {
// given
const manager = createBackgroundManager()
const task: BackgroundTask = {
@@ -3624,7 +3547,7 @@ describe("BackgroundManager.completionTimers - Memory Leak Fix", () => {
})
// then
expect(completionTimers.has(task.id)).toBe(true)
expect(completionTimers.has(task.id)).toBe(false)
manager.shutdown()
})

View File

@@ -390,6 +390,7 @@ export class BackgroundManager {
}).catch(() => {})
this.markForNotification(existingTask)
this.cleanupPendingByParent(existingTask)
this.enqueueNotificationForParent(existingTask.parentSessionID, () => this.notifyParentSession(existingTask)).catch(err => {
log("[background-agent] Failed to notify on error:", err)
})
@@ -660,6 +661,7 @@ export class BackgroundManager {
}
this.markForNotification(existingTask)
this.cleanupPendingByParent(existingTask)
this.enqueueNotificationForParent(existingTask.parentSessionID, () => this.notifyParentSession(existingTask)).catch(err => {
log("[background-agent] Failed to notify on resume error:", err)
})
@@ -802,14 +804,16 @@ export class BackgroundManager {
this.idleDeferralTimers.delete(task.id)
}
if (task.sessionID) {
SessionCategoryRegistry.remove(task.sessionID)
this.cleanupPendingByParent(task)
this.tasks.delete(task.id)
this.clearNotificationsForTask(task.id)
const toastManager = getTaskToastManager()
if (toastManager) {
toastManager.removeTask(task.id)
}
if (task.sessionID) {
subagentSessions.delete(task.sessionID)
}
this.markForNotification(task)
this.enqueueNotificationForParent(task.parentSessionID, () => this.notifyParentSession(task)).catch(err => {
log("[background-agent] Error in notifyParentSession for errored task:", { taskId: task.id, error: err })
})
}
if (event.type === "session.deleted") {
@@ -830,30 +834,47 @@ export class BackgroundManager {
if (tasksToCancel.size === 0) return
const deletedSessionIDs = new Set<string>([sessionID])
for (const task of tasksToCancel.values()) {
if (task.sessionID) {
deletedSessionIDs.add(task.sessionID)
}
}
for (const task of tasksToCancel.values()) {
if (task.status === "running" || task.status === "pending") {
void this.cancelTask(task.id, {
source: "session.deleted",
reason: "Session deleted",
}).then(() => {
if (deletedSessionIDs.has(task.parentSessionID)) {
this.pendingNotifications.delete(task.parentSessionID)
}
skipNotification: true,
}).catch(err => {
if (deletedSessionIDs.has(task.parentSessionID)) {
this.pendingNotifications.delete(task.parentSessionID)
}
log("[background-agent] Failed to cancel task on session.deleted:", { taskId: task.id, error: err })
})
}
const existingTimer = this.completionTimers.get(task.id)
if (existingTimer) {
clearTimeout(existingTimer)
this.completionTimers.delete(task.id)
}
const idleTimer = this.idleDeferralTimers.get(task.id)
if (idleTimer) {
clearTimeout(idleTimer)
this.idleDeferralTimers.delete(task.id)
}
this.cleanupPendingByParent(task)
this.tasks.delete(task.id)
this.clearNotificationsForTask(task.id)
const toastManager = getTaskToastManager()
if (toastManager) {
toastManager.removeTask(task.id)
}
if (task.sessionID) {
subagentSessions.delete(task.sessionID)
}
}
for (const task of tasksToCancel.values()) {
if (task.parentSessionID) {
this.pendingNotifications.delete(task.parentSessionID)
}
}
SessionCategoryRegistry.remove(sessionID)
}
@@ -1073,6 +1094,8 @@ export class BackgroundManager {
this.idleDeferralTimers.delete(task.id)
}
this.cleanupPendingByParent(task)
if (abortSession && task.sessionID) {
this.client.session.abort({
path: { id: task.sessionID },
@@ -1179,6 +1202,9 @@ export class BackgroundManager {
this.markForNotification(task)
// Ensure pending tracking is cleaned up even if notification fails
this.cleanupPendingByParent(task)
const idleTimer = this.idleDeferralTimers.get(task.id)
if (idleTimer) {
clearTimeout(idleTimer)
@@ -1234,10 +1260,7 @@ export class BackgroundManager {
this.pendingByParent.delete(task.parentSessionID)
}
} else {
remainingCount = Array.from(this.tasks.values())
.filter(t => t.parentSessionID === task.parentSessionID && t.id !== task.id && (t.status === "running" || t.status === "pending"))
.length
allComplete = remainingCount === 0
allComplete = true
}
const completedTasks = allComplete
@@ -1245,13 +1268,7 @@ export class BackgroundManager {
.filter(t => t.parentSessionID === task.parentSessionID && t.status !== "running" && t.status !== "pending")
: []
const statusText = task.status === "completed"
? "COMPLETED"
: task.status === "interrupt"
? "INTERRUPTED"
: task.status === "error"
? "ERROR"
: "CANCELLED"
const statusText = task.status === "completed" ? "COMPLETED" : task.status === "interrupt" ? "INTERRUPTED" : "CANCELLED"
const errorInfo = task.error ? `\n**Error:** ${task.error}` : ""
let notification: string
@@ -1382,13 +1399,8 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
}
const timer = setTimeout(() => {
this.completionTimers.delete(taskId)
const taskToRemove = this.tasks.get(taskId)
if (taskToRemove) {
if (this.tasks.has(taskId)) {
this.clearNotificationsForTask(taskId)
if (taskToRemove.sessionID) {
subagentSessions.delete(taskToRemove.sessionID)
SessionCategoryRegistry.remove(taskToRemove.sessionID)
}
this.tasks.delete(taskId)
log("[background-agent] Removed completed task from memory:", taskId)
}
@@ -1423,21 +1435,11 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
task.status = "error"
task.error = errorMessage
task.completedAt = new Date()
this.taskHistory.record(task.parentSessionID, { id: task.id, sessionID: task.sessionID, agent: task.agent, description: task.description, status: "error", category: task.category, startedAt: task.startedAt, completedAt: task.completedAt })
if (task.concurrencyKey) {
this.concurrencyManager.release(task.concurrencyKey)
task.concurrencyKey = undefined
}
const existingTimer = this.completionTimers.get(taskId)
if (existingTimer) {
clearTimeout(existingTimer)
this.completionTimers.delete(taskId)
}
const idleTimer = this.idleDeferralTimers.get(taskId)
if (idleTimer) {
clearTimeout(idleTimer)
this.idleDeferralTimers.delete(taskId)
}
this.cleanupPendingByParent(task)
if (wasPending) {
const key = task.model
? `${task.model.providerID}/${task.model.modelID}`
@@ -1453,10 +1455,16 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
}
}
}
this.markForNotification(task)
this.enqueueNotificationForParent(task.parentSessionID, () => this.notifyParentSession(task)).catch(err => {
log("[background-agent] Error in notifyParentSession for stale-pruned task:", { taskId: task.id, error: err })
})
this.clearNotificationsForTask(taskId)
const toastManager = getTaskToastManager()
if (toastManager) {
toastManager.removeTask(taskId)
}
this.tasks.delete(taskId)
if (task.sessionID) {
subagentSessions.delete(task.sessionID)
SessionCategoryRegistry.remove(task.sessionID)
}
},
})
}

View File

@@ -422,38 +422,4 @@ describe("pruneStaleTasksAndNotifications", () => {
//#then
expect(pruned).toContain("old-task")
})
it("should skip terminal tasks even when they exceeded TTL", () => {
//#given
const tasks = new Map<string, BackgroundTask>()
const oldStartedAt = new Date(Date.now() - 31 * 60 * 1000)
const terminalStatuses: BackgroundTask["status"][] = ["completed", "error", "cancelled", "interrupt"]
for (const status of terminalStatuses) {
tasks.set(status, {
id: status,
parentSessionID: "parent",
parentMessageID: "msg",
description: status,
prompt: status,
agent: "explore",
status,
startedAt: oldStartedAt,
completedAt: new Date(),
})
}
const pruned: string[] = []
//#when
pruneStaleTasksAndNotifications({
tasks,
notifications: new Map<string, BackgroundTask[]>(),
onTaskPruned: (taskId) => pruned.push(taskId),
})
//#then
expect(pruned).toEqual([])
expect(Array.from(tasks.keys())).toEqual(terminalStatuses)
})
})

View File

@@ -12,13 +12,6 @@ import {
TASK_TTL_MS,
} from "./constants"
const TERMINAL_TASK_STATUSES = new Set<BackgroundTask["status"]>([
"completed",
"error",
"cancelled",
"interrupt",
])
export function pruneStaleTasksAndNotifications(args: {
tasks: Map<string, BackgroundTask>
notifications: Map<string, BackgroundTask[]>
@@ -28,8 +21,6 @@ export function pruneStaleTasksAndNotifications(args: {
const now = Date.now()
for (const [taskId, task] of tasks.entries()) {
if (TERMINAL_TASK_STATUSES.has(task.status)) continue
const timestamp = task.status === "pending"
? task.queuedAt?.getTime()
: task.startedAt?.getTime()

View File

@@ -26,6 +26,11 @@ export const RETRYABLE_ERROR_PATTERNS = [
/rate.?limit/i,
/too.?many.?requests/i,
/quota.?exceeded/i,
/quota\s+will\s+reset\s+after/i,
/all\s+credentials\s+for\s+model/i,
/cool(?:ing)?\s+down/i,
/cooldown/i,
/exhausted\s+your\s+capacity/i,
/usage\s+limit\s+has\s+been\s+reached/i,
/service.?unavailable/i,
/overloaded/i,

View File

@@ -6,9 +6,11 @@ import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableErro
import { createFallbackState, prepareFallback } from "./fallback-state"
import { getFallbackModelsForSession } from "./fallback-models"
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
import { createSessionStatusHandler } from "./session-status-handler"
export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
const sessionStatusHandler = createSessionStatusHandler(deps, helpers)
const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
const sessionInfo = props?.info as { id?: string; model?: string } | undefined
@@ -33,6 +35,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
sessionRetryInFlight.delete(sessionID)
sessionAwaitingFallbackResult.delete(sessionID)
helpers.clearSessionFallbackTimeout(sessionID)
sessionStatusHandler.clearRetryKey(sessionID)
SessionCategoryRegistry.remove(sessionID)
}
}
@@ -191,6 +194,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
if (event.type === "session.deleted") { handleSessionDeleted(props); return }
if (event.type === "session.stop") { await handleSessionStop(props); return }
if (event.type === "session.idle") { handleSessionIdle(props); return }
if (event.type === "session.status") { await sessionStatusHandler.handleSessionStatus(props); return }
if (event.type === "session.error") { await handleSessionError(props); return }
}
}

View File

@@ -387,6 +387,133 @@ describe("runtime-fallback", () => {
expect(fallbackLog?.data).toMatchObject({ from: "openai/gpt-5.3-codex", to: "anthropic/claude-opus-4-6" })
})
test("should trigger fallback on session.status auto-retry signal", async () => {
const promptCalls: unknown[] = []
const hook = createRuntimeFallbackHook(
createMockPluginInput({
session: {
messages: async () => ({
data: [
{
info: { role: "user" },
parts: [{ type: "text", text: "continue" }],
},
],
}),
promptAsync: async (args: unknown) => {
promptCalls.push(args)
return {}
},
},
}),
{
config: createMockConfig({ notify_on_fallback: false }),
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.4"]),
}
)
const sessionID = "test-session-status-auto-retry"
SessionCategoryRegistry.register(sessionID, "test")
await hook.event({
event: {
type: "session.created",
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
},
})
await hook.event({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
attempt: 1,
next: 476,
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
},
},
},
})
const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal in session.status"))
expect(signalLog).toBeDefined()
const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
expect(fallbackLog).toBeDefined()
expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.4" })
expect(promptCalls).toHaveLength(1)
})
test("should deduplicate session.status countdown updates for the same retry attempt", async () => {
const promptCalls: unknown[] = []
const hook = createRuntimeFallbackHook(
createMockPluginInput({
session: {
messages: async () => ({
data: [
{
info: { role: "user" },
parts: [{ type: "text", text: "continue" }],
},
],
}),
promptAsync: async (args: unknown) => {
promptCalls.push(args)
return {}
},
},
}),
{
config: createMockConfig({ notify_on_fallback: false }),
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.4"]),
}
)
const sessionID = "test-session-status-countdown-dedup"
SessionCategoryRegistry.register(sessionID, "test")
await hook.event({
event: {
type: "session.created",
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
},
})
await hook.event({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
attempt: 1,
next: 476,
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
},
},
},
})
await hook.event({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
attempt: 1,
next: 475,
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]",
},
},
},
})
expect(promptCalls).toHaveLength(1)
})
test("should NOT trigger fallback on auto-retry signal when timeout_seconds is 0", async () => {
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 0 }),

View File

@@ -0,0 +1,160 @@
import type { HookDeps } from "./types"
import type { AutoRetryHelpers } from "./auto-retry"
import { HOOK_NAME } from "./constants"
import { log } from "../../shared/logger"
import { isRetryableError } from "./error-classifier"
import { createFallbackState, prepareFallback } from "./fallback-state"
import { getFallbackModelsForSession } from "./fallback-models"
import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../../shared/retry-status-utils"
type SessionStatus = {
type?: string
message?: string
attempt?: number
}
function resolveInitialModel(
props: Record<string, unknown> | undefined,
retryMessage: string,
resolvedAgent: string | undefined,
pluginConfig: HookDeps["pluginConfig"],
): string | undefined {
const eventModel = typeof props?.model === "string" ? props.model : undefined
if (eventModel) {
return eventModel
}
const retryModel = extractRetryStatusModel(retryMessage)
if (retryModel) {
return retryModel
}
const agentConfig = resolvedAgent
? pluginConfig?.agents?.[resolvedAgent as keyof typeof pluginConfig.agents]
: undefined
return typeof agentConfig?.model === "string" ? agentConfig.model : undefined
}
export function createSessionStatusHandler(deps: HookDeps, helpers: AutoRetryHelpers): {
clearRetryKey: (sessionID: string) => void
handleSessionStatus: (props: Record<string, unknown> | undefined) => Promise<void>
} {
const {
config,
pluginConfig,
sessionStates,
sessionLastAccess,
sessionRetryInFlight,
sessionAwaitingFallbackResult,
} = deps
const sessionStatusRetryKeys = new Map<string, string>()
const clearRetryKey = (sessionID: string): void => {
sessionStatusRetryKeys.delete(sessionID)
}
const handleSessionStatus = async (props: Record<string, unknown> | undefined): Promise<void> => {
const sessionID = props?.sessionID as string | undefined
const status = props?.status as SessionStatus | undefined
const agent = props?.agent as string | undefined
const timeoutEnabled = config.timeout_seconds > 0
if (!sessionID || status?.type !== "retry" || !timeoutEnabled) {
return
}
const retryMessage = typeof status.message === "string" ? status.message : ""
if (!retryMessage || !isRetryableError({ message: retryMessage }, config.retry_on_errors)) {
return
}
const currentState = sessionStates.get(sessionID)
const retryAttempt = extractRetryAttempt(status.attempt, retryMessage)
const retryModel =
(typeof props?.model === "string" ? props.model : undefined) ??
extractRetryStatusModel(retryMessage) ??
currentState?.currentModel ??
"unknown-model"
const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`
if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
return
}
sessionStatusRetryKeys.set(sessionID, retryKey)
if (sessionRetryInFlight.has(sessionID)) {
log(`[${HOOK_NAME}] Overriding in-flight retry due to provider session.status retry signal`, {
sessionID,
retryModel,
})
await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
sessionRetryInFlight.delete(sessionID)
}
sessionAwaitingFallbackResult.delete(sessionID)
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
if (fallbackModels.length === 0) {
log(`[${HOOK_NAME}] No fallback models configured`, { sessionID, agent: resolvedAgent ?? agent })
return
}
let state = currentState
if (!state) {
const initialModel = resolveInitialModel(props, retryMessage, resolvedAgent, pluginConfig)
if (!initialModel) {
log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
return
}
state = createFallbackState(initialModel)
sessionStates.set(sessionID, state)
}
sessionLastAccess.set(sessionID, Date.now())
if (state.pendingFallbackModel) {
log(`[${HOOK_NAME}] Clearing pending fallback due to provider session.status retry signal`, {
sessionID,
pendingFallbackModel: state.pendingFallbackModel,
})
state.pendingFallbackModel = undefined
}
log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
sessionID,
model: state.currentModel,
retryAttempt,
})
const result = prepareFallback(sessionID, state, fallbackModels, config)
if (result.success && config.notify_on_fallback) {
await deps.ctx.client.tui
.showToast({
body: {
title: "Model Fallback",
message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
variant: "warning",
duration: 5000,
},
})
.catch(() => {})
}
if (result.success && result.newModel) {
await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
return
}
log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
}
return {
clearRetryKey,
handleSessionStatus,
}
}

View File

@@ -6,7 +6,7 @@ import { _resetForTesting, setMainSession } from "../features/claude-code-sessio
import { createModelFallbackHook, clearPendingModelFallback } from "../hooks/model-fallback/hook"
describe("createEventHandler - model fallback", () => {
const createHandler = (args?: { hooks?: any }) => {
const createHandler = (args?: { hooks?: any; pluginConfig?: any }) => {
const abortCalls: string[] = []
const promptCalls: string[] = []
@@ -26,7 +26,7 @@ describe("createEventHandler - model fallback", () => {
},
},
} as any,
pluginConfig: {} as any,
pluginConfig: (args?.pluginConfig ?? {}) as any,
firstMessageVariantGate: {
markSessionCreated: () => {},
clear: () => {},
@@ -213,6 +213,121 @@ describe("createEventHandler - model fallback", () => {
expect(output.message["variant"]).toBe("max")
})
test("deduplicates session.status countdown updates for the same retry attempt", async () => {
//#given
const sessionID = "ses_status_retry_dedup"
setMainSession(sessionID)
clearPendingModelFallback(sessionID)
const modelFallback = createModelFallbackHook()
const { handler, abortCalls, promptCalls } = createHandler({ hooks: { modelFallback } })
await handler({
event: {
type: "message.updated",
properties: {
info: {
id: "msg_user_status_dedup",
sessionID,
role: "user",
modelID: "claude-opus-4-6-thinking",
providerID: "anthropic",
agent: "Sisyphus (Ultraworker)",
},
},
},
})
//#when
await handler({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
attempt: 1,
message:
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}} [retrying in 27s attempt #1]",
next: 27,
},
},
},
})
await handler({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
attempt: 1,
message:
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}} [retrying in 26s attempt #1]",
next: 26,
},
},
},
})
//#then
expect(abortCalls).toEqual([sessionID])
expect(promptCalls).toEqual([sessionID])
})
test("does not trigger model fallback from session.status when runtime fallback is enabled", async () => {
//#given
const sessionID = "ses_status_retry_runtime_enabled"
setMainSession(sessionID)
clearPendingModelFallback(sessionID)
const modelFallback = createModelFallbackHook()
const runtimeFallback = {
event: async () => {},
"chat.message": async () => {},
}
const { handler, abortCalls, promptCalls } = createHandler({
hooks: { modelFallback, runtimeFallback },
pluginConfig: { runtime_fallback: { enabled: true } },
})
await handler({
event: {
type: "message.updated",
properties: {
info: {
id: "msg_user_status_runtime_enabled",
sessionID,
role: "user",
modelID: "claude-opus-4-6-thinking",
providerID: "anthropic",
agent: "Sisyphus (Ultraworker)",
},
},
},
})
//#when
await handler({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
attempt: 1,
message:
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}} [retrying in 27s attempt #1]",
next: 27,
},
},
},
})
//#then
expect(abortCalls).toEqual([])
expect(promptCalls).toEqual([])
})
test("advances main-session fallback chain across repeated session.error retries end-to-end", async () => {
//#given
const abortCalls: string[] = []

View File

@@ -18,6 +18,7 @@ import {
import { resetMessageCursor } from "../shared";
import { log } from "../shared/logger";
import { shouldRetryError } from "../shared/model-error-classifier";
import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../shared/retry-status-utils";
import { clearSessionModel, setSessionModel } from "../shared/session-model-state";
import { deleteSessionTools } from "../shared/session-tools-store";
import { lspManager } from "../tools";
@@ -342,10 +343,15 @@ export function createEventHandler(args: {
const sessionID = props?.sessionID as string | undefined;
const status = props?.status as { type?: string; attempt?: number; message?: string; next?: number } | undefined;
if (sessionID && status?.type === "retry" && isModelFallbackEnabled) {
if (sessionID && status?.type === "retry" && !isRuntimeFallbackEnabled && isModelFallbackEnabled) {
try {
const retryMessage = typeof status.message === "string" ? status.message : "";
const retryKey = `${status.attempt ?? "?"}:${status.next ?? "?"}:${retryMessage}`;
const retryAttempt = extractRetryAttempt(status.attempt, retryMessage);
const retryModel =
extractRetryStatusModel(retryMessage) ??
lastKnownModelBySession.get(sessionID)?.modelID ??
"unknown-model";
const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`;
if (lastHandledRetryStatusKey.get(sessionID) === retryKey) {
return;
}

View File

@@ -0,0 +1,41 @@
import { describe, expect, test } from "bun:test"
import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "./retry-status-utils"
describe("retry-status-utils", () => {
test("extracts retry attempt from explicit status attempt", () => {
//#given
const attempt = 6
//#when
const result = extractRetryAttempt(attempt, "The usage limit has been reached [retrying in 27s attempt #6]")
//#then
expect(result).toBe(6)
})
test("extracts retry model from cooldown status text", () => {
//#given
const message = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]"
//#when
const result = extractRetryStatusModel(message)
//#then
expect(result).toBe("claude-opus-4-6")
})
test("normalizes countdown jitter to a stable cooldown class", () => {
//#given
const firstMessage = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]"
const secondMessage = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]"
//#when
const firstResult = normalizeRetryStatusMessage(firstMessage)
const secondResult = normalizeRetryStatusMessage(secondMessage)
//#then
expect(firstResult).toBe("cooldown")
expect(secondResult).toBe("cooldown")
})
})

View File

@@ -0,0 +1,51 @@
const RETRY_COUNTDOWN_PATTERN = /\[\s*retrying\s+in[^\]]*\]/gi
function collapseWhitespace(value: string): string {
return value.toLowerCase().replace(/\s+/g, " ").trim()
}
export function extractRetryAttempt(attempt: number | undefined, message: string): number | "?" {
if (typeof attempt === "number" && Number.isFinite(attempt)) {
return attempt
}
const parsedAttempt = message.match(/attempt\s*#\s*(\d+)/i)?.[1]
return parsedAttempt ? Number.parseInt(parsedAttempt, 10) : "?"
}
export function extractRetryStatusModel(message: string): string | undefined {
return message.match(/model\s+([a-z0-9._/-]+)(?=\s+(?:are|is)\b)/i)?.[1]?.toLowerCase()
}
export function normalizeRetryStatusMessage(message: string): string {
const normalizedMessage = collapseWhitespace(message.replace(RETRY_COUNTDOWN_PATTERN, " "))
if (!normalizedMessage) {
return "retry"
}
if (/all\s+credentials\s+for\s+model|cool(?:ing)?\s+down|cooldown|exhausted\s+your\s+capacity/.test(normalizedMessage)) {
return "cooldown"
}
if (/too\s+many\s+requests/.test(normalizedMessage)) {
return "too-many-requests"
}
if (/quota\s+will\s+reset\s+after|quota\s*exceeded/.test(normalizedMessage)) {
return "quota"
}
if (/usage\s+limit\s+has\s+been\s+reached|limit\s+reached/.test(normalizedMessage)) {
return "usage-limit"
}
if (/rate\s+limit/.test(normalizedMessage)) {
return "rate-limit"
}
if (/service.?unavailable|temporarily.?unavailable|overloaded/.test(normalizedMessage)) {
return "service-unavailable"
}
return normalizedMessage
}