Gate model fallback session.status retries

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

src/hooks/runtime-fallback/constants.ts

@@ -29,6 +29,7 @@ export const RETRYABLE_ERROR_PATTERNS = [
   /quota\s+will\s+reset\s+after/i,
   /all\s+credentials\s+for\s+model/i,
   /cool(?:ing)?\s+down/i,
+  /cooldown/i,
   /exhausted\s+your\s+capacity/i,
   /usage\s+limit\s+has\s+been\s+reached/i,
   /service.?unavailable/i,

src/hooks/runtime-fallback/event-handler.ts

@@ -2,15 +2,15 @@ import type { HookDeps } from "./types"
 import type { AutoRetryHelpers } from "./auto-retry"
 import { HOOK_NAME } from "./constants"
 import { log } from "../../shared/logger"
-import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError, extractAutoRetrySignal } from "./error-classifier"
+import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError } from "./error-classifier"
 import { createFallbackState, prepareFallback } from "./fallback-state"
 import { getFallbackModelsForSession } from "./fallback-models"
 import { SessionCategoryRegistry } from "../../shared/session-category-registry"
-import { normalizeRetryStatusMessage, extractRetryAttempt } from "../../shared/retry-status-utils"
+import { createSessionStatusHandler } from "./session-status-handler"
 
 export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
   const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
-  const sessionStatusRetryKeys = new Map<string, string>()
+  const sessionStatusHandler = createSessionStatusHandler(deps, helpers)
 
   const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
     const sessionInfo = props?.info as { id?: string; model?: string } | undefined
@@ -35,7 +35,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
       sessionRetryInFlight.delete(sessionID)
       sessionAwaitingFallbackResult.delete(sessionID)
       helpers.clearSessionFallbackTimeout(sessionID)
-      sessionStatusRetryKeys.delete(sessionID)
+      sessionStatusHandler.clearRetryKey(sessionID)
       SessionCategoryRegistry.remove(sessionID)
     }
   }
@@ -185,88 +185,6 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
     }
   }
 
-  const handleSessionStatus = async (props: Record<string, unknown> | undefined) => {
-    const sessionID = props?.sessionID as string | undefined
-    const status = props?.status as { type?: string; message?: string; attempt?: number } | undefined
-    const agent = props?.agent as string | undefined
-    const model = props?.model as string | undefined
-
-    if (!sessionID || status?.type !== "retry") return
-
-    const retryMessage = typeof status.message === "string" ? status.message : ""
-    const retrySignal = extractAutoRetrySignal({ status: retryMessage, message: retryMessage })
-    if (!retrySignal) return
-
-    const retryKey = `${extractRetryAttempt(status.attempt, retryMessage)}:${normalizeRetryStatusMessage(retryMessage)}`
-    if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
-      return
-    }
-    sessionStatusRetryKeys.set(sessionID, retryKey)
-
-    if (sessionRetryInFlight.has(sessionID)) {
-      log(`[${HOOK_NAME}] session.status retry skipped — retry already in flight`, { sessionID })
-      return
-    }
-
-    const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
-    const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
-    if (fallbackModels.length === 0) return
-
-    let state = sessionStates.get(sessionID)
-    if (!state) {
-      const detectedAgent = resolvedAgent
-      const agentConfig = detectedAgent
-        ? pluginConfig?.agents?.[detectedAgent as keyof typeof pluginConfig.agents]
-        : undefined
-      const inferredModel = model || (agentConfig?.model as string | undefined)
-      if (!inferredModel) {
-        log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
-        return
-      }
-      state = createFallbackState(inferredModel)
-      sessionStates.set(sessionID, state)
-    }
-    sessionLastAccess.set(sessionID, Date.now())
-
-    if (state.pendingFallbackModel) {
-      log(`[${HOOK_NAME}] session.status retry skipped (pending fallback in progress)`, {
-        sessionID,
-        pendingFallbackModel: state.pendingFallbackModel,
-      })
-      return
-    }
-
-    log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
-      sessionID,
-      model: state.currentModel,
-      retryAttempt: status.attempt,
-    })
-
-    await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
-
-    const result = prepareFallback(sessionID, state, fallbackModels, config)
-    if (result.success && config.notify_on_fallback) {
-      await deps.ctx.client.tui
-        .showToast({
-          body: {
-            title: "Model Fallback",
-            message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
-            variant: "warning",
-            duration: 5000,
-          },
-        })
-        .catch(() => {})
-    }
-
-    if (result.success && result.newModel) {
-      await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
-    }
-
-    if (!result.success) {
-      log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
-    }
-  }
-
   return async ({ event }: { event: { type: string; properties?: unknown } }) => {
     if (!config.enabled) return
 
@@ -276,7 +194,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
     if (event.type === "session.deleted") { handleSessionDeleted(props); return }
     if (event.type === "session.stop") { await handleSessionStop(props); return }
     if (event.type === "session.idle") { handleSessionIdle(props); return }
-    if (event.type === "session.status") { await handleSessionStatus(props); return }
+    if (event.type === "session.status") { await sessionStatusHandler.handleSessionStatus(props); return }
     if (event.type === "session.error") { await handleSessionError(props); return }
   }
 }

src/hooks/runtime-fallback/session-status-handler.ts

@@ -0,0 +1,160 @@
+import type { HookDeps } from "./types"
+import type { AutoRetryHelpers } from "./auto-retry"
+import { HOOK_NAME } from "./constants"
+import { log } from "../../shared/logger"
+import { isRetryableError } from "./error-classifier"
+import { createFallbackState, prepareFallback } from "./fallback-state"
+import { getFallbackModelsForSession } from "./fallback-models"
+import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../../shared/retry-status-utils"
+
+type SessionStatus = {
+  type?: string
+  message?: string
+  attempt?: number
+}
+
+function resolveInitialModel(
+  props: Record<string, unknown> | undefined,
+  retryMessage: string,
+  resolvedAgent: string | undefined,
+  pluginConfig: HookDeps["pluginConfig"],
+): string | undefined {
+  const eventModel = typeof props?.model === "string" ? props.model : undefined
+  if (eventModel) {
+    return eventModel
+  }
+
+  const retryModel = extractRetryStatusModel(retryMessage)
+  if (retryModel) {
+    return retryModel
+  }
+
+  const agentConfig = resolvedAgent
+    ? pluginConfig?.agents?.[resolvedAgent as keyof typeof pluginConfig.agents]
+    : undefined
+
+  return typeof agentConfig?.model === "string" ? agentConfig.model : undefined
+}
+
+export function createSessionStatusHandler(deps: HookDeps, helpers: AutoRetryHelpers): {
+  clearRetryKey: (sessionID: string) => void
+  handleSessionStatus: (props: Record<string, unknown> | undefined) => Promise<void>
+} {
+  const {
+    config,
+    pluginConfig,
+    sessionStates,
+    sessionLastAccess,
+    sessionRetryInFlight,
+    sessionAwaitingFallbackResult,
+  } = deps
+  const sessionStatusRetryKeys = new Map<string, string>()
+
+  const clearRetryKey = (sessionID: string): void => {
+    sessionStatusRetryKeys.delete(sessionID)
+  }
+
+  const handleSessionStatus = async (props: Record<string, unknown> | undefined): Promise<void> => {
+    const sessionID = props?.sessionID as string | undefined
+    const status = props?.status as SessionStatus | undefined
+    const agent = props?.agent as string | undefined
+    const timeoutEnabled = config.timeout_seconds > 0
+
+    if (!sessionID || status?.type !== "retry" || !timeoutEnabled) {
+      return
+    }
+
+    const retryMessage = typeof status.message === "string" ? status.message : ""
+    if (!retryMessage || !isRetryableError({ message: retryMessage }, config.retry_on_errors)) {
+      return
+    }
+
+    const currentState = sessionStates.get(sessionID)
+    const retryAttempt = extractRetryAttempt(status.attempt, retryMessage)
+    const retryModel =
+      (typeof props?.model === "string" ? props.model : undefined) ??
+      extractRetryStatusModel(retryMessage) ??
+      currentState?.currentModel ??
+      "unknown-model"
+    const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`
+
+    if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
+      return
+    }
+    sessionStatusRetryKeys.set(sessionID, retryKey)
+
+    if (sessionRetryInFlight.has(sessionID)) {
+      log(`[${HOOK_NAME}] Overriding in-flight retry due to provider session.status retry signal`, {
+        sessionID,
+        retryModel,
+      })
+      await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
+      sessionRetryInFlight.delete(sessionID)
+    }
+
+    sessionAwaitingFallbackResult.delete(sessionID)
+
+    const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
+    const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
+
+    if (fallbackModels.length === 0) {
+      log(`[${HOOK_NAME}] No fallback models configured`, { sessionID, agent: resolvedAgent ?? agent })
+      return
+    }
+
+    let state = currentState
+    if (!state) {
+      const initialModel = resolveInitialModel(props, retryMessage, resolvedAgent, pluginConfig)
+      if (!initialModel) {
+        log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
+        return
+      }
+
+      state = createFallbackState(initialModel)
+      sessionStates.set(sessionID, state)
+    }
+
+    sessionLastAccess.set(sessionID, Date.now())
+
+    if (state.pendingFallbackModel) {
+      log(`[${HOOK_NAME}] Clearing pending fallback due to provider session.status retry signal`, {
+        sessionID,
+        pendingFallbackModel: state.pendingFallbackModel,
+      })
+      state.pendingFallbackModel = undefined
+    }
+
+    log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
+      sessionID,
+      model: state.currentModel,
+      retryAttempt,
+    })
+
+    const result = prepareFallback(sessionID, state, fallbackModels, config)
+
+    if (result.success && config.notify_on_fallback) {
+      await deps.ctx.client.tui
+        .showToast({
+          body: {
+            title: "Model Fallback",
+            message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
+            variant: "warning",
+            duration: 5000,
+          },
+        })
+        .catch(() => {})
+    }
+
+    if (result.success && result.newModel) {
+      await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
+      return
+    }
+
+    log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
+  }
+
+  return {
+    clearRetryKey,
+    handleSessionStatus,
+  }
+}

src/plugin/event.ts

@@ -22,7 +22,7 @@ import { getAgentConfigKey } from "../shared/agent-display-names";
 import { log } from "../shared/logger";
 import { shouldRetryError } from "../shared/model-error-classifier";
 import { buildFallbackChainFromModels } from "../shared/fallback-chain-from-models";
-import { extractRetryAttempt, normalizeRetryStatusMessage } from "../shared/retry-status-utils";
+import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../shared/retry-status-utils";
 import { clearSessionModel, setSessionModel } from "../shared/session-model-state";
 import { deleteSessionTools } from "../shared/session-tools-store";
 import { lspManager } from "../tools";
@@ -387,11 +387,14 @@ export function createEventHandler(args: {
       if (sessionID && status?.type === "retry" && isModelFallbackEnabled && !isRuntimeFallbackEnabled) {
         try {
           const retryMessage = typeof status.message === "string" ? status.message : "";
-          const parsedForKey = extractProviderModelFromErrorMessage(retryMessage);
           const retryAttempt = extractRetryAttempt(status.attempt, retryMessage);
+          const retryModel =
+            extractRetryStatusModel(retryMessage) ??
+            lastKnownModelBySession.get(sessionID)?.modelID ??
+            "unknown-model";
           // Deduplicate countdown updates for the same retry attempt/model.
           // Messages like "retrying in 7m 56s" change every second but should only trigger once.
-          const retryKey = `${retryAttempt}:${parsedForKey.providerID ?? ""}/${parsedForKey.modelID ?? ""}:${normalizeRetryStatusMessage(retryMessage)}`;
+          const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`;
           if (lastHandledRetryStatusKey.get(sessionID) === retryKey) {
             return;
           }

src/shared/retry-status-utils.test.ts

@@ -0,0 +1,42 @@
+declare const require: (name: string) => any
+const { describe, expect, test } = require("bun:test")
+
+import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "./retry-status-utils"
+
+describe("retry-status-utils", () => {
+  test("extracts retry attempt from explicit status attempt", () => {
+    //#given
+    const attempt = 6
+
+    //#when
+    const result = extractRetryAttempt(attempt, "The usage limit has been reached [retrying in 27s attempt #6]")
+
+    //#then
+    expect(result).toBe(6)
+  })
+
+  test("extracts retry model from cooldown status text", () => {
+    //#given
+    const message = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]"
+
+    //#when
+    const result = extractRetryStatusModel(message)
+
+    //#then
+    expect(result).toBe("claude-opus-4-6")
+  })
+
+  test("normalizes countdown jitter to a stable cooldown class", () => {
+    //#given
+    const firstMessage = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]"
+    const secondMessage = "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]"
+
+    //#when
+    const firstResult = normalizeRetryStatusMessage(firstMessage)
+    const secondResult = normalizeRetryStatusMessage(secondMessage)
+
+    //#then
+    expect(firstResult).toBe("cooldown")
+    expect(secondResult).toBe("cooldown")
+  })
+})

src/shared/retry-status-utils.ts

@@ -1,19 +1,51 @@
-export function normalizeRetryStatusMessage(message: string): string {
-  return message
-    .replace(/\[retrying in [^\]]*attempt\s*#\d+\]/gi, "[retrying]")
-    .replace(/retrying in\s+[^(]*attempt\s*#\d+/gi, "retrying")
-    .replace(/\s+/g, " ")
-    .trim()
-    .toLowerCase()
+const RETRY_COUNTDOWN_PATTERN = /\[\s*retrying\s+in[^\]]*\]/gi
+
+function collapseWhitespace(value: string): string {
+  return value.toLowerCase().replace(/\s+/g, " ").trim()
 }
 
-export function extractRetryAttempt(statusAttempt: unknown, message: string): string {
-  if (typeof statusAttempt === "number" && Number.isFinite(statusAttempt)) {
-    return String(statusAttempt)
+export function extractRetryAttempt(attempt: number | undefined, message: string): number | "?" {
+  if (typeof attempt === "number" && Number.isFinite(attempt)) {
+    return attempt
   }
-  const attemptMatch = message.match(/attempt\s*#\s*(\d+)/i)
-  if (attemptMatch?.[1]) {
-    return attemptMatch[1]
-  }
-  return "?"
+
+  const parsedAttempt = message.match(/attempt\s*#\s*(\d+)/i)?.[1]
+  return parsedAttempt ? Number.parseInt(parsedAttempt, 10) : "?"
+}
+
+export function extractRetryStatusModel(message: string): string | undefined {
+  return message.match(/model\s+([a-z0-9._/-]+)(?=\s+(?:are|is)\b)/i)?.[1]?.toLowerCase()
+}
+
+export function normalizeRetryStatusMessage(message: string): string {
+  const normalizedMessage = collapseWhitespace(message.replace(RETRY_COUNTDOWN_PATTERN, " "))
+  if (!normalizedMessage) {
+    return "retry"
+  }
+
+  if (/all\s+credentials\s+for\s+model|cool(?:ing)?\s+down|cooldown|exhausted\s+your\s+capacity/.test(normalizedMessage)) {
+    return "cooldown"
+  }
+
+  if (/too\s+many\s+requests/.test(normalizedMessage)) {
+    return "too-many-requests"
+  }
+
+  if (/quota\s+will\s+reset\s+after|quota\s*exceeded/.test(normalizedMessage)) {
+    return "quota"
+  }
+
+  if (/usage\s+limit\s+has\s+been\s+reached|limit\s+reached/.test(normalizedMessage)) {
+    return "usage-limit"
+  }
+
+  if (/rate\s+limit/.test(normalizedMessage)) {
+    return "rate-limit"
+  }
+
+  if (/service.?unavailable|temporarily.?unavailable|overloaded/.test(normalizedMessage)) {
+    return "service-unavailable"
+  }
+
+  return normalizedMessage
 }