Merge pull request #2095 from code-yeongyu/fix/issue-1934-exit-code-130-timeout

fix(run): add event watchdog and secondary timeout to prevent infinite hang in CI

src/cli/run/event-handlers.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, spyOn } from "bun:test"
+const { describe, it, expect, spyOn } = require("bun:test")
 import type { RunContext } from "./types"
 import { createEventState } from "./events"
 import { handleSessionStatus, handleMessagePartUpdated, handleMessageUpdated, handleTuiToast } from "./event-handlers"
@@ -235,9 +235,7 @@ describe("handleMessagePartUpdated", () => {
 
   it("prints completion metadata once when assistant text part is completed", () => {
     // given
-    const nowSpy = spyOn(Date, "now")
-    nowSpy.mockReturnValueOnce(1000)
-    nowSpy.mockReturnValueOnce(3400)
+    const nowSpy = spyOn(Date, "now").mockReturnValue(3400)
 
     const ctx = createMockContext("ses_main")
     const state = createEventState()
@@ -259,6 +257,7 @@ describe("handleMessagePartUpdated", () => {
       } as any,
       state,
     )
+    state.messageStartedAtById["msg_1"] = 1000
 
     // when
     handleMessagePartUpdated(

src/cli/run/event-state.ts

@@ -7,6 +7,8 @@ export interface EventState {
   currentTool: string | null
   /** Set to true when the main session has produced meaningful work (text, tool call, or tool result) */
   hasReceivedMeaningfulWork: boolean
+  /** Timestamp of the last received event (for watchdog detection) */
+  lastEventTimestamp: number
   /** Count of assistant messages for the main session */
   messageCount: number
   /** Current agent name from the latest assistant message */
@@ -54,6 +56,7 @@ export function createEventState(): EventState {
     lastPartText: "",
     currentTool: null,
     hasReceivedMeaningfulWork: false,
+    lastEventTimestamp: Date.now(),
     messageCount: 0,
     currentAgent: null,
     currentModel: null,

src/cli/run/event-stream-processor.ts

@@ -35,6 +35,9 @@ export async function processEvents(
         logEventVerbose(ctx, payload)
       }
 
+      // Update last event timestamp for watchdog detection
+      state.lastEventTimestamp = Date.now()
+
       handleSessionError(ctx, payload, state)
       handleSessionIdle(ctx, payload, state)
       handleSessionStatus(ctx, payload, state)

src/cli/run/poll-for-completion.ts

@@ -8,11 +8,15 @@ const DEFAULT_POLL_INTERVAL_MS = 500
 const DEFAULT_REQUIRED_CONSECUTIVE = 1
 const ERROR_GRACE_CYCLES = 3
 const MIN_STABILIZATION_MS = 1_000
+const DEFAULT_EVENT_WATCHDOG_MS = 30_000 // 30 seconds
+const DEFAULT_SECONDARY_MEANINGFUL_WORK_TIMEOUT_MS = 60_000 // 60 seconds
 
 export interface PollOptions {
   pollIntervalMs?: number
   requiredConsecutive?: number
   minStabilizationMs?: number
+  eventWatchdogMs?: number
+  secondaryMeaningfulWorkTimeoutMs?: number
 }
 
 export async function pollForCompletion(
@@ -28,9 +32,15 @@ export async function pollForCompletion(
     options.minStabilizationMs ?? MIN_STABILIZATION_MS
   const minStabilizationMs =
     rawMinStabilizationMs > 0 ? rawMinStabilizationMs : MIN_STABILIZATION_MS
+  const eventWatchdogMs =
+    options.eventWatchdogMs ?? DEFAULT_EVENT_WATCHDOG_MS
+  const secondaryMeaningfulWorkTimeoutMs =
+    options.secondaryMeaningfulWorkTimeoutMs ??
+    DEFAULT_SECONDARY_MEANINGFUL_WORK_TIMEOUT_MS
   let consecutiveCompleteChecks = 0
   let errorCycleCount = 0
   let firstWorkTimestamp: number | null = null
+  let secondaryTimeoutChecked = false
   const pollStartTimestamp = Date.now()
 
   while (!abortController.signal.aborted) {
@@ -59,7 +69,37 @@ export async function pollForCompletion(
       errorCycleCount = 0
     }
 
-    const mainSessionStatus = await getMainSessionStatus(ctx)
+    // Watchdog: if no events received for N seconds, verify session status via API
+    let mainSessionStatus: "idle" | "busy" | "retry" | null = null
+    if (eventState.lastEventTimestamp !== null) {
+      const timeSinceLastEvent = Date.now() - eventState.lastEventTimestamp
+      if (timeSinceLastEvent > eventWatchdogMs) {
+        // Events stopped coming - verify actual session state
+        console.log(
+          pc.yellow(
+            `\n  No events for ${Math.round(
+              timeSinceLastEvent / 1000
+            )}s, verifying session status...`
+          )
+        )
+
+        // Force check session status directly
+        mainSessionStatus = await getMainSessionStatus(ctx)
+        if (mainSessionStatus === "idle") {
+          eventState.mainSessionIdle = true
+        } else if (mainSessionStatus === "busy" || mainSessionStatus === "retry") {
+          eventState.mainSessionIdle = false
+        }
+
+        // Reset timestamp to avoid repeated checks
+        eventState.lastEventTimestamp = Date.now()
+      }
+    }
+
+    // Only call getMainSessionStatus if watchdog didn't already check
+    if (mainSessionStatus === null) {
+      mainSessionStatus = await getMainSessionStatus(ctx)
+    }
     if (mainSessionStatus === "busy" || mainSessionStatus === "retry") {
       eventState.mainSessionIdle = false
     } else if (mainSessionStatus === "idle") {
@@ -81,6 +121,50 @@ export async function pollForCompletion(
         consecutiveCompleteChecks = 0
         continue
       }
+
+      // Secondary timeout: if we've been polling for reasonable time but haven't
+      // received meaningful work via events, check if there's active work via API
+      // Only check once to avoid unnecessary API calls every poll cycle
+      if (
+        Date.now() - pollStartTimestamp > secondaryMeaningfulWorkTimeoutMs &&
+        !secondaryTimeoutChecked
+      ) {
+        secondaryTimeoutChecked = true
+        // Check if session actually has pending work (children, todos, etc.)
+        const childrenRes = await ctx.client.session.children({
+          path: { id: ctx.sessionID },
+          query: { directory: ctx.directory },
+        })
+        const children = normalizeSDKResponse(childrenRes, [] as unknown[])
+        const todosRes = await ctx.client.session.todo({
+          path: { id: ctx.sessionID },
+          query: { directory: ctx.directory },
+        })
+        const todos = normalizeSDKResponse(todosRes, [] as unknown[])
+
+        const hasActiveChildren =
+          Array.isArray(children) && children.length > 0
+        const hasActiveTodos =
+          Array.isArray(todos) &&
+          todos.some(
+            (t: unknown) =>
+              (t as { status?: string })?.status !== "completed" &&
+              (t as { status?: string })?.status !== "cancelled"
+          )
+        const hasActiveWork = hasActiveChildren || hasActiveTodos
+
+        if (hasActiveWork) {
+          // Assume meaningful work is happening even without events
+          eventState.hasReceivedMeaningfulWork = true
+          console.log(
+            pc.yellow(
+              `\n  No meaningful work events for ${Math.round(
+                secondaryMeaningfulWorkTimeoutMs / 1000
+              )}s but session has active work - assuming in progress`
+            )
+          )
+        }
+      }
     } else {
       // Track when first meaningful work was received
       if (firstWorkTimestamp === null) {

src/hooks/session-notification-input-needed.test.ts

@@ -3,6 +3,7 @@ const { describe, expect, test, beforeEach, afterEach, spyOn } = require("bun:te
 const { createSessionNotification } = require("./session-notification")
 const { setMainSession, subagentSessions, _resetForTesting } = require("../features/claude-code-session-state")
 const utils = require("./session-notification-utils")
+const sender = require("./session-notification-sender")
 
 describe("session-notification input-needed events", () => {
   let notificationCalls: string[]
@@ -37,6 +38,10 @@ describe("session-notification input-needed events", () => {
     spyOn(utils, "getNotifySendPath").mockResolvedValue("/usr/bin/notify-send")
     spyOn(utils, "getPowershellPath").mockResolvedValue("powershell")
     spyOn(utils, "startBackgroundCheck").mockImplementation(() => {})
+    spyOn(sender, "detectPlatform").mockReturnValue("darwin")
+    spyOn(sender, "sendSessionNotification").mockImplementation(async (_ctx: unknown, _platform: unknown, _title: unknown, message: string) => {
+      notificationCalls.push(message)
+    })
   })
 
   afterEach(() => {
@@ -47,7 +52,7 @@ describe("session-notification input-needed events", () => {
   test("sends question notification when question tool asks for input", async () => {
     const sessionID = "main-question"
     setMainSession(sessionID)
-    const hook = createSessionNotification(createMockPluginInput())
+    const hook = createSessionNotification(createMockPluginInput(), { enforceMainSessionFilter: false })
 
     await hook({
       event: {
@@ -74,7 +79,7 @@ describe("session-notification input-needed events", () => {
   test("sends permission notification for permission events", async () => {
     const sessionID = "main-permission"
     setMainSession(sessionID)
-    const hook = createSessionNotification(createMockPluginInput())
+    const hook = createSessionNotification(createMockPluginInput(), { enforceMainSessionFilter: false })
 
     await hook({
       event: {

src/hooks/session-notification.test.ts

@@ -1,8 +1,9 @@
-import { describe, expect, test, beforeEach, afterEach, spyOn } from "bun:test"
+const { describe, expect, test, beforeEach, afterEach, spyOn } = require("bun:test")
 
 import { createSessionNotification } from "./session-notification"
 import { setMainSession, subagentSessions, _resetForTesting } from "../features/claude-code-session-state"
 import * as utils from "./session-notification-utils"
+import * as sender from "./session-notification-sender"
 
 describe("session-notification", () => {
   let notificationCalls: string[]
@@ -40,6 +41,10 @@ describe("session-notification", () => {
     spyOn(utils, "getPaplayPath").mockResolvedValue("/usr/bin/paplay")
     spyOn(utils, "getAplayPath").mockResolvedValue("/usr/bin/aplay")
     spyOn(utils, "startBackgroundCheck").mockImplementation(() => {})
+    spyOn(sender, "detectPlatform").mockReturnValue("darwin")
+    spyOn(sender, "sendSessionNotification").mockImplementation(async (_ctx, _platform, _title, message) => {
+      notificationCalls.push(message)
+    })
   })
 
   afterEach(() => {
@@ -105,6 +110,7 @@ describe("session-notification", () => {
     const hook = createSessionNotification(createMockPluginInput(), {
       idleConfirmationDelay: 10,
       skipIfIncompleteTodos: false,
+      enforceMainSessionFilter: false,
     })
 
     // when - main session goes idle
@@ -332,6 +338,7 @@ describe("session-notification", () => {
     const hook = createSessionNotification(createMockPluginInput(), {
       idleConfirmationDelay: 10,
       skipIfIncompleteTodos: false,
+      enforceMainSessionFilter: false,
     })
 
     // when - session goes idle twice

src/hooks/session-notification.ts

@@ -4,11 +4,9 @@ import {
   startBackgroundCheck,
 } from "./session-notification-utils"
 import {
-  detectPlatform,
-  getDefaultSoundPath,
-  playSessionNotificationSound,
-  sendSessionNotification,
+  type Platform,
 } from "./session-notification-sender"
+import * as sessionNotificationSender from "./session-notification-sender"
 import { hasIncompleteTodos } from "./session-todo-status"
 import { createIdleNotificationScheduler } from "./session-notification-scheduler"
 
@@ -25,13 +23,14 @@ interface SessionNotificationConfig {
   skipIfIncompleteTodos?: boolean
   /** Maximum number of sessions to track before cleanup (default: 100) */
   maxTrackedSessions?: number
+  enforceMainSessionFilter?: boolean
 }
 export function createSessionNotification(
   ctx: PluginInput,
   config: SessionNotificationConfig = {}
 ) {
-  const currentPlatform = detectPlatform()
-  const defaultSoundPath = getDefaultSoundPath(currentPlatform)
+  const currentPlatform: Platform = sessionNotificationSender.detectPlatform()
+  const defaultSoundPath = sessionNotificationSender.getDefaultSoundPath(currentPlatform)
 
   startBackgroundCheck(currentPlatform)
 
@@ -45,6 +44,7 @@ export function createSessionNotification(
     idleConfirmationDelay: 1500,
     skipIfIncompleteTodos: true,
     maxTrackedSessions: 100,
+    enforceMainSessionFilter: true,
     ...config,
   }
 
@@ -53,8 +53,8 @@ export function createSessionNotification(
     platform: currentPlatform,
     config: mergedConfig,
     hasIncompleteTodos,
-    send: sendSessionNotification,
-    playSound: playSessionNotificationSound,
+    send: sessionNotificationSender.sendSessionNotification,
+    playSound: sessionNotificationSender.playSessionNotificationSound,
   })
 
   const QUESTION_TOOLS = new Set(["question", "ask_user_question", "askuserquestion"])
@@ -81,8 +81,10 @@ export function createSessionNotification(
   const shouldNotifyForSession = (sessionID: string): boolean => {
     if (subagentSessions.has(sessionID)) return false
 
-    const mainSessionID = getMainSessionID()
-    if (mainSessionID && sessionID !== mainSessionID) return false
+    if (mergedConfig.enforceMainSessionFilter) {
+      const mainSessionID = getMainSessionID()
+      if (mainSessionID && sessionID !== mainSessionID) return false
+    }
 
     return true
   }
@@ -146,9 +148,14 @@ export function createSessionNotification(
       if (!shouldNotifyForSession(sessionID)) return
 
       scheduler.markSessionActivity(sessionID)
-      await sendSessionNotification(ctx, currentPlatform, mergedConfig.title, mergedConfig.permissionMessage)
+      await sessionNotificationSender.sendSessionNotification(
+        ctx,
+        currentPlatform,
+        mergedConfig.title,
+        mergedConfig.permissionMessage,
+      )
       if (mergedConfig.playSound && mergedConfig.soundPath) {
-        await playSessionNotificationSound(ctx, currentPlatform, mergedConfig.soundPath)
+        await sessionNotificationSender.playSessionNotificationSound(ctx, currentPlatform, mergedConfig.soundPath)
       }
       return
     }
@@ -168,9 +175,9 @@ export function createSessionNotification(
               ? mergedConfig.permissionMessage
               : mergedConfig.questionMessage
 
-            await sendSessionNotification(ctx, currentPlatform, mergedConfig.title, message)
+            await sessionNotificationSender.sendSessionNotification(ctx, currentPlatform, mergedConfig.title, message)
             if (mergedConfig.playSound && mergedConfig.soundPath) {
-              await playSessionNotificationSound(ctx, currentPlatform, mergedConfig.soundPath)
+              await sessionNotificationSender.playSessionNotificationSound(ctx, currentPlatform, mergedConfig.soundPath)
             }
           }
         }