From aaaeb6997cdd5987161e5f3aeab7a1f14e006668 Mon Sep 17 00:00:00 2001 From: MoerAI Date: Mon, 23 Mar 2026 20:46:10 +0900 Subject: [PATCH 1/3] fix(ralph-loop): add semantic completion detection as fallback for natural language (fixes #2489) --- .../completion-promise-detector.test.ts | 124 +++++++++++++++++- .../ralph-loop/completion-promise-detector.ts | 25 ++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/src/hooks/ralph-loop/completion-promise-detector.test.ts b/src/hooks/ralph-loop/completion-promise-detector.test.ts index b63640457..704517732 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.test.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.test.ts @@ -1,7 +1,7 @@ /// import { describe, expect, test } from "bun:test" import type { PluginInput } from "@opencode-ai/plugin" -import { detectCompletionInSessionMessages } from "./completion-promise-detector" +import { detectCompletionInSessionMessages, detectSemanticCompletion } from "./completion-promise-detector" type SessionMessage = { info?: { role?: string } @@ -184,4 +184,126 @@ describe("detectCompletionInSessionMessages", () => { expect(detected).toBe(false) }) }) + + describe("#given semantic completion patterns", () => { + test("#when agent says 'task is complete' #then should detect semantic completion", async () => { + // #given + const messages: SessionMessage[] = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "The task is complete. All work has been finished." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-123", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(true) + }) + + test("#when agent says 'all items are done' #then should detect semantic completion", async () => { + // #given + const messages: SessionMessage[] = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "All items are done and marked as complete." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-123", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(true) + }) + + test("#when agent says 'nothing left to do' #then should detect semantic completion", async () => { + // #given + const messages: SessionMessage[] = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "There is nothing left to do. Everything is finished." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-123", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(true) + }) + + test("#when agent says 'successfully completed all' #then should detect semantic completion", async () => { + // #given + const messages: SessionMessage[] = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "I have successfully completed all the required tasks." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-123", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(true) + }) + }) +}) + +describe("detectSemanticCompletion", () => { + describe("#given semantic completion patterns", () => { + test("#when text contains 'task is complete' #then should return true", () => { + expect(detectSemanticCompletion("The task is complete.")).toBe(true) + }) + + test("#when text contains 'all items are done' #then should return true", () => { + expect(detectSemanticCompletion("All items are done.")).toBe(true) + }) + + test("#when text contains 'nothing left to do' #then should return true", () => { + expect(detectSemanticCompletion("There is nothing left to do.")).toBe(true) + }) + + test("#when text contains 'successfully completed all' #then should return true", () => { + expect(detectSemanticCompletion("Successfully completed all tasks.")).toBe(true) + }) + + test("#when text contains 'everything is finished' #then should return true", () => { + expect(detectSemanticCompletion("Everything is finished.")).toBe(true) + }) + + test("#when text does NOT contain completion patterns #then should return false", () => { + expect(detectSemanticCompletion("Working on the next task.")).toBe(false) + }) + + test("#when text is empty #then should return false", () => { + expect(detectSemanticCompletion("")).toBe(false) + }) + }) }) diff --git a/src/hooks/ralph-loop/completion-promise-detector.ts b/src/hooks/ralph-loop/completion-promise-detector.ts index 40e9e1af7..39322e536 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.ts @@ -17,6 +17,18 @@ function buildPromisePattern(promise: string): RegExp { return new RegExp(`\\s*${escapeRegex(promise)}\\s*`, "is") } +const SEMANTIC_COMPLETION_PATTERNS = [ + /\b(?:task|work|implementation|all\s+tasks?)\s+(?:is|are)\s+(?:complete|completed|done|finished)\b/i, + /\ball\s+(?:items?|todos?|steps?)\s+(?:are\s+)?(?:complete|completed|done|finished|marked)\b/i, + /\b(?:everything|all\s+work)\s+(?:is\s+)?(?:complete|completed|done|finished)\b/i, + /\bsuccessfully\s+completed?\s+all\b/i, + /\bnothing\s+(?:left|more|remaining)\s+to\s+(?:do|implement|fix)\b/i, +] + +export function detectSemanticCompletion(text: string): boolean { + return SEMANTIC_COMPLETION_PATTERNS.some((pattern) => pattern.test(text)) +} + export function detectCompletionInTranscript( transcriptPath: string | undefined, promise: string, @@ -37,6 +49,11 @@ export function detectCompletionInTranscript( if (entry.type === "user") continue if (startedAt && entry.timestamp && entry.timestamp < startedAt) continue if (pattern.test(line)) return true + // Fallback: check for semantic completion + if (detectSemanticCompletion(line)) { + log("[ralph-loop] WARNING: Semantic completion detected in transcript (agent used natural language instead of DONE)") + return true + } } catch { continue } @@ -100,6 +117,14 @@ export async function detectCompletionInSessionMessages( if (pattern.test(responseText)) { return true } + + // Fallback: check for semantic completion + if (detectSemanticCompletion(responseText)) { + log("[ralph-loop] WARNING: Semantic completion detected (agent used natural language instead of DONE)", { + sessionID: options.sessionID, + }) + return true + } } return false From 774d0bd84dc7c183a5b82942d8f0926fc07adcfd Mon Sep 17 00:00:00 2001 From: MoerAI Date: Wed, 25 Mar 2026 16:49:53 +0900 Subject: [PATCH 2/3] fix(ralph-loop): restrict semantic completion to DONE promise and assistant entries - Gate semantic detection on promise === 'DONE' in both transcript and session message paths to prevent false positives on VERIFIED promises - Restrict transcript semantic fallback to assistant/text entries only, skipping tool_use/tool_result to avoid matching file content - Add regression test for VERIFIED promise not triggering semantic detection --- .../completion-promise-detector.test.ts | 22 +++++++++++++++++++ .../ralph-loop/completion-promise-detector.ts | 9 ++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/hooks/ralph-loop/completion-promise-detector.test.ts b/src/hooks/ralph-loop/completion-promise-detector.test.ts index 704517732..bbb2206fb 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.test.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.test.ts @@ -273,6 +273,28 @@ describe("detectCompletionInSessionMessages", () => { // #then expect(detected).toBe(true) }) + + test("#when promise is VERIFIED #then semantic completion should NOT trigger", async () => { + // #given + const messages: SessionMessage[] = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "The task is complete. All work has been finished." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-123", + promise: "VERIFIED", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) }) }) diff --git a/src/hooks/ralph-loop/completion-promise-detector.ts b/src/hooks/ralph-loop/completion-promise-detector.ts index 39322e536..cd02ce7f7 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.ts @@ -49,8 +49,9 @@ export function detectCompletionInTranscript( if (entry.type === "user") continue if (startedAt && entry.timestamp && entry.timestamp < startedAt) continue if (pattern.test(line)) return true - // Fallback: check for semantic completion - if (detectSemanticCompletion(line)) { + // Fallback: semantic completion only for DONE promise and assistant entries + const isAssistantEntry = entry.type === "assistant" || entry.type === "text" + if (promise === "DONE" && isAssistantEntry && detectSemanticCompletion(line)) { log("[ralph-loop] WARNING: Semantic completion detected in transcript (agent used natural language instead of DONE)") return true } @@ -118,8 +119,8 @@ export async function detectCompletionInSessionMessages( return true } - // Fallback: check for semantic completion - if (detectSemanticCompletion(responseText)) { + // Fallback: semantic completion only for DONE promise + if (options.promise === "DONE" && detectSemanticCompletion(responseText)) { log("[ralph-loop] WARNING: Semantic completion detected (agent used natural language instead of DONE)", { sessionID: options.sessionID, }) From 95801a485022b360ad658154dc1a418744b3dab7 Mon Sep 17 00:00:00 2001 From: MoerAI Date: Thu, 26 Mar 2026 18:16:07 +0900 Subject: [PATCH 3/3] fix(ralph-loop): extract text from parsed entry instead of testing raw JSONL --- .../ralph-loop/completion-promise-detector.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/hooks/ralph-loop/completion-promise-detector.ts b/src/hooks/ralph-loop/completion-promise-detector.ts index cd02ce7f7..57b85ca9c 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.ts @@ -9,6 +9,20 @@ interface OpenCodeSessionMessage { parts?: Array<{ type: string; text?: string }> } +interface TranscriptEntry { + type?: string + timestamp?: string + content?: string + tool_output?: { output?: string } | string +} + +function extractTranscriptEntryText(entry: TranscriptEntry): string { + if (typeof entry.content === "string") return entry.content + if (typeof entry.tool_output === "string") return entry.tool_output + if (entry.tool_output && typeof entry.tool_output === "object" && typeof entry.tool_output.output === "string") return entry.tool_output.output + return "" +} + function escapeRegex(str: string): string { return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") } @@ -45,13 +59,15 @@ export function detectCompletionInTranscript( for (const line of lines) { try { - const entry = JSON.parse(line) as { type?: string; timestamp?: string } + const entry = JSON.parse(line) as TranscriptEntry if (entry.type === "user") continue if (startedAt && entry.timestamp && entry.timestamp < startedAt) continue - if (pattern.test(line)) return true + const entryText = extractTranscriptEntryText(entry) + if (!entryText) continue + if (pattern.test(entryText)) return true // Fallback: semantic completion only for DONE promise and assistant entries const isAssistantEntry = entry.type === "assistant" || entry.type === "text" - if (promise === "DONE" && isAssistantEntry && detectSemanticCompletion(line)) { + if (promise === "DONE" && isAssistantEntry && detectSemanticCompletion(entryText)) { log("[ralph-loop] WARNING: Semantic completion detected in transcript (agent used natural language instead of DONE)") return true }