From d50c38f037e632aa107012aa3638bc41017495b1 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Tue, 17 Mar 2026 16:41:02 +0900 Subject: [PATCH] refactor(tests): rename benchmarks/ to tests/hashline/, remove FriendliAI dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move benchmarks/ → tests/hashline/ - Replace @friendliai/ai-provider with @ai-sdk/openai-compatible - Remove all 'benchmark' naming (package name, scripts, env vars, session IDs) - Fix import paths for new directory depth (../src → ../../src) - Fix pre-existing syntax error in headless.ts (unclosed case block) - Inject HASHLINE_EDIT_DESCRIPTION into test system prompt - Scripts renamed: bench:* → test:* --- tests/hashline/bun.lock | 12 ++++++----- tests/hashline/headless.ts | 32 +++++++++++++++++++----------- tests/hashline/package.json | 14 ++++++------- tests/hashline/test-multi-model.ts | 5 +---- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/tests/hashline/bun.lock b/tests/hashline/bun.lock index 3772bbfbc..bc7404903 100644 --- a/tests/hashline/bun.lock +++ b/tests/hashline/bun.lock @@ -5,7 +5,7 @@ "": { "name": "hashline-edit-benchmark", "dependencies": { - "@friendliai/ai-provider": "^1.0.9", + "@ai-sdk/openai-compatible": "^2.0.35", "ai": "^6.0.94", "zod": "^4.1.0", }, @@ -14,13 +14,11 @@ "packages": { "@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.55", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15", "@vercel/oidc": "3.1.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7xMeTJnCjwRwXKVCiv4Ly4qzWvDuW3+W1WIV0X1EFu6W83d4mEhV9bFArto10MeTw40ewuDjrbrZd21mXKohkw=="], - "@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.30", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-iTjumHf1/u4NhjXYFn/aONM2GId3/o7J1Lp5ql8FCbgIMyRwrmanR5xy1S3aaVkfTscuDvLTzWiy1mAbGzK3nQ=="], + "@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.35", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.19" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-g3wA57IAQFb+3j4YuFndgkUdXyRETZVvbfAWM+UX7bZSxA3xjes0v3XKgIdKdekPtDGsh4ZX2byHD0gJIMPfiA=="], "@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="], - "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="], - - "@friendliai/ai-provider": ["@friendliai/ai-provider@1.1.4", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.30", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.12" } }, "sha512-9TU4B1QFqPhbkONjI5afCF7Ox4jOqtGg1xw8mA9QHZdtlEbZxU+mBNvMPlI5pU5kPoN6s7wkXmFmxpID+own1A=="], + "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.19", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-3eG55CrSWCu2SXlqq2QCsFjo3+E7+Gmg7i/oRVoSZzIodTuDSfLb3MRje67xE9RFea73Zao7Lm4mADIfUETKGg=="], "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], @@ -35,5 +33,9 @@ "json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="], "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + + "@ai-sdk/gateway/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="], + + "ai/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="], } } diff --git a/tests/hashline/headless.ts b/tests/hashline/headless.ts index ae18853af..50fa1f3e9 100644 --- a/tests/hashline/headless.ts +++ b/tests/hashline/headless.ts @@ -3,16 +3,17 @@ import { readFile, writeFile, mkdir } from "node:fs/promises" import { join, dirname } from "node:path" import { stepCountIs, streamText, type CoreMessage } from "ai" import { tool } from "ai" -import { createFriendli } from "@friendliai/ai-provider" +import { createOpenAICompatible } from "@ai-sdk/openai-compatible" import { z } from "zod" -import { formatHashLines } from "../src/tools/hashline-edit/hash-computation" -import { normalizeHashlineEdits } from "../src/tools/hashline-edit/normalize-edits" -import { applyHashlineEditsWithReport } from "../src/tools/hashline-edit/edit-operations" -import { canonicalizeFileText, restoreFileText } from "../src/tools/hashline-edit/file-text-canonicalization" +import { formatHashLines } from "../../src/tools/hashline-edit/hash-computation" +import { normalizeHashlineEdits } from "../../src/tools/hashline-edit/normalize-edits" +import { applyHashlineEditsWithReport } from "../../src/tools/hashline-edit/edit-operations" +import { canonicalizeFileText, restoreFileText } from "../../src/tools/hashline-edit/file-text-canonicalization" +import { HASHLINE_EDIT_DESCRIPTION } from "../../src/tools/hashline-edit/tool-description" -const DEFAULT_MODEL = "MiniMaxAI/MiniMax-M2.5" +const DEFAULT_MODEL = "minimax-m2.5-free" const MAX_STEPS = 50 -const sessionId = `bench-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` +const sessionId = `hashline-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` const emit = (event: Record) => console.log(JSON.stringify({ sessionId, timestamp: new Date().toISOString(), ...event })) @@ -33,7 +34,7 @@ function parseArgs(): { prompt: string; modelId: string } { // --no-translate, --think consumed silently } if (!prompt) { - console.error("Usage: bun run benchmarks/headless.ts -p [-m ]") + console.error("Usage: bun run tests/hashline/headless.ts -p [-m ]") process.exit(1) } return { prompt, modelId } @@ -57,7 +58,7 @@ const readFileTool = tool({ }) const editFileTool = tool({ - description: "Edit a file using hashline anchors (LINE#ID format)", + description: HASHLINE_EDIT_DESCRIPTION, inputSchema: z.object({ path: z.string(), edits: z.array( @@ -116,8 +117,12 @@ const editFileTool = tool({ async function run() { const { prompt, modelId } = parseArgs() - const friendli = createFriendli({ apiKey: process.env.FRIENDLI_TOKEN! }) - const model = friendli(modelId) + const provider = createOpenAICompatible({ + name: "hashline-test", + baseURL: process.env.HASHLINE_TEST_BASE_URL ?? "https://quotio.mengmota.com/v1", + apiKey: process.env.HASHLINE_TEST_API_KEY ?? "quotio-local-60A613FE-DB74-40FF-923E-A14151951E5D", + }) + const model = provider.chatModel(modelId) const tools = { read_file: readFileTool, edit_file: editFileTool } emit({ type: "user", content: prompt }) @@ -125,7 +130,8 @@ async function run() { const messages: CoreMessage[] = [{ role: "user", content: prompt }] const system = "You are a code editing assistant. Use read_file to read files and edit_file to edit them. " + - "Always read a file before editing it to get fresh LINE#ID anchors." + "Always read a file before editing it to get fresh LINE#ID anchors.\n\n" + + "edit_file tool description:\n" + HASHLINE_EDIT_DESCRIPTION for (let step = 0; step < MAX_STEPS; step++) { const stream = streamText({ @@ -161,6 +167,7 @@ async function run() { ...(isError ? { error: output } : {}), }) break + } } } @@ -191,3 +198,4 @@ run() const elapsed = ((Date.now() - startTime) / 1000).toFixed(2) console.error(`[headless] Completed in ${elapsed}s`) }) + diff --git a/tests/hashline/package.json b/tests/hashline/package.json index 52dd75d58..f8b462e8f 100644 --- a/tests/hashline/package.json +++ b/tests/hashline/package.json @@ -1,17 +1,17 @@ { - "name": "hashline-edit-benchmark", + "name": "hashline-edit-tests", "version": "0.1.0", "private": true, "type": "module", - "description": "Hashline edit tool benchmark using Vercel AI SDK with FriendliAI provider", + "description": "Hashline edit tool integration tests using Vercel AI SDK", "scripts": { - "bench:basic": "bun run test-edit-ops.ts", - "bench:edge": "bun run test-edge-cases.ts", - "bench:multi": "bun run test-multi-model.ts", - "bench:all": "bun run bench:basic && bun run bench:edge" + "test:basic": "bun run test-edit-ops.ts", + "test:edge": "bun run test-edge-cases.ts", + "test:multi": "bun run test-multi-model.ts", + "test:all": "bun run test:basic && bun run test:edge" }, "dependencies": { - "@friendliai/ai-provider": "^1.0.9", + "@ai-sdk/openai-compatible": "^2.0.35", "ai": "^6.0.94", "zod": "^4.1.0" } diff --git a/tests/hashline/test-multi-model.ts b/tests/hashline/test-multi-model.ts index 29ee4bb93..3a999fbe3 100644 --- a/tests/hashline/test-multi-model.ts +++ b/tests/hashline/test-multi-model.ts @@ -14,10 +14,7 @@ import { resolve } from "node:path"; // ── Models ──────────────────────────────────────────────────── const MODELS = [ - { id: "MiniMaxAI/MiniMax-M2.5", short: "M2.5" }, - // { id: "MiniMaxAI/MiniMax-M2.1", short: "M2.1" }, // masked: slow + timeout-prone - // { id: "zai-org/GLM-5", short: "GLM-5" }, // masked: API 503 - { id: "zai-org/GLM-4.7", short: "GLM-4.7" }, + { id: "minimax-m2.5-free", short: "M2.5-Free" }, ]; // ── CLI args ──────────────────────────────────────────────────