refactor(tests): rename benchmarks/ to tests/hashline/, remove FriendliAI dependency

- Move benchmarks/ → tests/hashline/ - Replace @friendliai/ai-provider with @ai-sdk/openai-compatible - Remove all 'benchmark' naming (package name, scripts, env vars, session IDs) - Fix import paths for new directory depth (../src → ../../src) - Fix pre-existing syntax error in headless.ts (unclosed case block) - Inject HASHLINE_EDIT_DESCRIPTION into test system prompt - Scripts renamed: bench:* → test:*
2026-03-17 16:41:02 +09:00
parent f2d5f4ca92
commit d50c38f037
4 changed files with 35 additions and 28 deletions
--- a/tests/hashline/bun.lock
+++ b/tests/hashline/bun.lock
@@ -5,7 +5,7 @@
    "": {
      "name": "hashline-edit-benchmark",
      "dependencies": {
-        "@friendliai/ai-provider": "^1.0.9",
+        "@ai-sdk/openai-compatible": "^2.0.35",
        "ai": "^6.0.94",
        "zod": "^4.1.0",
      },
@@ -14,13 +14,11 @@
  "packages": {
    "@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.55", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15", "@vercel/oidc": "3.1.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7xMeTJnCjwRwXKVCiv4Ly4qzWvDuW3+W1WIV0X1EFu6W83d4mEhV9bFArto10MeTw40ewuDjrbrZd21mXKohkw=="],

-    "@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.30", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-iTjumHf1/u4NhjXYFn/aONM2GId3/o7J1Lp5ql8FCbgIMyRwrmanR5xy1S3aaVkfTscuDvLTzWiy1mAbGzK3nQ=="],
+    "@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.35", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.19" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-g3wA57IAQFb+3j4YuFndgkUdXyRETZVvbfAWM+UX7bZSxA3xjes0v3XKgIdKdekPtDGsh4ZX2byHD0gJIMPfiA=="],

    "@ai-sdk/provider": ["@ai-sdk/provider@3.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ=="],

-    "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
-
-    "@friendliai/ai-provider": ["@friendliai/ai-provider@1.1.4", "", { "dependencies": { "@ai-sdk/openai-compatible": "2.0.30", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.15" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.12" } }, "sha512-9TU4B1QFqPhbkONjI5afCF7Ox4jOqtGg1xw8mA9QHZdtlEbZxU+mBNvMPlI5pU5kPoN6s7wkXmFmxpID+own1A=="],
+    "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.19", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-3eG55CrSWCu2SXlqq2QCsFjo3+E7+Gmg7i/oRVoSZzIodTuDSfLb3MRje67xE9RFea73Zao7Lm4mADIfUETKGg=="],

    "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],

@@ -35,5 +33,9 @@
    "json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],

    "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
+
+    "@ai-sdk/gateway/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
+
+    "ai/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.15", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w=="],
  }
 }
--- a/tests/hashline/headless.ts
+++ b/tests/hashline/headless.ts
@@ -3,16 +3,17 @@ import { readFile, writeFile, mkdir } from "node:fs/promises"
 import { join, dirname } from "node:path"
 import { stepCountIs, streamText, type CoreMessage } from "ai"
 import { tool } from "ai"
-import { createFriendli } from "@friendliai/ai-provider"
+import { createOpenAICompatible } from "@ai-sdk/openai-compatible"
 import { z } from "zod"
-import { formatHashLines } from "../src/tools/hashline-edit/hash-computation"
-import { normalizeHashlineEdits } from "../src/tools/hashline-edit/normalize-edits"
-import { applyHashlineEditsWithReport } from "../src/tools/hashline-edit/edit-operations"
-import { canonicalizeFileText, restoreFileText } from "../src/tools/hashline-edit/file-text-canonicalization"
+import { formatHashLines } from "../../src/tools/hashline-edit/hash-computation"
+import { normalizeHashlineEdits } from "../../src/tools/hashline-edit/normalize-edits"
+import { applyHashlineEditsWithReport } from "../../src/tools/hashline-edit/edit-operations"
+import { canonicalizeFileText, restoreFileText } from "../../src/tools/hashline-edit/file-text-canonicalization"
+import { HASHLINE_EDIT_DESCRIPTION } from "../../src/tools/hashline-edit/tool-description"

-const DEFAULT_MODEL = "MiniMaxAI/MiniMax-M2.5"
+const DEFAULT_MODEL = "minimax-m2.5-free"
 const MAX_STEPS = 50
-const sessionId = `bench-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
+const sessionId = `hashline-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`

 const emit = (event: Record<string, unknown>) =>
  console.log(JSON.stringify({ sessionId, timestamp: new Date().toISOString(), ...event }))
@@ -33,7 +34,7 @@ function parseArgs(): { prompt: string; modelId: string } {
    // --no-translate, --think consumed silently
  }
  if (!prompt) {
-    console.error("Usage: bun run benchmarks/headless.ts -p <prompt> [-m <model>]")
+    console.error("Usage: bun run tests/hashline/headless.ts -p <prompt> [-m <model>]")
    process.exit(1)
  }
  return { prompt, modelId }
@@ -57,7 +58,7 @@ const readFileTool = tool({
 })

 const editFileTool = tool({
-  description: "Edit a file using hashline anchors (LINE#ID format)",
+  description: HASHLINE_EDIT_DESCRIPTION,
  inputSchema: z.object({
    path: z.string(),
    edits: z.array(
@@ -116,8 +117,12 @@ const editFileTool = tool({
 async function run() {
  const { prompt, modelId } = parseArgs()

-  const friendli = createFriendli({ apiKey: process.env.FRIENDLI_TOKEN! })
-  const model = friendli(modelId)
+  const provider = createOpenAICompatible({
+    name: "hashline-test",
+    baseURL: process.env.HASHLINE_TEST_BASE_URL ?? "https://quotio.mengmota.com/v1",
+    apiKey: process.env.HASHLINE_TEST_API_KEY ?? "quotio-local-60A613FE-DB74-40FF-923E-A14151951E5D",
+  })
+  const model = provider.chatModel(modelId)
  const tools = { read_file: readFileTool, edit_file: editFileTool }

  emit({ type: "user", content: prompt })
@@ -125,7 +130,8 @@ async function run() {
  const messages: CoreMessage[] = [{ role: "user", content: prompt }]
  const system =
    "You are a code editing assistant. Use read_file to read files and edit_file to edit them. " +
-    "Always read a file before editing it to get fresh LINE#ID anchors."
+    "Always read a file before editing it to get fresh LINE#ID anchors.\n\n" +
+    "edit_file tool description:\n" + HASHLINE_EDIT_DESCRIPTION

  for (let step = 0; step < MAX_STEPS; step++) {
    const stream = streamText({
@@ -163,6 +169,7 @@ async function run() {
          break
        }
      }
+    }

    const response = await stream.response
    messages.push(...response.messages)
@@ -191,3 +198,4 @@ run()
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(2)
    console.error(`[headless] Completed in ${elapsed}s`)
  })
+
--- a/tests/hashline/package.json
+++ b/tests/hashline/package.json
@@ -1,17 +1,17 @@
 {
-  "name": "hashline-edit-benchmark",
+  "name": "hashline-edit-tests",
  "version": "0.1.0",
  "private": true,
  "type": "module",
-  "description": "Hashline edit tool benchmark using Vercel AI SDK with FriendliAI provider",
+  "description": "Hashline edit tool integration tests using Vercel AI SDK",
  "scripts": {
-    "bench:basic": "bun run test-edit-ops.ts",
-    "bench:edge": "bun run test-edge-cases.ts",
-    "bench:multi": "bun run test-multi-model.ts",
-    "bench:all": "bun run bench:basic && bun run bench:edge"
+    "test:basic": "bun run test-edit-ops.ts",
+    "test:edge": "bun run test-edge-cases.ts",
+    "test:multi": "bun run test-multi-model.ts",
+    "test:all": "bun run test:basic && bun run test:edge"
  },
  "dependencies": {
-    "@friendliai/ai-provider": "^1.0.9",
+    "@ai-sdk/openai-compatible": "^2.0.35",
    "ai": "^6.0.94",
    "zod": "^4.1.0"
  }
--- a/tests/hashline/test-multi-model.ts
+++ b/tests/hashline/test-multi-model.ts
@@ -14,10 +14,7 @@ import { resolve } from "node:path";

 // ── Models ────────────────────────────────────────────────────
 const MODELS = [
-  { id: "MiniMaxAI/MiniMax-M2.5", short: "M2.5" },
-  // { id: "MiniMaxAI/MiniMax-M2.1", short: "M2.1" },  // masked: slow + timeout-prone
-  // { id: "zai-org/GLM-5", short: "GLM-5" },            // masked: API 503
-  { id: "zai-org/GLM-4.7", short: "GLM-4.7" },
+  { id: "minimax-m2.5-free", short: "M2.5-Free" },
 ];

 // ── CLI args ──────────────────────────────────────────────────