Merge pull request #1490 from itsmylife44/fix/custom-skills-delegation-emphasis

fix(agents): emphasize user-installed custom skills in delegation prompts
2026-02-05 01:47:04 +09:00
parent 708d15ebcc a298a2f063
commit 8049ceb947
3 changed files with 290 additions and 20 deletions
--- a/src/agents/atlas/utils.ts
+++ b/src/agents/atlas/utils.ts
@@ -56,21 +56,66 @@ export function buildSkillsSection(skills: AvailableSkill[]): string {
    return ""
  }

-  const skillRows = skills.map((s) => {
+  const builtinSkills = skills.filter((s) => s.location === "plugin")
+  const customSkills = skills.filter((s) => s.location !== "plugin")
+
+  const builtinRows = builtinSkills.map((s) => {
    const shortDesc = s.description.split(".")[0] || s.description
    return `| \`${s.name}\` | ${shortDesc} |`
  })

+  const customRows = customSkills.map((s) => {
+    const shortDesc = s.description.split(".")[0] || s.description
+    const source = s.location === "project" ? "project" : "user"
+    return `| \`${s.name}\` | ${shortDesc} | ${source} |`
+  })
+
+  const customSkillNames = customSkills.map((s) => `"${s.name}"`).join(", ")
+
+  let skillsTable: string
+
+  if (customSkills.length > 0 && builtinSkills.length > 0) {
+    skillsTable = `**Built-in Skills:**
+
+| Skill | When to Use |
+|-------|-------------|
+${builtinRows.join("\n")}
+
+**User-Installed Skills (HIGH PRIORITY):**
+
+The user installed these for their workflow. They MUST be evaluated for EVERY delegation.
+
+| Skill | When to Use | Source |
+|-------|-------------|--------|
+${customRows.join("\n")}
+
+> **CRITICAL**: The user installed ${customSkillNames} for a reason — USE THEM when the task overlaps with their domain.
+> When in doubt, INCLUDE a user-installed skill rather than omit it.`
+  } else if (customSkills.length > 0) {
+    skillsTable = `**User-Installed Skills (HIGH PRIORITY):**
+
+The user installed these for their workflow. They MUST be evaluated for EVERY delegation.
+
+| Skill | When to Use | Source |
+|-------|-------------|--------|
+${customRows.join("\n")}
+
+> **CRITICAL**: The user installed ${customSkillNames} for a reason — USE THEM when the task overlaps with their domain.
+> When in doubt, INCLUDE a user-installed skill rather than omit it.`
+  } else {
+    skillsTable = `| Skill | When to Use |
+|-------|-------------|
+${builtinRows.join("\n")}`
+  }
+
  return `
 #### 3.2.2: Skill Selection (PREPEND TO PROMPT)

 **Skills are specialized instructions that guide subagent behavior. Consider them alongside category selection.**

-| Skill | When to Use |
-|-------|-------------|
-${skillRows.join("\n")}
+${skillsTable}

-**MANDATORY: Evaluate ALL skills for relevance to your task.**
+**MANDATORY: Evaluate ALL skills (built-in AND user-installed) for relevance to your task.**

 Read each skill's description and ask: "Does this skill's domain overlap with my task?"
 - If YES: INCLUDE in load_skills=[...]
--- a/src/agents/dynamic-agent-prompt-builder.test.ts
+++ b/src/agents/dynamic-agent-prompt-builder.test.ts
@@ -0,0 +1,161 @@
+/// <reference types="bun-types" />
+
+import { describe, it, expect } from "bun:test"
+import {
+  buildCategorySkillsDelegationGuide,
+  buildUltraworkSection,
+  type AvailableSkill,
+  type AvailableCategory,
+  type AvailableAgent,
+} from "./dynamic-agent-prompt-builder"
+
+describe("buildCategorySkillsDelegationGuide", () => {
+  const categories: AvailableCategory[] = [
+    { name: "visual-engineering", description: "Frontend, UI/UX" },
+    { name: "quick", description: "Trivial tasks" },
+  ]
+
+  const builtinSkills: AvailableSkill[] = [
+    { name: "playwright", description: "Browser automation via Playwright", location: "plugin" },
+    { name: "frontend-ui-ux", description: "Designer-turned-developer", location: "plugin" },
+  ]
+
+  const customUserSkills: AvailableSkill[] = [
+    { name: "react-19", description: "React 19 patterns and best practices", location: "user" },
+    { name: "tailwind-4", description: "Tailwind CSS v4 utilities", location: "user" },
+  ]
+
+  const customProjectSkills: AvailableSkill[] = [
+    { name: "our-design-system", description: "Internal design system components", location: "project" },
+  ]
+
+  it("should separate builtin and custom skills into distinct sections", () => {
+    //#given: mix of builtin and custom skills
+    const allSkills = [...builtinSkills, ...customUserSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: should have separate sections
+    expect(result).toContain("Built-in Skills")
+    expect(result).toContain("User-Installed Skills")
+    expect(result).toContain("HIGH PRIORITY")
+  })
+
+  it("should include custom skill names in CRITICAL warning", () => {
+    //#given: custom skills installed
+    const allSkills = [...builtinSkills, ...customUserSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: should mention custom skills by name in the warning
+    expect(result).toContain('"react-19"')
+    expect(result).toContain('"tailwind-4"')
+    expect(result).toContain("CRITICAL")
+  })
+
+  it("should show source column for custom skills (user vs project)", () => {
+    //#given: both user and project custom skills
+    const allSkills = [...builtinSkills, ...customUserSkills, ...customProjectSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: should show source for each custom skill
+    expect(result).toContain("| user |")
+    expect(result).toContain("| project |")
+  })
+
+  it("should not show custom skill section when only builtin skills exist", () => {
+    //#given: only builtin skills
+    const allSkills = [...builtinSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: should not contain custom skill emphasis
+    expect(result).not.toContain("User-Installed Skills")
+    expect(result).not.toContain("HIGH PRIORITY")
+    expect(result).toContain("Available Skills")
+  })
+
+  it("should handle only custom skills (no builtins)", () => {
+    //#given: only custom skills, no builtins
+    const allSkills = [...customUserSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: should show custom skills with emphasis, no builtin section
+    expect(result).toContain("User-Installed Skills")
+    expect(result).toContain("HIGH PRIORITY")
+    expect(result).not.toContain("Built-in Skills")
+  })
+
+  it("should include priority note for custom skills in evaluation step", () => {
+    //#given: custom skills present
+    const allSkills = [...builtinSkills, ...customUserSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: evaluation section should mention user-installed priority
+    expect(result).toContain("User-installed skills get PRIORITY")
+    expect(result).toContain("INCLUDE it rather than omit it")
+  })
+
+  it("should NOT include priority note when no custom skills", () => {
+    //#given: only builtin skills
+    const allSkills = [...builtinSkills]
+
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide(categories, allSkills)
+
+    //#then: no priority note for custom skills
+    expect(result).not.toContain("User-installed skills get PRIORITY")
+  })
+
+  it("should return empty string when no categories and no skills", () => {
+    //#given: no categories and no skills
+    //#when: building the delegation guide
+    const result = buildCategorySkillsDelegationGuide([], [])
+
+    //#then: should return empty string
+    expect(result).toBe("")
+  })
+})
+
+describe("buildUltraworkSection", () => {
+  const agents: AvailableAgent[] = []
+
+  it("should separate builtin and custom skills", () => {
+    //#given: mix of builtin and custom skills
+    const skills: AvailableSkill[] = [
+      { name: "playwright", description: "Browser automation", location: "plugin" },
+      { name: "react-19", description: "React 19 patterns", location: "user" },
+    ]
+
+    //#when: building ultrawork section
+    const result = buildUltraworkSection(agents, [], skills)
+
+    //#then: should have separate sections
+    expect(result).toContain("Built-in Skills")
+    expect(result).toContain("User-Installed Skills")
+    expect(result).toContain("HIGH PRIORITY")
+  })
+
+  it("should not separate when only builtin skills", () => {
+    //#given: only builtin skills
+    const skills: AvailableSkill[] = [
+      { name: "playwright", description: "Browser automation", location: "plugin" },
+    ]
+
+    //#when: building ultrawork section
+    const result = buildUltraworkSection(agents, [], skills)
+
+    //#then: should have single section
+    expect(result).toContain("Built-in Skills")
+    expect(result).not.toContain("User-Installed Skills")
+  })
+})
--- a/src/agents/dynamic-agent-prompt-builder.ts
+++ b/src/agents/dynamic-agent-prompt-builder.ts
@@ -174,11 +174,64 @@ export function buildCategorySkillsDelegationGuide(categories: AvailableCategory
    return `| \`${c.name}\` | ${desc} |`
  })

-  const skillRows = skills.map((s) => {
+  const builtinSkills = skills.filter((s) => s.location === "plugin")
+  const customSkills = skills.filter((s) => s.location !== "plugin")
+
+  const builtinRows = builtinSkills.map((s) => {
    const desc = s.description.split(".")[0] || s.description
    return `| \`${s.name}\` | ${desc} |`
  })

+  const customRows = customSkills.map((s) => {
+    const desc = s.description.split(".")[0] || s.description
+    const source = s.location === "project" ? "project" : "user"
+    return `| \`${s.name}\` | ${desc} | ${source} |`
+  })
+
+  const customSkillNames = customSkills.map((s) => `"${s.name}"`).join(", ")
+
+  let skillsSection: string
+
+  if (customSkills.length > 0 && builtinSkills.length > 0) {
+    skillsSection = `#### Built-in Skills
+
+| Skill | Expertise Domain |
+|-------|------------------|
+${builtinRows.join("\n")}
+
+#### User-Installed Skills (HIGH PRIORITY)
+
+**The user has installed these custom skills. They MUST be evaluated for EVERY delegation.**
+Subagents are STATELESS — they lose all custom knowledge unless you pass these skills via \`load_skills\`.
+
+| Skill | Expertise Domain | Source |
+|-------|------------------|--------|
+${customRows.join("\n")}
+
+> **CRITICAL**: Ignoring user-installed skills when they match the task domain is a failure.
+> The user installed ${customSkillNames} for a reason — USE THEM when the task overlaps with their domain.`
+  } else if (customSkills.length > 0) {
+    skillsSection = `#### User-Installed Skills (HIGH PRIORITY)
+
+**The user has installed these custom skills. They MUST be evaluated for EVERY delegation.**
+Subagents are STATELESS — they lose all custom knowledge unless you pass these skills via \`load_skills\`.
+
+| Skill | Expertise Domain | Source |
+|-------|------------------|--------|
+${customRows.join("\n")}
+
+> **CRITICAL**: Ignoring user-installed skills when they match the task domain is a failure.
+> The user installed ${customSkillNames} for a reason — USE THEM when the task overlaps with their domain.`
+  } else {
+    skillsSection = `#### Available Skills (Domain Expertise Injection)
+
+Skills inject specialized instructions into the subagent. Read the description to understand when each skill applies.
+
+| Skill | Expertise Domain |
+|-------|------------------|
+${builtinRows.join("\n")}`
+  }
+
  return `### Category + Skills Delegation System

 **delegate_task() combines categories and skills for optimal task execution.**
@@ -191,13 +244,7 @@ Each category is configured with a model optimized for that domain. Read the des
 |----------|-------------------|
 ${categoryRows.join("\n")}

-#### Available Skills (Domain Expertise Injection)
-
-Skills inject specialized instructions into the subagent. Read the description to understand when each skill applies.
-
-| Skill | Expertise Domain |
-|-------|------------------|
-${skillRows.join("\n")}
+${skillsSection}

 ---

@@ -208,12 +255,15 @@ ${skillRows.join("\n")}
 - Match task requirements to category domain
 - Select the category whose domain BEST fits the task

-**STEP 2: Evaluate ALL Skills**
+**STEP 2: Evaluate ALL Skills (Built-in AND User-Installed)**
 For EVERY skill listed above, ask yourself:
 > "Does this skill's expertise domain overlap with my task?"

 - If YES → INCLUDE in \`load_skills=[...]\`
 - If NO → You MUST justify why (see below)
+${customSkills.length > 0 ? `
+> **User-installed skills get PRIORITY.** The user explicitly installed them for their workflow.
+> When in doubt about a user-installed skill, INCLUDE it rather than omit it.` : ""}

 **STEP 3: Justify Omissions**

@@ -240,7 +290,7 @@ SKILL EVALUATION for "[skill-name]":
 \`\`\`typescript
 delegate_task(
  category="[selected-category]",
-  load_skills=["skill-1", "skill-2"],  // Include ALL relevant skills
+  load_skills=["skill-1", "skill-2"],  // Include ALL relevant skills — ESPECIALLY user-installed ones
  prompt="..."
 )
 \`\`\`
@@ -328,12 +378,26 @@ export function buildUltraworkSection(
  }

  if (skills.length > 0) {
-    lines.push("**Skills** (combine with categories - EVALUATE ALL for relevance):")
-    for (const skill of skills) {
-      const shortDesc = skill.description.split(".")[0] || skill.description
-      lines.push(`- \`${skill.name}\`: ${shortDesc}`)
+    const builtinSkills = skills.filter((s) => s.location === "plugin")
+    const customSkills = skills.filter((s) => s.location !== "plugin")
+
+    if (builtinSkills.length > 0) {
+      lines.push("**Built-in Skills** (combine with categories):")
+      for (const skill of builtinSkills) {
+        const shortDesc = skill.description.split(".")[0] || skill.description
+        lines.push(`- \`${skill.name}\`: ${shortDesc}`)
+      }
+      lines.push("")
+    }
+
+    if (customSkills.length > 0) {
+      lines.push("**User-Installed Skills** (HIGH PRIORITY - user installed these for their workflow):")
+      for (const skill of customSkills) {
+        const shortDesc = skill.description.split(".")[0] || skill.description
+        lines.push(`- \`${skill.name}\`: ${shortDesc}`)
+      }
+      lines.push("")
    }
-    lines.push("")
  }

  if (agents.length > 0) {