fix(agents): emphasize user-installed skills in delegation prompts
Custom skills from .config/opencode/skills/ were visible in agent prompts but the model consistently ignored them when delegating via delegate_task(). The flat skill table made no distinction between built-in and user-installed skills, causing the model to default to built-in ones only. - Separate skills into 'Built-in Skills' and 'User-Installed Skills (HIGH PRIORITY)' sections in buildCategorySkillsDelegationGuide() - Add CRITICAL warning naming each custom skill explicitly - Add priority note: 'When in doubt, INCLUDE rather than omit' - Show source column (user/project) for custom skills - Apply same separation in buildUltraworkSection() - Add 10 unit tests covering all skill combination scenarios Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
itsmylife44
161
src/agents/dynamic-agent-prompt-builder.test.ts
Normal file
161
src/agents/dynamic-agent-prompt-builder.test.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
/// <reference types="bun-types" />
|
||||
|
||||
import { describe, it, expect } from "bun:test"
|
||||
import {
|
||||
buildCategorySkillsDelegationGuide,
|
||||
buildUltraworkSection,
|
||||
type AvailableSkill,
|
||||
type AvailableCategory,
|
||||
type AvailableAgent,
|
||||
} from "./dynamic-agent-prompt-builder"
|
||||
|
||||
describe("buildCategorySkillsDelegationGuide", () => {
|
||||
const categories: AvailableCategory[] = [
|
||||
{ name: "visual-engineering", description: "Frontend, UI/UX" },
|
||||
{ name: "quick", description: "Trivial tasks" },
|
||||
]
|
||||
|
||||
const builtinSkills: AvailableSkill[] = [
|
||||
{ name: "playwright", description: "Browser automation via Playwright", location: "plugin" },
|
||||
{ name: "frontend-ui-ux", description: "Designer-turned-developer", location: "plugin" },
|
||||
]
|
||||
|
||||
const customUserSkills: AvailableSkill[] = [
|
||||
{ name: "react-19", description: "React 19 patterns and best practices", location: "user" },
|
||||
{ name: "tailwind-4", description: "Tailwind CSS v4 utilities", location: "user" },
|
||||
]
|
||||
|
||||
const customProjectSkills: AvailableSkill[] = [
|
||||
{ name: "our-design-system", description: "Internal design system components", location: "project" },
|
||||
]
|
||||
|
||||
it("should separate builtin and custom skills into distinct sections", () => {
|
||||
//#given: mix of builtin and custom skills
|
||||
const allSkills = [...builtinSkills, ...customUserSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: should have separate sections
|
||||
expect(result).toContain("Built-in Skills")
|
||||
expect(result).toContain("User-Installed Skills")
|
||||
expect(result).toContain("HIGH PRIORITY")
|
||||
})
|
||||
|
||||
it("should include custom skill names in CRITICAL warning", () => {
|
||||
//#given: custom skills installed
|
||||
const allSkills = [...builtinSkills, ...customUserSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: should mention custom skills by name in the warning
|
||||
expect(result).toContain('"react-19"')
|
||||
expect(result).toContain('"tailwind-4"')
|
||||
expect(result).toContain("CRITICAL")
|
||||
})
|
||||
|
||||
it("should show source column for custom skills (user vs project)", () => {
|
||||
//#given: both user and project custom skills
|
||||
const allSkills = [...builtinSkills, ...customUserSkills, ...customProjectSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: should show source for each custom skill
|
||||
expect(result).toContain("| user |")
|
||||
expect(result).toContain("| project |")
|
||||
})
|
||||
|
||||
it("should not show custom skill section when only builtin skills exist", () => {
|
||||
//#given: only builtin skills
|
||||
const allSkills = [...builtinSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: should not contain custom skill emphasis
|
||||
expect(result).not.toContain("User-Installed Skills")
|
||||
expect(result).not.toContain("HIGH PRIORITY")
|
||||
expect(result).toContain("Available Skills")
|
||||
})
|
||||
|
||||
it("should handle only custom skills (no builtins)", () => {
|
||||
//#given: only custom skills, no builtins
|
||||
const allSkills = [...customUserSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: should show custom skills with emphasis, no builtin section
|
||||
expect(result).toContain("User-Installed Skills")
|
||||
expect(result).toContain("HIGH PRIORITY")
|
||||
expect(result).not.toContain("Built-in Skills")
|
||||
})
|
||||
|
||||
it("should include priority note for custom skills in evaluation step", () => {
|
||||
//#given: custom skills present
|
||||
const allSkills = [...builtinSkills, ...customUserSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: evaluation section should mention user-installed priority
|
||||
expect(result).toContain("User-installed skills get PRIORITY")
|
||||
expect(result).toContain("INCLUDE it rather than omit it")
|
||||
})
|
||||
|
||||
it("should NOT include priority note when no custom skills", () => {
|
||||
//#given: only builtin skills
|
||||
const allSkills = [...builtinSkills]
|
||||
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide(categories, allSkills)
|
||||
|
||||
//#then: no priority note for custom skills
|
||||
expect(result).not.toContain("User-installed skills get PRIORITY")
|
||||
})
|
||||
|
||||
it("should return empty string when no categories and no skills", () => {
|
||||
//#given: no categories and no skills
|
||||
//#when: building the delegation guide
|
||||
const result = buildCategorySkillsDelegationGuide([], [])
|
||||
|
||||
//#then: should return empty string
|
||||
expect(result).toBe("")
|
||||
})
|
||||
})
|
||||
|
||||
describe("buildUltraworkSection", () => {
|
||||
const agents: AvailableAgent[] = []
|
||||
|
||||
it("should separate builtin and custom skills", () => {
|
||||
//#given: mix of builtin and custom skills
|
||||
const skills: AvailableSkill[] = [
|
||||
{ name: "playwright", description: "Browser automation", location: "plugin" },
|
||||
{ name: "react-19", description: "React 19 patterns", location: "user" },
|
||||
]
|
||||
|
||||
//#when: building ultrawork section
|
||||
const result = buildUltraworkSection(agents, [], skills)
|
||||
|
||||
//#then: should have separate sections
|
||||
expect(result).toContain("Built-in Skills")
|
||||
expect(result).toContain("User-Installed Skills")
|
||||
expect(result).toContain("HIGH PRIORITY")
|
||||
})
|
||||
|
||||
it("should not separate when only builtin skills", () => {
|
||||
//#given: only builtin skills
|
||||
const skills: AvailableSkill[] = [
|
||||
{ name: "playwright", description: "Browser automation", location: "plugin" },
|
||||
]
|
||||
|
||||
//#when: building ultrawork section
|
||||
const result = buildUltraworkSection(agents, [], skills)
|
||||
|
||||
//#then: should have single section
|
||||
expect(result).toContain("Built-in Skills")
|
||||
expect(result).not.toContain("User-Installed Skills")
|
||||
})
|
||||
})
|
||||
@@ -174,11 +174,64 @@ export function buildCategorySkillsDelegationGuide(categories: AvailableCategory
|
||||
return `| \`${c.name}\` | ${desc} |`
|
||||
})
|
||||
|
||||
const skillRows = skills.map((s) => {
|
||||
const builtinSkills = skills.filter((s) => s.location === "plugin")
|
||||
const customSkills = skills.filter((s) => s.location !== "plugin")
|
||||
|
||||
const builtinRows = builtinSkills.map((s) => {
|
||||
const desc = s.description.split(".")[0] || s.description
|
||||
return `| \`${s.name}\` | ${desc} |`
|
||||
})
|
||||
|
||||
const customRows = customSkills.map((s) => {
|
||||
const desc = s.description.split(".")[0] || s.description
|
||||
const source = s.location === "project" ? "project" : "user"
|
||||
return `| \`${s.name}\` | ${desc} | ${source} |`
|
||||
})
|
||||
|
||||
const customSkillNames = customSkills.map((s) => `"${s.name}"`).join(", ")
|
||||
|
||||
let skillsSection: string
|
||||
|
||||
if (customSkills.length > 0 && builtinSkills.length > 0) {
|
||||
skillsSection = `#### Built-in Skills
|
||||
|
||||
| Skill | Expertise Domain |
|
||||
|-------|------------------|
|
||||
${builtinRows.join("\n")}
|
||||
|
||||
#### User-Installed Skills (HIGH PRIORITY)
|
||||
|
||||
**The user has installed these custom skills. They MUST be evaluated for EVERY delegation.**
|
||||
Subagents are STATELESS — they lose all custom knowledge unless you pass these skills via \`load_skills\`.
|
||||
|
||||
| Skill | Expertise Domain | Source |
|
||||
|-------|------------------|--------|
|
||||
${customRows.join("\n")}
|
||||
|
||||
> **CRITICAL**: Ignoring user-installed skills when they match the task domain is a failure.
|
||||
> The user installed ${customSkillNames} for a reason — USE THEM when the task overlaps with their domain.`
|
||||
} else if (customSkills.length > 0) {
|
||||
skillsSection = `#### User-Installed Skills (HIGH PRIORITY)
|
||||
|
||||
**The user has installed these custom skills. They MUST be evaluated for EVERY delegation.**
|
||||
Subagents are STATELESS — they lose all custom knowledge unless you pass these skills via \`load_skills\`.
|
||||
|
||||
| Skill | Expertise Domain | Source |
|
||||
|-------|------------------|--------|
|
||||
${customRows.join("\n")}
|
||||
|
||||
> **CRITICAL**: Ignoring user-installed skills when they match the task domain is a failure.
|
||||
> The user installed ${customSkillNames} for a reason — USE THEM when the task overlaps with their domain.`
|
||||
} else {
|
||||
skillsSection = `#### Available Skills (Domain Expertise Injection)
|
||||
|
||||
Skills inject specialized instructions into the subagent. Read the description to understand when each skill applies.
|
||||
|
||||
| Skill | Expertise Domain |
|
||||
|-------|------------------|
|
||||
${builtinRows.join("\n")}`
|
||||
}
|
||||
|
||||
return `### Category + Skills Delegation System
|
||||
|
||||
**delegate_task() combines categories and skills for optimal task execution.**
|
||||
@@ -191,13 +244,7 @@ Each category is configured with a model optimized for that domain. Read the des
|
||||
|----------|-------------------|
|
||||
${categoryRows.join("\n")}
|
||||
|
||||
#### Available Skills (Domain Expertise Injection)
|
||||
|
||||
Skills inject specialized instructions into the subagent. Read the description to understand when each skill applies.
|
||||
|
||||
| Skill | Expertise Domain |
|
||||
|-------|------------------|
|
||||
${skillRows.join("\n")}
|
||||
${skillsSection}
|
||||
|
||||
---
|
||||
|
||||
@@ -208,12 +255,15 @@ ${skillRows.join("\n")}
|
||||
- Match task requirements to category domain
|
||||
- Select the category whose domain BEST fits the task
|
||||
|
||||
**STEP 2: Evaluate ALL Skills**
|
||||
**STEP 2: Evaluate ALL Skills (Built-in AND User-Installed)**
|
||||
For EVERY skill listed above, ask yourself:
|
||||
> "Does this skill's expertise domain overlap with my task?"
|
||||
|
||||
- If YES → INCLUDE in \`load_skills=[...]\`
|
||||
- If NO → You MUST justify why (see below)
|
||||
${customSkills.length > 0 ? `
|
||||
> **User-installed skills get PRIORITY.** The user explicitly installed them for their workflow.
|
||||
> When in doubt about a user-installed skill, INCLUDE it rather than omit it.` : ""}
|
||||
|
||||
**STEP 3: Justify Omissions**
|
||||
|
||||
@@ -240,7 +290,7 @@ SKILL EVALUATION for "[skill-name]":
|
||||
\`\`\`typescript
|
||||
delegate_task(
|
||||
category="[selected-category]",
|
||||
load_skills=["skill-1", "skill-2"], // Include ALL relevant skills
|
||||
load_skills=["skill-1", "skill-2"], // Include ALL relevant skills — ESPECIALLY user-installed ones
|
||||
prompt="..."
|
||||
)
|
||||
\`\`\`
|
||||
@@ -328,12 +378,26 @@ export function buildUltraworkSection(
|
||||
}
|
||||
|
||||
if (skills.length > 0) {
|
||||
lines.push("**Skills** (combine with categories - EVALUATE ALL for relevance):")
|
||||
for (const skill of skills) {
|
||||
const shortDesc = skill.description.split(".")[0] || skill.description
|
||||
lines.push(`- \`${skill.name}\`: ${shortDesc}`)
|
||||
const builtinSkills = skills.filter((s) => s.location === "plugin")
|
||||
const customSkills = skills.filter((s) => s.location !== "plugin")
|
||||
|
||||
if (builtinSkills.length > 0) {
|
||||
lines.push("**Built-in Skills** (combine with categories):")
|
||||
for (const skill of builtinSkills) {
|
||||
const shortDesc = skill.description.split(".")[0] || skill.description
|
||||
lines.push(`- \`${skill.name}\`: ${shortDesc}`)
|
||||
}
|
||||
lines.push("")
|
||||
}
|
||||
|
||||
if (customSkills.length > 0) {
|
||||
lines.push("**User-Installed Skills** (HIGH PRIORITY - user installed these for their workflow):")
|
||||
for (const skill of customSkills) {
|
||||
const shortDesc = skill.description.split(".")[0] || skill.description
|
||||
lines.push(`- \`${skill.name}\`: ${shortDesc}`)
|
||||
}
|
||||
lines.push("")
|
||||
}
|
||||
lines.push("")
|
||||
}
|
||||
|
||||
if (agents.length > 0) {
|
||||
|
||||
Reference in New Issue
Block a user