Skills/agent-evals/assets/trigger-queries.skill-creator.json
|
[ { "id": "pos-01", "query": "this SKILL.md is 700 lines now, it's getting unwieldy — sort it out", "should_trigger": true, "split": "train" }, { "id": "pos-02", "query": "we keep doing the same release-notes dance every sprint. can you package that up so it's reusable?", "should_trigger": true, "split": "train" }, { "id": "pos-03", "query": "why does my skill never fire when I ask about certificates?", "should_trigger": true, "split": "train" }, { "id": "pos-04", "query": "review the skills under Skills/ and tell me which ones overlap", "should_trigger": true, "split": "train" }, { "id": "pos-05", "query": "the description got rejected for being too long, trim it", "should_trigger": true, "split": "train" }, { "id": "pos-06", "query": "turn this workflow into something the agent can pick up on its own", "should_trigger": true, "split": "validation" }, { "id": "pos-07", "query": "kannst du aus dieser Anleitung einen wiederverwendbaren Skill bauen?", "should_trigger": true, "split": "validation" }, { "id": "pos-08", "query": "audit skill-creator against the current spec and tell me what's missing", "should_trigger": true, "split": "validation" }, { "id": "pos-09", "query": "should this be one skill or split into references?", "should_trigger": true, "split": "validation" }, { "id": "pos-10", "query": "ich möchte diesen Freigabeprozess als Skill verpacken, damit der Agent ihn künftig selbst findet", "should_trigger": true, "split": "train", "note": "a German packaging request. It passes, which is what disproved the theory that non-English phrasing is why pos-07 misses; kept as the regression guard for that" }, { "id": "pos-11", "query": "is this big enough to be its own skill, or should the extra material live in a reference file?", "should_trigger": true, "split": "train", "note": "train stand-in for pos-09: a structure decision phrased as a question rather than as an instruction" }, { "id": "pos-12", "query": "we have a how-to document for this — build a skill from it", "should_trigger": true, "split": "train", "note": "the concept pos-07 actually tests. A probe measured 2026-08-11 showed pos-07's literal English translation failing 0/3 while a differently worded German request passed 3/3, so the variable is converting existing material, not the language" }, { "id": "neg-01", "query": "write an instructions file that applies to all our yaml", "should_trigger": false, "split": "train", "note": "Instructions authoring - explicitly excluded by DO NOT USE FOR" }, { "id": "neg-02", "query": "run the eval harness against the docx skill and give me the pass rates", "should_trigger": false, "split": "train", "note": "belongs to agent-evals - the nearest neighbour, sharpest boundary" }, { "id": "neg-03", "query": "add a new MCP server to my config", "should_trigger": false, "split": "train" }, { "id": "neg-04", "query": "my agent keeps ignoring the system prompt, what's wrong with it", "should_trigger": false, "split": "train", "note": "agent authoring, not skill authoring - lexically very close" }, { "id": "neg-05", "query": "the markdown table in this readme renders wrong on github", "should_trigger": false, "split": "train" }, { "id": "neg-06", "query": "what skills do I have available right now?", "should_trigger": false, "split": "validation", "note": "mentions skills but asks for inventory, not authoring" }, { "id": "neg-07", "query": "write a prompt file for the weekly status report", "should_trigger": false, "split": "validation" }, { "id": "neg-08", "query": "our onboarding doc is 900 lines, can you split it into sections", "should_trigger": false, "split": "validation", "note": "same 'too long, split it' shape as pos-01 but not a skill - the key near-miss" }, { "id": "neg-09", "query": "package this powershell module for the gallery", "should_trigger": false, "split": "validation", "note": "'package this' collides with pos-02 phrasing" }, { "id": "neg-10", "query": "kannst du mir eine Instructions-Datei für unsere YAML-Dateien schreiben?", "should_trigger": false, "split": "train", "note": "German near-miss guarding pos-10: excluded work stated in German must still not trigger" }, { "id": "neg-11", "query": "should this be one agent or two separate ones?", "should_trigger": false, "split": "train", "note": "near-miss guarding pos-11: same 'one or split it' decision shape, but about agents" } ] |