feat: Implement skill description improvement and evaluation scripts

- Added `improve_description.py` to enhance skill descriptions based on evaluation results using Claude.
- Introduced `run_eval.py` to assess whether skill descriptions trigger correctly for a set of queries.
- Created `run_loop.py` to combine evaluation and improvement in a loop, tracking history and optimizing descriptions iteratively.
- Developed `utils.py` for shared utility functions, including parsing SKILL.md files.
- Enhanced the overall workflow for skill description optimization, supporting train/test splits to prevent overfitting.
This commit is contained in:
Misaka Server
2026-03-25 15:04:20 +08:00
parent 1352f0875b
commit 81fd98b4fb
19 changed files with 5133 additions and 662 deletions

View File

@@ -10,10 +10,33 @@ Example:
python utils/package_skill.py skills/public/my-skill ./dist
"""
import fnmatch
import sys
import zipfile
from pathlib import Path
from quick_validate import validate_skill
from scripts.quick_validate import validate_skill
# Patterns to exclude when packaging skills.
EXCLUDE_DIRS = {"__pycache__", "node_modules"}
EXCLUDE_GLOBS = {"*.pyc"}
EXCLUDE_FILES = {".DS_Store"}
# Directories excluded only at the skill root (not when nested deeper).
ROOT_EXCLUDE_DIRS = {"evals"}
def should_exclude(rel_path: Path) -> bool:
"""Check if a path should be excluded from packaging."""
parts = rel_path.parts
if any(part in EXCLUDE_DIRS for part in parts):
return True
# rel_path is relative to skill_path.parent, so parts[0] is the skill
# folder name and parts[1] (if present) is the first subdir.
if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS:
return True
name = rel_path.name
if name in EXCLUDE_FILES:
return True
return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS)
def package_skill(skill_path, output_dir=None):
@@ -66,13 +89,16 @@ def package_skill(skill_path, output_dir=None):
# Create the .skill file (zip format)
try:
with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
# Walk through the skill directory
# Walk through the skill directory, excluding build artifacts
for file_path in skill_path.rglob('*'):
if file_path.is_file():
# Calculate the relative path within the zip
arcname = file_path.relative_to(skill_path.parent)
zipf.write(file_path, arcname)
print(f" Added: {arcname}")
if not file_path.is_file():
continue
arcname = file_path.relative_to(skill_path.parent)
if should_exclude(arcname):
print(f" Skipped: {arcname}")
continue
zipf.write(file_path, arcname)
print(f" Added: {arcname}")
print(f"\n✅ Successfully packaged skill to: {skill_filename}")
return skill_filename