Compare commits
13 Commits
6350f7a92a
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
0f781f7b75 | ||
|
|
ba2c096323 | ||
|
|
687ab5ee28 | ||
|
|
55da2b74a4 | ||
|
|
1351371bd2 | ||
|
|
57b797196c | ||
|
|
786038d22e | ||
|
|
81fd98b4fb | ||
|
|
1352f0875b | ||
|
|
3c073b73e2 | ||
|
|
22fbff551d | ||
|
|
18fdf0c55c | ||
|
|
802208b9f2 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -41,6 +41,7 @@ Thumbs.db
|
||||
|
||||
# .claude
|
||||
.claude/
|
||||
|
||||
.agents/
|
||||
skills-lock.json
|
||||
#temp files
|
||||
temp/
|
||||
128
demo_scripts/check-mermaid.js
Normal file
128
demo_scripts/check-mermaid.js
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const { execSync } = require('child_process');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
// 获取命令行传入的 Markdown 文件路径
|
||||
const fileArg = process.argv[2];
|
||||
if (!fileArg) {
|
||||
console.error('❌ 请提供 Markdown 文件路径。用法: node check-mermaid.js <file.md>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const filePath = path.resolve(fileArg);
|
||||
const fileName = path.basename(filePath);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`❌ 文件不存在: ${filePath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
|
||||
// 正则匹配 ```mermaid ... ``` 代码块，兼容 Windows(\r\n) 和 Linux(\n)
|
||||
const mermaidRegex = /```mermaid\r?\n([\s\S]*?)```/g;
|
||||
let match;
|
||||
let blockCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
// 用于收集所有格式化后的错误报告块
|
||||
const errorReports = [];
|
||||
|
||||
while ((match = mermaidRegex.exec(content)) !== null) {
|
||||
blockCount++;
|
||||
|
||||
const fullMatch = match[0];
|
||||
const code = match[1];
|
||||
|
||||
// 1. 计算在 Markdown 文件中的绝对行号
|
||||
const textBeforeMatch = content.substring(0, match.index);
|
||||
const startLine = textBeforeMatch.split(/\r?\n/).length;
|
||||
const endLine = startLine + fullMatch.split(/\r?\n/).length - 1;
|
||||
|
||||
// 创建临时文件存放单个 Mermaid 代码
|
||||
const tmpFile = path.join(os.tmpdir(), `mermaid-check-${Date.now()}-${blockCount}.mmd`);
|
||||
fs.writeFileSync(tmpFile, code.trim());
|
||||
|
||||
try {
|
||||
// 调用 mmdc 进行静默渲染检查
|
||||
execSync(`npx mmdc -i "${tmpFile}" -o "${tmpFile}.svg" -q`, { stdio: 'pipe' });
|
||||
} catch (error) {
|
||||
errorCount++;
|
||||
|
||||
// 2. 净化报错信息：剔除底层执行堆栈
|
||||
const stderr = error.stderr ? error.stderr.toString() : error.message;
|
||||
const errorLines = stderr.split(/\r?\n/);
|
||||
const cleanErrorLines = [];
|
||||
|
||||
for (const line of errorLines) {
|
||||
if (line.trim().startsWith('at ') ||
|
||||
line.includes('Parser3.parseError') ||
|
||||
line.includes('fromText')) {
|
||||
break;
|
||||
}
|
||||
cleanErrorLines.push(line);
|
||||
}
|
||||
const cleanErrorOutput = cleanErrorLines.join('\n').trim();
|
||||
|
||||
// 3. 截断代码内容：最少3行，最多6行
|
||||
const codeLines = code.split(/\r?\n/).filter(l => l.trim() !== '');
|
||||
let displayCode = '';
|
||||
|
||||
if (codeLines.length <= 6) {
|
||||
displayCode = codeLines.join('\n');
|
||||
} else {
|
||||
const head = codeLines.slice(0, 3).join('\n');
|
||||
const tail = codeLines.slice(-3).join('\n');
|
||||
displayCode = `${head}\n ...\n ... (中间省略 ${codeLines.length - 6} 行) ...\n ...\n${tail}`;
|
||||
}
|
||||
|
||||
// 4. 组装单个错误的 Markdown 块
|
||||
errorReports.push(`### ❌ 错误 #${errorCount} (代码块 #${blockCount})
|
||||
|
||||
- **文档位置：** 第 \`${startLine}\` 行至第 \`${endLine}\` 行
|
||||
|
||||
#### 核心错误详情
|
||||
\`\`\`text
|
||||
${cleanErrorOutput}
|
||||
\`\`\`
|
||||
|
||||
#### 代码内容片段
|
||||
\`\`\`text
|
||||
${displayCode}
|
||||
\`\`\``);
|
||||
|
||||
} finally {
|
||||
// 清理临时文件
|
||||
if (fs.existsSync(tmpFile)) fs.unlinkSync(tmpFile);
|
||||
if (fs.existsSync(`${tmpFile}.svg`)) fs.unlinkSync(`${tmpFile}.svg`);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 生成最终的 Markdown 报告输出
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
if (errorCount > 0) {
|
||||
// 存在错误的情况
|
||||
console.log(`## 🚨 Mermaid 语法检查报告\n`);
|
||||
console.log(`**检查文件：** \`${fileName}\``);
|
||||
console.log(`**检查结果：** ❌ 发现 ${errorCount} 处语法错误 (共检测到 ${blockCount} 个代码块)\n`);
|
||||
console.log(`---\n`);
|
||||
console.log(errorReports.join('\n\n---\n\n'));
|
||||
|
||||
// 返回非零状态码，确保在 CI/CD 或 Git Hook 中能够阻断流程
|
||||
process.exit(1);
|
||||
} else {
|
||||
// 全部正确或没有代码块的情况
|
||||
console.log(`## 🎉 Mermaid 语法检查报告\n`);
|
||||
console.log(`**检查文件：** \`${fileName}\``);
|
||||
if (blockCount === 0) {
|
||||
console.log(`**检查结果：** ⚠️ 未检测到 Mermaid 代码块`);
|
||||
} else {
|
||||
console.log(`**检查结果：** ✅ 全部通过 (共检测到 ${blockCount} 个代码块)`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
482
demo_scripts/demo.md
Normal file
482
demo_scripts/demo.md
Normal file
@@ -0,0 +1,482 @@
|
||||
# Admin 用户物料清理数据流说明文档
|
||||
|
||||
## 概述
|
||||
|
||||
本文档详细说明 ERPAuto 系统中，Admin 用户执行物料清理操作时，被清理物料的完整获取流程、数据来源和处理链路。
|
||||
|
||||
## 核心结论
|
||||
|
||||
**Admin 用户清理的物料来源**：被清理的物料代码从数据库表 `dbo.MaterialsToBeDeleted` 中获取，根据 Admin 用户在 UI 界面选择的负责人（Manager）进行过滤。
|
||||
|
||||
---
|
||||
|
||||
## 数据流总览
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant UI as CleanerPage (UI)
|
||||
participant Hook as useCleaner Hook
|
||||
participant API as Renderer API
|
||||
participant IPC as IPC Channel
|
||||
participant Service as ValidationService
|
||||
participant DB as Database
|
||||
|
||||
UI->>Hook: handleExecuteDeletion()
|
||||
Hook->>API: runCleanerExecution()
|
||||
API->>IPC: validation.getCleanerData({selectedManagers})
|
||||
|
||||
IPC->>Service: getCleanerData(userInfo, selectedManagers)
|
||||
|
||||
alt Admin User
|
||||
Service->>Service: loadMaterialCodesForCleaner()
|
||||
Service->>DB: SELECT MaterialCode FROM dbo.MaterialsToBeDeleted<br/>WHERE ManagerName IN (selectedManagers)
|
||||
DB-->>Service: materialCodes[]
|
||||
Service-->>IPC: {orderNumbers[], materialCodes[]}
|
||||
else Regular User
|
||||
Service->>DB: SELECT MaterialCode FROM dbo.MaterialsToBeDeleted<br/>WHERE ManagerName = username
|
||||
DB-->>Service: materialCodes[]
|
||||
Service-->>IPC: {orderNumbers[], materialCodes[]}
|
||||
end
|
||||
|
||||
IPC-->>API: cleanerData
|
||||
API->>IPC: cleaner.runCleaner({orderNumbers, materialCodes})
|
||||
IPC->>Service: CleanerApplicationService.runCleaner()
|
||||
Service->>UI: 执行清理 (ERP 删除)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 架构分层
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph "渲染进程 (Renderer)"
|
||||
UI[CleanerPage.tsx]
|
||||
HOOK[useCleaner.ts]
|
||||
RAPI[renderer/api.ts]
|
||||
end
|
||||
|
||||
subgraph "主进程 (Main)"
|
||||
IPC_VAL[IPC: validation.getCleanerData]
|
||||
IPC_CLEAN[IPC: cleaner.runCleaner]
|
||||
VAS[ValidationApplicationService]
|
||||
CAS[CleanerApplicationService]
|
||||
ERP["CleanerService (ERP)"]
|
||||
end
|
||||
|
||||
subgraph "数据库 (Database)"
|
||||
MTBD[(dbo.MaterialsToBeDeleted)]
|
||||
DMPD[(dbo.DiscreteMaterialPlanData)]
|
||||
MTTD[(dbo.MaterialsTypeToBeDeleted)]
|
||||
end
|
||||
|
||||
UI --> HOOK
|
||||
HOOK --> RAPI
|
||||
RAPI --> IPC_VAL
|
||||
RAPI --> IPC_CLEAN
|
||||
IPC_VAL --> VAS
|
||||
IPC_CLEAN --> CAS
|
||||
VAS --> MTBD
|
||||
VAS --> DMPD
|
||||
VAS --> MTTD
|
||||
CAS --> ERP
|
||||
|
||||
style MTBD fill:#f9f,stroke:#333
|
||||
style MTTD fill:#f9f,stroke:#333
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 关键数据表
|
||||
|
||||
### 1. `dbo.MaterialsToBeDeleted` (核心来源表)
|
||||
|
||||
**作用**：存储所有被标记为待删除的物料代码及其负责人。
|
||||
|
||||
**表结构**：
|
||||
| 字段 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `ID` | int | 主键 |
|
||||
| `MaterialCode` | varchar | **物料代码 (被清理的目标)** |
|
||||
| `ManagerName` | varchar | 负责人姓名 |
|
||||
|
||||
**Admin 获取物料的 SQL 查询**：
|
||||
|
||||
```sql
|
||||
SELECT MaterialCode
|
||||
FROM dbo.MaterialsToBeDeleted
|
||||
WHERE ManagerName IN (@manager0, @manager1, ...)
|
||||
AND MaterialCode IS NOT NULL
|
||||
```
|
||||
|
||||
### 2. `dbo.MaterialsTypeToBeDeleted` (物料类型配置表)
|
||||
|
||||
**作用**：定义物料名称关键词与负责人的映射关系，用于自动分配负责人。
|
||||
|
||||
**表结构**：
|
||||
| 字段 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `MaterialName` | varchar | 物料名称关键词 |
|
||||
| `ManagerName` | varchar | 对应的负责人 |
|
||||
|
||||
**示例**：
|
||||
| MaterialName | ManagerName |
|
||||
|-------------|-------------|
|
||||
| "电阻" | "张三" |
|
||||
| "电容" | "李四" |
|
||||
|
||||
### 3. `dbo.DiscreteMaterialPlanData` (物料计划数据表)
|
||||
|
||||
**作用**：存储从 ERP 提取的完整物料计划数据，用于校验和展示物料详情。
|
||||
|
||||
**关键字段**：
|
||||
|
||||
- `MaterialCode` - 物料代码
|
||||
- `MaterialName` - 物料名称
|
||||
- `Specification` - 规格
|
||||
- `Model` - 型号
|
||||
- `SourceNo` - 订单号
|
||||
|
||||
---
|
||||
|
||||
## Admin 用户完整数据流
|
||||
|
||||
### 阶段 1: 校验 (Validation)
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph "步骤 1: 用户触发的数据来源"
|
||||
A1[Admin 点击'数据校验']
|
||||
end
|
||||
|
||||
subgraph "步骤 2: 数据获取模式"
|
||||
A2{校验模式？}
|
||||
A21[Full Mode<br/>全量数据]
|
||||
A22[Filtered Mode<br/>筛选数据]
|
||||
end
|
||||
|
||||
subgraph "步骤 3: 数据库查询"
|
||||
A3[DiscreteMaterialPlanDAO]
|
||||
A31["queryAllDistinctByMaterialCode()"]
|
||||
A32["queryBySourceNumbersDistinct(orderNumbers)"]
|
||||
end
|
||||
|
||||
subgraph "步骤 4: 数据增强"
|
||||
A4[加载负责人信息]
|
||||
A41[加载 MaterialsTypeToBeDeleted<br/>关键词匹配]
|
||||
A42[加载 MaterialsToBeDeleted<br/>已标记记录]
|
||||
end
|
||||
|
||||
subgraph "结果"
|
||||
A5["ValidationResult[]<br/>包含 isMarkedForDeletion 标志"]
|
||||
end
|
||||
|
||||
A1 --> A2
|
||||
A2 -->|Full| A21
|
||||
A2 -->|Filtered| A22
|
||||
A21 --> A31
|
||||
A22 --> A32
|
||||
A31 --> A4
|
||||
A32 --> A4
|
||||
A4 --> A41
|
||||
A4 --> A42
|
||||
A4 --> A5
|
||||
|
||||
style A5 fill:#9f9,stroke:#333
|
||||
```
|
||||
|
||||
**Admin 特殊逻辑**：
|
||||
|
||||
- Admin 可以看到**所有负责人**的物料
|
||||
- UI 会显示 Manager 列下拉筛选器
|
||||
- Admin 默认选中所有 Manager
|
||||
|
||||
### 阶段 2: 保存删除计划 (可选)
|
||||
|
||||
用户在 UI 上勾选物料 → 点击"确认删除" → 数据写入 `MaterialsToBeDeleted` 表：
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant UI as 用户界面
|
||||
participant DAO as MaterialsToBeDeletedDAO
|
||||
participant DB as Database
|
||||
|
||||
UI->>DAO: upsertBatch(materials[])
|
||||
|
||||
loop 对每个物料
|
||||
DAO->>DB: MERGE INTO dbo.MaterialsToBeDeleted<br/>ON MaterialCode<br/>WHEN MATCHED UPDATE<br/>WHEN NOT MATCHED INSERT
|
||||
end
|
||||
|
||||
DB-->>DAO: 成功/失败统计
|
||||
DAO-->>UI: { total, success, failed }
|
||||
```
|
||||
|
||||
**SQL 逻辑**（MERGE UPSERT）：
|
||||
|
||||
```sql
|
||||
MERGE INTO dbo.MaterialsToBeDeleted AS target
|
||||
USING (VALUES (@MaterialCode, @ManagerName)) AS source (MaterialCode, ManagerName)
|
||||
ON target.MaterialCode = source.MaterialCode
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET ManagerName = source.ManagerName
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (MaterialCode, ManagerName) VALUES (source.MaterialCode, source.ManagerName);
|
||||
```
|
||||
|
||||
### 阶段 3: 执行清理 (关键步骤)
|
||||
|
||||
这是 Admin 用户执行实际删除操作的核心流程：
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph "Renderer 层"
|
||||
S1["handleExecuteDeletion()"]
|
||||
S1-->S2["runCleanerExecution {selectedManagers}"]
|
||||
end
|
||||
|
||||
subgraph "获取清理数据 (主进程)"
|
||||
S2-->S3[ValidationApplicationService.getCleanerData]
|
||||
S3-->S4{用户类型？}
|
||||
|
||||
S4-->|Admin + selectedManagers| S5[loadMaterialCodesForCleaner<br/>Admin With Managers]
|
||||
S5-->S6[SELECT FROM MaterialsToBeDeleted<br/>WHERE ManagerName IN selectedManagers]
|
||||
|
||||
S4-->|Admin no managers| S7[Admin Without Managers]
|
||||
S7-->S8[SELECT FROM DiscreteMaterialPlanData<br/>WHERE SourceNo IN orderNumbers]
|
||||
|
||||
S4-->|普通用户 | S9[SELECT FROM MaterialsToBeDeleted<br/>WHERE ManagerName = username]
|
||||
end
|
||||
|
||||
subgraph "执行清理"
|
||||
S6-->S10["返回 materialCodes[]"]
|
||||
S8-->S10
|
||||
S9-->S10
|
||||
S10-->S11[CleanerApplicationService.runCleaner]
|
||||
S11-->S12[CleanerService.clean<br/>连接 ERP 执行删除]
|
||||
end
|
||||
|
||||
S12-->S13[清理完成报告]
|
||||
|
||||
style S5 fill:#ff9,stroke:#333
|
||||
style S6 fill:#f96,stroke:#333
|
||||
style S12 fill:#f99,stroke:#333
|
||||
```
|
||||
|
||||
**关键代码路径**：
|
||||
|
||||
```typescript
|
||||
// src/main/services/validation/validation-application-service.ts
|
||||
// loadMaterialCodesForCleaner() 方法
|
||||
|
||||
// Admin with selected managers:
|
||||
if (isAdmin && selectedManagers && selectedManagers.length > 0) {
|
||||
const materialCodes = await this.queryMaterialCodesByManagers(
|
||||
dbService,
|
||||
markedTableName, // dbo.MaterialsToBeDeleted
|
||||
selectedManagers
|
||||
)
|
||||
return materialCodes
|
||||
}
|
||||
|
||||
// Admin without selected managers (fallback):
|
||||
if (isAdmin) {
|
||||
if (orderNumbers.length === 0) {
|
||||
return [] // 无数据可处理
|
||||
}
|
||||
const materialDao = new DiscreteMaterialPlanDAO()
|
||||
const records = await materialDao.queryBySourceNumbersDistinct(orderNumbers)
|
||||
const materialCodes = [...new Set(records.map((r) => r.MaterialCode as string).filter(Boolean))]
|
||||
return materialCodes
|
||||
}
|
||||
```
|
||||
|
||||
### 阶段 4: ERP 删除执行
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant CAS as CleanerApplicationService
|
||||
participant Cleaner as CleanerService
|
||||
participant ERP as ERP System
|
||||
|
||||
CAS->>Cleaner: clean({orderNumbers, materialCodes})
|
||||
|
||||
loop 每个订单号
|
||||
Cleaner->>ERP: 登录 (一次)
|
||||
loop 每个物料代码
|
||||
Cleaner->>ERP: 查询物料计划
|
||||
alt 物料存在
|
||||
Cleaner->>ERP: 执行删除
|
||||
ERP-->>Cleaner: 删除结果
|
||||
else 物料不存在
|
||||
Cleaner-->>Cleaner: 跳过
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Cleaner-->>CAS: CleanerResult
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Admin 用户的数据来源路径
|
||||
|
||||
整理出 Admin 用户获取物料的完整路径：
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
Start[Admin 点击执行清理] --> Check{有 selectedManagers?}
|
||||
|
||||
Check -->|Yes| Path1[路径 1: MaterialsToBeDeleted 表]
|
||||
Path1 --> Q1["SELECT MaterialCode FROM MaterialsToBeDeleted<br/>WHERE ManagerName IN (selectedManagers)"]
|
||||
Q1 --> Merge[合并所有物料代码]
|
||||
|
||||
Check -->|No| Path2[路径 2: DiscreteMaterialPlanData 表]
|
||||
Path2 --> NeedProd{有共享 Production ID?}
|
||||
NeedProd -->|Yes| GetOrder[从共享 ID 解析订单号]
|
||||
GetOrder --> Q2["SELECT DISTINCT MaterialCode FROM DiscreteMaterialPlanData<br/>WHERE SourceNo IN (orderNumbers)"]
|
||||
Q2 --> Merge
|
||||
|
||||
NeedProd -->|No| Empty[返回空数组<br/>无法执行清理]
|
||||
|
||||
Merge --> Execute[传递给 CleanerService<br/>执行 ERP 删除]
|
||||
|
||||
style Path1 fill:#9f9,stroke:#333,stroke-width:2px
|
||||
style Path2 fill:#ff9,stroke:#333,stroke-width:2px
|
||||
style Empty fill:#f99,stroke:#333
|
||||
style Execute fill:#f96,stroke:#333,stroke-width:3px
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 关键代码位置参考
|
||||
|
||||
### 渲染层 (Renderer)
|
||||
|
||||
| 文件 | 函数 | 说明 |
|
||||
| ---------------------------------------- | ----------------------- | -------------------------- |
|
||||
| `src/renderer/src/pages/CleanerPage.tsx` | `handleExecuteDeletion` | 清理入口 |
|
||||
| `src/renderer/src/hooks/useCleaner.ts` | `handleExecuteDeletion` | 调用 `runCleanerExecution` |
|
||||
| `src/renderer/src/hooks/cleaner/api.ts` | `runCleanerExecution` | 先获取数据再执行清理 |
|
||||
|
||||
### 主进程层 (Main)
|
||||
|
||||
| 文件 | 类/函数 | 说明 |
|
||||
| ---------------------------------------------------------------- | ----------------------------------------------- | ---------------- |
|
||||
| `src/main/ipc/validation-handler.ts` | `VALIDATION_GET_CLEANER_DATA` | IPC 入口 |
|
||||
| `src/main/services/validation/validation-application-service.ts` | `getCleanerData`, `loadMaterialCodesForCleaner` | **核心逻辑** |
|
||||
| `src/main/services/cleaner/cleaner-application-service.ts` | `runCleaner` | 执行 ERP 清理 |
|
||||
| `src/main/services/erp/cleaner.ts` | `CleanerService.clean` | ERP 浏览器自动化 |
|
||||
|
||||
### 数据库层 (DAO)
|
||||
|
||||
| 文件 | 类 | 说明 |
|
||||
| ----------------------------------------------------------- | ------------------------- | ---------------------------- |
|
||||
| `src/main/services/database/materials-to-be-deleted-dao.ts` | `MaterialsToBeDeletedDAO` | `MaterialsToBeDeleted`表操作 |
|
||||
| `src/main/services/database/discrete-material-plan-dao.ts` | `DiscreteMaterialPlanDAO` | 物料计划数据查询 |
|
||||
|
||||
---
|
||||
|
||||
## 数据流状态图
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> 物料录入：用户在 UI 输入物料
|
||||
物料录入 --> 待校验：保存到 MaterialsToBeDeleted 表
|
||||
|
||||
待校验 --> 校验中：点击"数据校验"
|
||||
校验中 --> 已标记：通过关键词匹配负责人
|
||||
已标记 --> 待清理：用户勾选物料
|
||||
|
||||
待清理 --> 清理执行中：点击"执行清理"
|
||||
清理执行中 --> 清理完成：ERP 删除成功
|
||||
清理执行中 --> 部分失败：部分物料删除失败
|
||||
|
||||
清理完成 --> [*]
|
||||
部分失败 --> [*]
|
||||
|
||||
note right of 待清理
|
||||
Admin 可以查看和选择
|
||||
所有负责人的物料
|
||||
end note
|
||||
|
||||
note right of 清理执行中
|
||||
从 MaterialsToBeDeleted 表
|
||||
根据 selectedManagers 过滤
|
||||
获取要删除的物料代码
|
||||
end note
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 数据流对比：Admin vs 普通用户
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph "Admin 用户"
|
||||
A1[勾选多个 Manager]
|
||||
A2[SELECT FROM MaterialsToBeDeleted<br/>WHERE ManagerName IN selectedManagers]
|
||||
A3[获取所有选中负责人的物料]
|
||||
end
|
||||
|
||||
subgraph "普通用户"
|
||||
B1[只能看到自己的物料]
|
||||
B2[SELECT FROM MaterialsToBeDeleted<br/>WHERE ManagerName = username]
|
||||
B3[只能删除自己的物料]
|
||||
end
|
||||
|
||||
A1 --> A2 --> A3 --> Exec[执行清理]
|
||||
B1 --> B2 --> B3 --> Exec
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 常见问题解答
|
||||
|
||||
### Q1: **物料代码是如何被录入到 `MaterialsToBeDeleted` 表的？**
|
||||
|
||||
**A**: 有三种方式：
|
||||
|
||||
1. **用户手动勾选** → 在 CleanerPage 勾选物料 → 点击"确认删除" → `upsertBatch`
|
||||
2. **关键词自动匹配** → 校验时根据 `MaterialsTypeToBeDeleted` 配置自动分配负责人和标记
|
||||
3. **API 直接写入** → 其他服务调用 `materials.upsertBatch` IPC
|
||||
|
||||
### Q2: **Admin 如果不选 Manager 会怎样？**
|
||||
|
||||
**A**: Admin 可以不选 Manager，此时：
|
||||
|
||||
- 系统会尝试从"共享 Production ID"解析订单号
|
||||
- 然后从 `DiscreteMaterialPlanData` 表查询所有物料代码（不经过 `MaterialsToBeDeleted` 过滤）
|
||||
- 如果没有共享 Production ID，则返回空数组，无法执行清理
|
||||
|
||||
### Q3: **物料代码会在清理后被自动删除吗？**
|
||||
|
||||
**A**:
|
||||
|
||||
- ✅ **干运行模式**：不删除 ERP 数据，但会保留 UI 状态
|
||||
- ✅ **正式执行**：
|
||||
- ERP 中的物料计划被删除
|
||||
- `MaterialsToBeDeleted` 表中记录**不会被自动删除**（需要手动清理）
|
||||
|
||||
### Q4: **如何清理已删除的物料记录？**
|
||||
|
||||
**A**: Admin 可以在物料管理界面：
|
||||
|
||||
- 按 Manager 筛选
|
||||
- 批量删除已处理的物料记录
|
||||
- 调用 `materials.delete` IPC 接口
|
||||
|
||||
---
|
||||
|
||||
## 总结
|
||||
|
||||
Admin 用户执行清理时，被清理物料的来源路径：
|
||||
|
||||
1. **主要来源**: `dbo.MaterialsToBeDeleted` 表
|
||||
2. **过滤条件**: `ManagerName IN (selectedManagers)`
|
||||
3. **执行流程**:
|
||||
- 从数据库查询物料代码
|
||||
- 结合订单号列表
|
||||
- 通过 Playwright 连接 ERP 系统
|
||||
- 逐个物料执行删除操作
|
||||
|
||||
**关键点**：被清理的物料**必须**先在 `MaterialsToBeDeleted` 表中存在记录，并且其 `ManagerName` 与 Admin 选择的负责人匹配。
|
||||
@@ -1,262 +1,290 @@
|
||||
---
|
||||
name: excel-report-converter
|
||||
description: Generate Python scripts to convert report-style Excel files to database-record format. Use for converting Excel files with multiple stacked reports into flat database tables, analyzing report structure, creating custom conversion scripts for specific Excel report formats, and transforming hierarchical report data (header + detail lines + footer) into normalized database records
|
||||
description: 当用户需要创建Python脚本将报表形式的Excel数据转换为结构化数据表时使用此技能。当用户提到：将Excel报表转换为数据表、解析Excel报表结构、创建Excel数据转换脚本、从打印模板格式提取数据、或者有类似"请购单维护"、"订单打印模板"等报表格式需要转换为标准表格时，即使没有明确说"创建脚本"，也应该触发此技能。这个技能专门处理那些面向打印/展示的报表布局（数据分散在多行多列、有表头/明细/页脚区块）到标准关系型数据表的转换。
|
||||
---
|
||||
|
||||
# Excel Report Converter
|
||||
# Excel报表数据转换技能
|
||||
|
||||
Generate Python scripts to convert report-style Excel files (multiple reports per worksheet) into database-record format (flat table).
|
||||
这个技能帮助你创建Python脚本，将各种报表形式的Excel数据转换为标准的数据表格式。
|
||||
|
||||
## When to Use
|
||||
## 适用场景
|
||||
|
||||
Use this skill when:
|
||||
- User provides an Excel file with multiple similar reports stacked vertically in one worksheet
|
||||
- Each report has hierarchical structure: header information + detail data rows + footer information
|
||||
- User wants to convert to database-record format where header fields are repeated for each detail row
|
||||
- The report structure is consistent across all reports in the file
|
||||
当你遇到以下情况时，使用此技能：
|
||||
|
||||
## Workflow
|
||||
- **报表式布局**：数据不是标准的行列表格，而是分散在多个区域
|
||||
- **有表头/明细/页脚结构**：一个文档包含主信息、明细列表、汇总信息
|
||||
- **打印模板格式**：为打印设计的Excel文件，需要提取其中的数据
|
||||
- **多区块重复**：同一个Excel文件包含多个相同格式的报表区块
|
||||
- **字段映射复杂**：目标字段与源单元格位置有复杂的对应关系
|
||||
- **大数据量提取（性能敏感）**：需要快速提取数千行以上、包含大量单元格的复杂Excel报表
|
||||
|
||||
### Step 1: Extract and Analyze Report Structure
|
||||
---
|
||||
|
||||
Use the `excel-to-markdown` skill to convert the Excel file to markdown format for analysis:
|
||||
## 🚀 性能优化核心技术 (提速法则)
|
||||
在处理中大型Excel报表时，传统的 `openpyxl` 逐个单元格读取方法会导致极严重的性能瓶颈。本技能强制采用以下高阶优化方案：
|
||||
|
||||
```bash
|
||||
# Convert Excel to markdown to analyze structure
|
||||
python3 scripts/excel_to_markdown.py input.xlsx -o /tmp/analysis.md --show-rows --show-cols
|
||||
```
|
||||
1. **空间换时间（内存二维数组）**：禁止在循环中频繁调用 `sheet.cell(row, col).value`。必须使用 `sheet.iter_rows(values_only=True)` 将整表数据一次性读入 Python 的 `List[List]` 中。后续所有的查找全部基于内存列表索引进行，速度可提升数十倍。
|
||||
2. **预先构建映射字典**：将 `column_index_from_string` (字母转数字索引) 等固定操作移出循环，在脚本初始化时预先计算好映射字典。
|
||||
3. **Pandas 向量化降维打击**：在保存 Excel 自动调整列宽时，放弃 `openpyxl` 的逐单元格遍历，直接利用 Pandas 的底层 C 语言级别操作 `df[col].astype(str).map(len).max()` 秒算列宽。
|
||||
4. **避开 `read_only` 的“公式陷阱”**：绝不能为了加载速度盲目开启 `read_only=True`，这会导致由 Excel 隐式公式（如自动递增序号 `=A1+1`）生成的值返回 `None`。必须坚持使用默认加载模式配合 `data_only=True`，然后依靠“内存二维数组”来解决速度问题。
|
||||
|
||||
Read the markdown file and identify:
|
||||
1. **Report delimiters**: How to identify where each report starts/ends (e.g., specific title in column 1)
|
||||
2. **Header structure**: Which rows contain header information and what fields are in each column
|
||||
3. **Detail table**: Which row contains column headers and where detail data starts/ends
|
||||
4. **Footer structure**: Which rows contain footer information and where the data is located
|
||||
5. **Row separators**: Are there empty rows between reports? After the last row?
|
||||
---
|
||||
|
||||
### Step 2: Generate Conversion Script
|
||||
## 工作流程
|
||||
|
||||
Create a Python script with the following structure:
|
||||
### 第一步：分析源Excel文件结构并构建内存视图
|
||||
|
||||
使用openpyxl读取Excel文件，并立即将其转换为内存二维数组以提升后续处理速度：
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Convert report-style Excel files to database-record format
|
||||
Customized for: [describe the report format]
|
||||
"""
|
||||
import openpyxl
|
||||
from typing import List, Any
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from openpyxl import load_workbook
|
||||
# 1. 加载工作簿 (保留 data_only=True, 弃用 read_only=True 保证公式值完整)
|
||||
wb = openpyxl.load_workbook(file_path, data_only=True)
|
||||
sheet = wb.active
|
||||
|
||||
# 2. 【核心提速机制】将整表数据一次性抽取为 Python 二维数组
|
||||
# 填充一行 [None]，并为每一行填充一列 [None]，使后续列表索引(1-based)与Excel坐标严格对齐
|
||||
excel_data = [[None]]
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
excel_data.append([None] + list(row))
|
||||
wb.close() # 释放文件句柄
|
||||
|
||||
# 3. 辅助读取函数 (替代慢速的 sheet.cell().value)
|
||||
def get_val(data: List[List[Any]], row: int, col: int) -> Any:
|
||||
try:
|
||||
if row < len(data) and col < len(data[row]):
|
||||
return data[row][col]
|
||||
except IndexError:
|
||||
pass
|
||||
return None
|
||||
````
|
||||
|
||||
### 第二步：识别报表区块
|
||||
|
||||
大多数报表式Excel有以下特征，按优先级在 `excel_data` 中进行检测：
|
||||
|
||||
|**特征**|**检测方法**|**示例**|
|
||||
|---|---|---|
|
||||
|区块标题|A列包含特定关键词|"请购单维护"、"订单"|
|
||||
|表头标签|冒号结尾的标签|"请购单号："、"日期："|
|
||||
|明细表头|包含列名的行|"行号"、"物料编码"、"数量"|
|
||||
|数据行|标签行下方连续的数据|非空的具体数据值|
|
||||
|页脚行|包含"制单"、"审批"等|"制单人："、"审批人："|
|
||||
|
||||
**识别规则**：
|
||||
|
||||
1. 区块开始：通常在A列，包含报表类型名称
|
||||
|
||||
2. 表头区域：区块开始后3-6行，包含带冒号的字段标签
|
||||
|
||||
3. 明细表头：表头区域后，A列包含"行号"或类似列名
|
||||
|
||||
4. 明细数据：明细表头后，连续的非空行
|
||||
|
||||
5. 页脚区域：明细数据后，包含签名/审批信息
|
||||
|
||||
|
||||
### 第三步：设计数据结构
|
||||
|
||||
根据识别结果，设计输出表结构：
|
||||
|
||||
**扁平化单表**
|
||||
|
||||
- 每条明细记录携带完整的表头信息
|
||||
|
||||
- 适合数据分析和导出
|
||||
|
||||
|
||||
### 第四步：生成转换脚本
|
||||
|
||||
基于分析结果，生成包含以下函数的Python脚本：
|
||||
|
||||
Python
|
||||
|
||||
```
|
||||
# 必需的核心函数
|
||||
def find_sections(excel_data)
|
||||
"""识别所有报表区块的起始行"""
|
||||
|
||||
def extract_header_data(excel_data, start_row)
|
||||
"""提取表头信息"""
|
||||
|
||||
def extract_line_items(excel_data, header_row, section_end)
|
||||
"""提取明细行数据"""
|
||||
|
||||
def extract_footer_data(excel_data, line_items_end)
|
||||
"""提取页脚信息"""
|
||||
|
||||
def parse_section(excel_data, start_row, next_section)
|
||||
"""解析单个报表区块"""
|
||||
|
||||
def parse_excel_file(file_path)
|
||||
"""读取Excel文件，转换为二维数组并解析所有区块"""
|
||||
|
||||
def save_to_excel(data_df, output_path)
|
||||
"""保存转换结果，利用Pandas向量化计算列宽"""
|
||||
```
|
||||
|
||||
## 字段提取模式
|
||||
|
||||
### 模式1：固定偏移量（利用预计算）
|
||||
|
||||
当表头字段位置固定时使用：
|
||||
|
||||
Python
|
||||
|
||||
```
|
||||
# 提取表头信息 (从内存数组快速读取)
|
||||
def extract_header_data(excel_data, start_row):
|
||||
header = {}
|
||||
header['请购单号'] = get_val(excel_data, start_row + 3, 2) # Row+3, B列(2)
|
||||
return header
|
||||
```
|
||||
|
||||
### 模式2：标签查找
|
||||
|
||||
当字段位置不固定但标签唯一时使用：
|
||||
|
||||
Python
|
||||
|
||||
```
|
||||
def find_field_by_label(excel_data, label, start_row, search_range=10):
|
||||
"""通过标签查找字段位置"""
|
||||
max_row = len(excel_data) - 1
|
||||
for row in range(start_row, min(start_row + search_range, max_row + 1)):
|
||||
# 假设标签在前10列中
|
||||
for col in range(1, min(11, len(excel_data[row]))):
|
||||
cell_value = get_val(excel_data, row, col)
|
||||
if cell_value and label in str(cell_value):
|
||||
# 返回值的位置（通常在标签的右侧）
|
||||
return get_val(excel_data, row, col + 1)
|
||||
return None
|
||||
```
|
||||
|
||||
## 明细行提取策略
|
||||
|
||||
### 策略：预计算列映射字典 (极速匹配)
|
||||
|
||||
避免在双重循环中调用 `column_index_from_string`。
|
||||
|
||||
Python
|
||||
|
||||
```
|
||||
from openpyxl.utils import column_index_from_string
|
||||
|
||||
LINE_ITEM_COLUMNS = {
|
||||
'A': '行号', 'B': '排产号', 'C': '物料编码'
|
||||
}
|
||||
|
||||
# 全局预计算列索引
|
||||
LINE_ITEM_COLUMNS_IDX = {
|
||||
column_index_from_string(col): field
|
||||
for col, field in LINE_ITEM_COLUMNS.items()
|
||||
}
|
||||
|
||||
def extract_line_items(excel_data, header_row, section_end):
|
||||
line_items = []
|
||||
for row in range(header_row + 1, section_end):
|
||||
# ... 判断跳出逻辑 ...
|
||||
item = {}
|
||||
for col_num, field_name in LINE_ITEM_COLUMNS_IDX.items():
|
||||
item[field_name] = get_val(excel_data, row, col_num)
|
||||
if any(item.values()):
|
||||
line_items.append(item)
|
||||
return line_items
|
||||
```
|
||||
|
||||
## 处理特殊情况
|
||||
|
||||
### 1. 空区块处理
|
||||
|
||||
当某个区块没有明细数据时，仍需创建一条记录：
|
||||
|
||||
Python
|
||||
|
||||
```
|
||||
if not line_items:
|
||||
# 创建一条空记录，保留表头和页脚信息
|
||||
record = {**header_data, **footer_data}
|
||||
for field in LINE_ITEM_COLUMNS.values():
|
||||
record[field] = None
|
||||
flat_records.append(record)
|
||||
```
|
||||
|
||||
## 输出格式
|
||||
|
||||
### Excel格式（结合 Pandas 向量化提速）
|
||||
|
||||
Python
|
||||
|
||||
```
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
class ReportParser:
|
||||
def __init__(self, worksheet):
|
||||
self.ws = worksheet
|
||||
self.max_row = worksheet.max_row
|
||||
self.max_col = worksheet.max_column
|
||||
def save_to_excel(data_df, output_path):
|
||||
"""保存为Excel文件并极速调整列宽"""
|
||||
with pd.ExcelWriter(output_path, engine='openpyxl') as writer:
|
||||
data_df.to_excel(writer, sheet_name='转换结果', index=False)
|
||||
worksheet = writer.sheets['转换结果']
|
||||
|
||||
def find_report_blocks(self):
|
||||
"""Identify start and end rows for each report block"""
|
||||
# Implement block detection logic
|
||||
# Look for report delimiters (e.g., title in column 1)
|
||||
# Exclude trailing empty rows
|
||||
pass
|
||||
|
||||
def parse_header(self, start_row):
|
||||
"""Extract header fields from the report header section"""
|
||||
# Map cell positions to field names
|
||||
pass
|
||||
|
||||
def parse_detail_rows(self, start_row, end_row):
|
||||
"""Extract detail rows from the report detail section"""
|
||||
# Identify column header row
|
||||
# Extract data until empty row or footer starts
|
||||
pass
|
||||
|
||||
def parse_footer(self, end_row):
|
||||
"""Extract footer fields from the report footer section"""
|
||||
# Map cell positions to field names
|
||||
pass
|
||||
|
||||
def parse_report(self, start_row, end_row):
|
||||
"""Parse complete report: header + details + footer"""
|
||||
header = self.parse_header(start_row)
|
||||
details = self.parse_detail_rows(start_row, end_row)
|
||||
footer = self.parse_footer(end_row)
|
||||
return header, details, footer
|
||||
|
||||
def convert_to_database_format(input_file, output_file, sheet_name=None):
|
||||
"""Convert report format to database-record format"""
|
||||
wb = load_workbook(input_file, data_only=True)
|
||||
ws = wb[sheet_name] if sheet_name else wb.active
|
||||
|
||||
parser = ReportParser(ws)
|
||||
blocks = parser.find_report_blocks()
|
||||
|
||||
# Collect all records
|
||||
all_records = []
|
||||
for start_row, end_row in blocks:
|
||||
header, details, footer = parser.parse_report(start_row, end_row)
|
||||
# Merge header + each detail row + footer
|
||||
for detail in details:
|
||||
record = {**header, **detail, **footer}
|
||||
all_records.append(record)
|
||||
|
||||
# Write output
|
||||
from openpyxl import Workbook
|
||||
wb_out = Workbook()
|
||||
ws_out = wb_out.active
|
||||
|
||||
# Write header row
|
||||
fields = list(all_records[0].keys())
|
||||
for col_idx, field_name in enumerate(fields, start=1):
|
||||
ws_out.cell(row=1, column=col_idx, value=field_name)
|
||||
|
||||
# Write data rows
|
||||
for row_idx, record in enumerate(all_records, start=2):
|
||||
for col_idx, field_name in enumerate(fields, start=1):
|
||||
ws_out.cell(row=row_idx, column=col_idx, value=record.get(field_name))
|
||||
|
||||
wb_out.save(output_file)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Convert report-style Excel to database-record format')
|
||||
parser.add_argument('input_file', help='Input Excel file')
|
||||
parser.add_argument('output_file', help='Output Excel file')
|
||||
parser.add_argument('--sheet', help='Worksheet name (default: active)')
|
||||
args = parser.parse_args()
|
||||
|
||||
convert_to_database_format(args.input_file, args.output_file, args.sheet)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
# 【提速】利用 Pandas 的向量化操作一次性算出最大列宽
|
||||
for idx, col in enumerate(data_df.columns):
|
||||
max_len = max(data_df[col].astype(str).map(len).max() if not data_df.empty else 0, len(str(col)))
|
||||
adjusted_width = min(max_len + 2, 50)
|
||||
col_letter = get_column_letter(idx + 1)
|
||||
worksheet.column_dimensions[col_letter].width = adjusted_width
|
||||
```
|
||||
|
||||
**Key Implementation Points:**
|
||||
## 依赖库
|
||||
|
||||
- **Column mapping**: Use fixed column indices (1-indexed) or use openpyxl's column letters
|
||||
- **Empty row detection**: Check if all cells in a row are None/empty to identify boundaries
|
||||
- **Date handling**: Excel dates may be numeric - use appropriate conversion if needed
|
||||
- **Merged cells**: Check for merged cells in header/footer sections
|
||||
- **Data types**: Preserve original data types (strings, numbers, dates) from source
|
||||
脚本需要以下依赖，确保在运行前安装：
|
||||
|
||||
### Step 3: Test and Verify
|
||||
|
||||
Run the generated script to convert the file:
|
||||
|
||||
```bash
|
||||
python3 scripts/custom_converter.py input.xlsx output.xlsx
|
||||
```
|
||||
|
||||
Use `excel-to-markdown` to verify the output:
|
||||
|
||||
```bash
|
||||
# For large files, only verify first 10 records
|
||||
python3 scripts/excel_to_markdown.py output.xlsx -o /tmp/verify.md --show-rows --show-cols --rows 1:11
|
||||
```
|
||||
|
||||
Read `/tmp/verify.md` and verify:
|
||||
1. All records are extracted (count should match total detail rows across all reports)
|
||||
2. Header fields are correctly populated for each record
|
||||
3. Detail fields are correctly mapped
|
||||
4. Footer fields are correctly populated
|
||||
5. No data loss or corruption
|
||||
|
||||
**Important:** For large output files with many records, use `--rows 1:11` to only convert and verify the first 10 data records (plus header row). This avoids processing time and memory issues when verifying large exports.
|
||||
|
||||
### Step 4: Iterate if Needed
|
||||
|
||||
If verification reveals issues:
|
||||
1. Identify the specific problem (wrong column index, missing field, incorrect row detection)
|
||||
2. Fix the script accordingly
|
||||
3. Re-run the conversion
|
||||
4. Re-verify with excel-to-markdown
|
||||
5. Repeat until output is correct
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Report Block Detection
|
||||
|
||||
Most report-style Excel files use one of these patterns:
|
||||
|
||||
**Pattern A: Title-based delimiters**
|
||||
```
|
||||
Row 1: Report Title
|
||||
Row 2: [header data]
|
||||
...
|
||||
Row N: [footer data]
|
||||
Row N+1: [empty or next report title]
|
||||
```
|
||||
|
||||
Look for a specific value in column 1 (e.g., "离散备料计划", "Purchase Order")
|
||||
|
||||
**Pattern B: Fixed-size reports**
|
||||
All reports have the same number of rows. Calculate block size and divide evenly.
|
||||
|
||||
### Header/Detail/Footer Separation
|
||||
|
||||
**Typical structure:**
|
||||
- Rows 1-4: Header information (labels and values in specific columns)
|
||||
- Row 5: Empty separator
|
||||
- Row 6: Detail table column headers
|
||||
- Rows 7+: Detail data rows
|
||||
- Row N-1: Footer row 1 (creator, date, approver)
|
||||
- Row N: Footer row 2 (printer, print date)
|
||||
- Row N+1: Empty separator or next report
|
||||
|
||||
### Column Mapping Strategies
|
||||
|
||||
**Strategy 1: Fixed column positions**
|
||||
Use when report format is consistent:
|
||||
```python
|
||||
factory = ws.cell(row=2, column=3).value # C列
|
||||
order_no = ws.cell(row=2, column=9).value # I列
|
||||
```
|
||||
|
||||
**Strategy 2: Search by label**
|
||||
Use when column positions may vary:
|
||||
```python
|
||||
# Find column by searching for label in first row
|
||||
for col in range(1, max_col + 1):
|
||||
if ws.cell(row=header_row, column=col).value == "Order No":
|
||||
order_no_col = col
|
||||
break
|
||||
```
|
||||
|
||||
### Field Merging Strategy
|
||||
|
||||
When creating database records:
|
||||
1. Parse header fields once per report
|
||||
2. Parse footer fields once per report
|
||||
3. For each detail row, create a merged record: `{**header, **detail_row, **footer}`
|
||||
4. This repeats header/footer fields for each detail line (database normalization)
|
||||
|
||||
## Example Report Structure Analysis
|
||||
|
||||
When analyzing the markdown output, look for:
|
||||
Bash
|
||||
|
||||
```
|
||||
Row 1: | Report Title | | | ...
|
||||
Row 2: | Field: | | Value | | Field: | | Value | ...
|
||||
Row 3: | Field: | | Value | | Field: | | Value | ...
|
||||
Row 4: | (empty or separator)
|
||||
Row 5: | Col1 | Col2 | Col3 | ... (detail table headers)
|
||||
Row 6: | val1 | val2 | val3 | ... (first detail row)
|
||||
Row 7: | val1 | val2 | val3 | ... (second detail row)
|
||||
...
|
||||
Row N: | Creator: | | Name | | Date: | | 2025-01-01 |
|
||||
Row N+1: | Printer: | | Name | | Print Date: | | 2025-01-02 |
|
||||
Row N+2: | (empty) or next report title
|
||||
pip install openpyxl pandas
|
||||
```
|
||||
|
||||
Map this structure to the script functions:
|
||||
- `parse_header(start_row)`: Extract fields from rows 2-3
|
||||
- `parse_detail_rows(start_row, end_row)`: Extract rows 6+ until empty/footer
|
||||
- `parse_footer(end_row)`: Extract fields from rows N to N+1
|
||||
## 验证清单
|
||||
|
||||
## Resources
|
||||
生成脚本后，验证以下内容：
|
||||
|
||||
The `excel-to-markdown` skill provides Excel-to-markdown conversion for structure analysis.
|
||||
- [ ] 成功识别所有报表区块
|
||||
- [ ] 表头字段提取正确
|
||||
- [ ] 明细行数据完整
|
||||
- [ ] 页脚信息准确
|
||||
- [ ] 空区块得到正确处理
|
||||
- [ ] 输出文件格式正确
|
||||
- [ ] 数据类型准确（日期、数字等）
|
||||
- [ ] 没有重复或遗漏的记录
|
||||
|
||||
This skill does not include bundled scripts or references - each conversion script is generated dynamically based on the specific report format being analyzed.
|
||||
## 调试技巧
|
||||
|
||||
当脚本出现问题时：
|
||||
|
||||
1. **打印中间结果**：在每个函数中添加print语句，查看提取的数据
|
||||
2. **检查单元格值**：确认openpyxl读取的值与预期一致
|
||||
3. **验证索引**：确保行号和列号计算正确
|
||||
4. **分步测试**：先测试单个区块，确认正确后再处理全部
|
||||
5. **对比原文件**：在Excel中查看原始数据和提取结果的差异
|
||||
|
||||
## 常见问题
|
||||
|
||||
**Q: 为什么提取时有些序列号或公式计算的值变成了 `None`？**
|
||||
|
||||
A: 这是因为在 `openpyxl` 中错误开启了 `read_only=True` 模式，导致依靠 Excel 公式生成的值无法正确读取缓存。**解决方案**：去掉 `read_only=True`，只保留 `data_only=True`，并使用二维数组提取法来保障速度。
|
||||
|
||||
**Q: 输出的数据需要进一步处理怎么办？**
|
||||
|
||||
A: 脚本生成后，可以在将其转换为 Pandas DataFrame 之后，利用 Pandas 强大的生态添加数据清洗、验证、格式转换（如日期格式化）等功能。
|
||||
|
||||
## 最佳实践
|
||||
|
||||
1. **绝对优先使用内存二维数组**：直接摒弃 `sheet.cell().value` 的传统思维，这是报表转换脚本能商用的性能基石。
|
||||
|
||||
2. **预先计算，拒绝重复**：所有能够确定位置或索引关系的映射字典，全部放在全局作用域一次性计算完成。
|
||||
|
||||
3. **先分析，后编码**：花时间理解报表结构和分页/分块标识，比直接编码更高效。
|
||||
|
||||
4. **异常容错机制**：数据提取行要进行 `if any(item.values())` 判断，过滤纯空行；对于越界索引使用 `try-except` 包裹。
|
||||
@@ -11,6 +11,7 @@ description: Gitea repository management via REST API. Use when user explicitly
|
||||
- **API Base**: `https://gitea.server10086.icu/api/v1`
|
||||
- **Default Account**: `admin`
|
||||
- **Auth Header**: `Authorization: token $GITEA_TOKEN`
|
||||
- **Git Remote URL**: `git@gitea.server10086.icu:admin/repo-name.git` (SSH)
|
||||
|
||||
**Always verify GITEA_TOKEN exists before any operation:**
|
||||
```bash
|
||||
@@ -66,7 +67,7 @@ curl -X POST "https://gitea.server10086.icu/api/v1/admin/repos" \
|
||||
git init
|
||||
git add .
|
||||
git commit -m "Initial commit"
|
||||
git remote add origin "https://gitea.server10086.icu/admin/$REPO_NAME.git"
|
||||
git remote add origin "git@gitea.server10086.icu:admin/$REPO_NAME.git"
|
||||
git push -u origin main
|
||||
```
|
||||
|
||||
|
||||
93
skills/meeting-minutes/SKILL.md
Normal file
93
skills/meeting-minutes/SKILL.md
Normal file
@@ -0,0 +1,93 @@
|
||||
---
|
||||
name: meeting-minutes
|
||||
description: Transform raw audio-transcription documents into clean, structured meeting minutes. Use this skill whenever the user provides a meeting transcript — any file with timestamps and speaker labels (说话人1 / Speaker 1), ASR/speech-to-text output, 转写结果, 录音转写 — and asks to organize, summarize, or 整理 it into 会议记录 / 会议纪要 / meeting minutes / action items. Also trigger when the user uploads a transcription of a meeting, review, training session, interview, or discussion and just says "整理一下" or "帮我总结", even without the word "会议记录". Do NOT use for writing minutes from scratch (no transcript) or for translating transcripts.
|
||||
---
|
||||
|
||||
# 录音转写 → 会议纪要
|
||||
|
||||
把口语化、带噪声的录音转写文档，整理成一份**详尽但不啰嗦**的正式会议纪要：所有实质信息（决议、数字、责任人、时限、分歧、疑点）一条不丢，所有口语噪声（重复、口头禅、寒暄、跑题碎语）一句不留。
|
||||
|
||||
成品的衡量标准：一个没参会的人读完纪要，能准确知道**会上定了什么、为什么这么定、还有什么没定、接下来谁在什么时间要做什么**。
|
||||
|
||||
## 工作流程
|
||||
|
||||
### 第 1 步：完整读取转写文件
|
||||
|
||||
**必须读完全文才能动笔。** 会议的决议常常在中途被推翻重定（先定方案 A，结尾改成 A+B 并存），只读开头或抽样阅读会把中间结论当成最终决议，这是此类任务最严重的错误。
|
||||
|
||||
- 先 `wc -c` / `wc -l` 探明文件大小，再按**行**分块读取（如 `sed -n '1,300p'`），逐块读完全部内容。
|
||||
- 中文转写是多字节 UTF-8，**不要用 `head -c` 等按字节截断的方式读取**，会切断字符导致输出乱码报错；始终按行读取。
|
||||
- 文件可能是 CRLF 行尾，属正常现象，无需处理。
|
||||
|
||||
### 第 2 步：边读边积累四类素材
|
||||
|
||||
读的过程中持续记录，读完即可直接成文：
|
||||
|
||||
1. **元数据**：源文件名、音频时长、说话人数量、分句数量（转写文件头部通常自带）。
|
||||
2. **同音误写词表**：见第 3 步。
|
||||
3. **说话人身份线索**：见第 4 步。
|
||||
4. **内容骨架**：议题切换点、每个议题下的讨论脉络、当场形成的决议（含被推翻的中间结论）、提到的待办（谁、做什么、什么时限）、被点名的人和精确数字。
|
||||
|
||||
### 第 3 步：同音误写校正（中文转写的核心难点）
|
||||
|
||||
方言口音和专业术语会让 ASR 产生大量同音/近音误写，且**同一个词在全文中会被写成多种错法**。典型规律：
|
||||
|
||||
- 专业缩写被写成日常词：BOM → "报幕/保姆/报模/爆品/泡沫/放牧"；VLOOKUP → "维鲁卡普"
|
||||
- 行业术语被写成同音常用词：接头→"截图"、螺纹→"论文"、量程→"量产/量成/量层"、选型→"血型/雪凝/选曲"、模板→"木板"、径向/轴向→"镜像/进项/主项"
|
||||
- 人名时对时错："郑工"被写成"正宫/正工"
|
||||
|
||||
处理规则：
|
||||
|
||||
- 同一上下文反复出现、读不通的词，按行业语境推断真实词义，**全文统一校正**。判断依据是上下文自洽：如果把"报幕"读成 BOM 后全文每一处都通顺，即可确认。
|
||||
- 在纪要开头用一段引述（blockquote）**集中声明校正对照**（"报幕/保姆"实为 BOM 等），让读者知道整理者做过什么，也便于核对。
|
||||
- **无法确认的专有名词（人名、系统名、工具名）不要硬猜**：模糊处理或标注"（名称待确认）"，并在交付时提醒用户核对。宁可标注疑点，不可编造确定性。
|
||||
|
||||
### 第 4 步：说话人身份归属（谨慎原则）
|
||||
|
||||
- 只依据**转写内部证据**归属身份：A 反复称呼 B 为"郑工"，则 B 是郑工；某人说"刘凯还有 7 个没做"且另一人应答"对，都快了"，则应答者**疑似**刘凯。
|
||||
- 证据充分的写实名（如：说话人 1，会中被称"郑工"）；证据单薄的加"疑似"；没有证据的保留"说话人 N"编号。
|
||||
- 在基本信息或正文中自然带出主持人、主讲人等**角色**（谁在部署工作、谁在演示、谁在记录待办），角色比名字更重要。
|
||||
- 会中被提及但未必在场的人名（被点名负责某事的人）照实写入待办事项。
|
||||
|
||||
### 第 5 步：按模板组织成文
|
||||
|
||||
整体结构和逐节写法**严格参照 `references/format-template.md`**（必读），成文前对照 `references/example.md` 中的真实范例校准颗粒度和语感。骨架为：
|
||||
|
||||
1. 标题：`# 会议纪要：<一句话概括的会议主题>`
|
||||
2. 会议基本信息（表格）+ 转写质量校正声明（blockquote）
|
||||
3. 正文若干部分：**按会议实际脉络划分议题章节**，不是按时间流水账，也不是套死固定章节数
|
||||
4. 会议总结与决议汇总（编号列表，关键决议加粗）
|
||||
5. 待办事项（表格：# / 事项 / 责任人 / 时限或备注）
|
||||
|
||||
### 第 6 步：交付
|
||||
|
||||
- 输出为 Markdown 文件，命名 `会议纪要_<会议日期>_<一句话概括的会议主题>.md`（日期取自源文件名或转写内容），保存到工作目录或用户指定位置，日期格式：`yyyy-MM-dd`。
|
||||
- 聊天回复保持简短：两三句概括会议规模与核心议题，**明确提醒用户核对**同音校正后的人名与专业术语，邀请反馈修订。不要在聊天里复述纪要内容。
|
||||
|
||||
## 风格红线（"详尽但不啰嗦"的具体含义）
|
||||
|
||||
**必须保留（详尽）：**
|
||||
|
||||
- 每一条实质决议、原则、要求，以及它的**理由**（"必须 1:1 配置油品，否则账实不符"——理由让纪要可信、可执行）
|
||||
- 精确数字、数量、时限、责任人（"刘凯尚余 7 个""下月初至中旬高勇检查"）
|
||||
- **决议的演变过程**：核心争论先呈现各方案的优缺点（适合用对比表格），再写表决与最终决议；会中先定后改的，最终决议为准、但要写明"经讨论改为"
|
||||
- 分歧与遗留问题：谁提出了什么顾虑、当场未解决的问题、"会后研究"的事项
|
||||
- 提出的疑点、特殊情况、边界条件（"径向件可代用于轴向，反之不可"）
|
||||
|
||||
**必须删除（不啰嗦）：**
|
||||
|
||||
- 全部口语噪声：语气词、重复、口吃式赘语（"那个那个""都都都"）、寒暄、应答碎语（"嗯""对头""好"）
|
||||
- 操作演示中的现场琐碎（"你打开了没""我看下是哪个文件"），只留演示所传达的**功能逻辑和操作流程**
|
||||
- 与议题无关的插话和跑题
|
||||
|
||||
**叙述纪律：**
|
||||
|
||||
- 严格区分四种性质并用措辞体现：**已形成决议** / **讨论中的观点** / **个人建议**（"主持人建议……"）/ **遗留待研究**
|
||||
- 只写转写中有的内容，不补充转写外的背景知识，不替会议做它没做的结论
|
||||
- 加粗只用于关键决议、关键约束、关键数字，宁少勿滥
|
||||
- 用转写的原语言写作（中文转写→中文纪要）；正文以陈述句为主，结构化信息（基本信息、方案对比、待办）用表格
|
||||
|
||||
## 参考文件
|
||||
|
||||
- `references/format-template.md` — 逐节模板与写法要点，**成文前必读**
|
||||
- `references/example.md` — 一份完整真实范例（制造业 BOM 工作会），用于校准详略与语感
|
||||
148
skills/meeting-minutes/references/example.md
Normal file
148
skills/meeting-minutes/references/example.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# 完整范例：制造业 BOM 工作会纪要
|
||||
|
||||
> 本文件是一份真实转写（83 分钟、9 名说话人、1124 条分句、川渝口音、大量同音误写）整理出的成品纪要，是本 skill 风格的基准。注意学习它的：议题章节如何跟随会议脉络、争论如何用对比表呈现、决议演变如何表述（先定单一方案、后改为两案并存）、待办如何落到人与时限、疑点如何标注。
|
||||
|
||||
---
|
||||
|
||||
# 会议纪要：BOM 维护工作部署暨"BOM 新增物料小工具"培训会
|
||||
|
||||
## 一、会议基本信息
|
||||
|
||||
| 项目 | 内容 |
|
||||
| --- | --- |
|
||||
| 会议时间 | 2026 年 6 月 3 日 15:45 起，时长约 83 分钟 |
|
||||
| 记录来源 | 会议录音转写（源文件：2026年06月03日 15点45分.mp3，共 1124 条分句） |
|
||||
| 参会人数 | 9 人（转写标记为说话人 1–9） |
|
||||
| 主持人 | 说话人 1（会中被称"郑工"，负责工作部署与总结） |
|
||||
| 主讲/演示 | 说话人 4（信息化/小工具开发人员，负责工具演示与改造） |
|
||||
| 会议主题 | 现场 BOM 维护工作要求部署；信息化"BOM 新增物料小工具"培训、问题讨论与改进决议 |
|
||||
|
||||
> 说明：原始转写因方言口音存在大量同音误写（如"报幕/保姆/爆品"实为 **BOM**、"截图"实为**接头**、"论文"实为**螺纹**、"血型/雪凝"实为**选型**、"木板"实为**模板**、"量产/量成"实为**量程**、"维鲁卡普"实为 **VLOOKUP** 等），本纪要已按上下文统一校正。个别人名、工具名称无法完全确认处已作模糊处理。
|
||||
|
||||
---
|
||||
|
||||
## 二、第一部分：BOM 维护工作部署与要求（主持人）
|
||||
|
||||
1. **弹性元件 BOM 优先完成**：当前优选工作中，弹性元件部分必须首先做完、做扎实，不能马虎，否则会造成很大问题。接头部分目前配不完，可随时间推移在后续逐步补充更新。
|
||||
2. **线下 BOM（物料表）必须建立**，其用途包括：
|
||||
- 生产现场遇到物料缺失、错误或不便使用现场 BOM 时，依靠线下 BOM 修改订单计划、备料计划；
|
||||
- 生产订单上没有挂靠 BOM 的，需通过线下 BOM 导入采购（购买）计划，便于生产端快速处理；
|
||||
- 线下 BOM 灵活、约束少，简单处理即可快速供生产使用，因此本版必须做对、做准确。
|
||||
3. **线上 BOM 完成进度与时限**：刘凯处尚余 7 个（已接近完成），小左已完成，主持人处尚余 20 余个。未完成人员须尽快完成；预计下月初至中旬高勇方面将进行检查复核。
|
||||
4. **线上 BOM 编制原则**：
|
||||
- 必须把**指针**和**包装物**加入 BOM；
|
||||
- 落实核对物料，检查有无错漏；
|
||||
- 耐震表油品必须按 **1:1** 配置，不得随意抛减用量，否则会造成账上无数、实物却存在的账实不符问题，编制时务必仔细检查。
|
||||
5. **配料统一原则**：做现场 BOM 时，所用**弹簧管物料必须配齐**；若不配齐，后续更新线上 BOM 的节拍会非常频繁。
|
||||
6. **BOM 数据与选型数据必须一一匹配**：选型端有的，BOM 端必须有；宁可 BOM 数据范围做大，**选型数据不能大于 BOM 数据**，否则会出现选型选得出、BOM 推不出（缺料）的情况。
|
||||
7. **责任清单**：每人维护的现场 BOM 需建立清单，生产端发现问题时可据此指定责任人处理。
|
||||
|
||||
---
|
||||
|
||||
## 三、第二部分："BOM 新增物料小工具"培训与演示（说话人 4 主讲）
|
||||
|
||||
### 3.1 工具定位与基本操作
|
||||
|
||||
- 工具面向**现场 BOM 维护人员**，包含两大功能：① 新增**过程连接（接头）**物料；② 新增**量程范围（弹性元件）**物料。
|
||||
- 操作流程：打开文件 → **加载本人负责的型号** → 选择处理类型（过程连接 / 量程范围）→ 填入规格值 → 点击"数据处理" → 核对结果 → 点击"提交"，数据即写入 BOM 并保存。
|
||||
- 结果界面以颜色区分：**绿色 = 本次新增物料，蓝色 = 所参照的模板，黄色 = 发生变化的数据**；提交后行号自动全部重排。
|
||||
- 现场以 YBF 型号为例演示了新增 M14 过程连接（系统按径向低压/径向高压/轴向低压/轴向高压自动生成四类接头），并实际提交了一条测试数据验证写入效果。
|
||||
|
||||
### 3.2 模板机制（现行逻辑）
|
||||
|
||||
- 系统将该型号下与接头相关的物料自动提取并分类，**每一类型默认取最后一条物料作为模板**，复制其全部选择条件，仅将过程连接规格替换为新值（与人工新增逻辑一致）。
|
||||
- 自动判断的模板不一定正确，用户可通过"是/否"手动改选正确模板；**每个型号只需甄别一次**，确认后系统记忆，后续不再提示。
|
||||
- 两件式表只需新增**紧固接头**：首次处理时将不需要的类别置"否"即可，之后不再出现；如确需新增其他类别，再手动开启。
|
||||
|
||||
### 3.3 培训中提出的问题与讨论
|
||||
|
||||
1. **新增量程须先查映射**：C 型管加 U 型管在选型表中共有 44 个标准量程。新增量程前必须先与**量程映射表**核对——若已有对应/可替代关系则不得新增，应通过映射替代；确无对应关系方可加入选型，同时必须通知技术中心同步录入（映射关系由技术中心管理），否则选型仍然出不来。映射表由小朱提供给说话人 4，工具中将加入自动核对功能。
|
||||
2. **接头识别逻辑**：按物料名称含"接头"判断不可靠（存在命名不叫接头的接头、紧固接头与普通接头并存等情况），讨论认为按**选择条件中含过程连接（结合量程范围）**判断更准确；但包装盒等也带过程连接（分大小螺纹），两件式情形需做特殊处理。
|
||||
3. **下拉框体验**：加载型号后的下拉清单显示条数太少，维护型号多的人操作不便。要求将下拉框加长/加大或改为旁侧清单展示，说话人 4 承诺优化。
|
||||
4. **查重功能**：演示中发现已存在的规格（如 M14、M20F）再次新增时系统未报错。确定**螺纹规格与量程规格新增时均须增加查重校验**，已有则提示、不得重复添加，避免 BOM 推出重复物料。
|
||||
5. **测试数据清理**：工具早期实验写入的数据已污染演示表，说话人 4 当场承诺重新复制一份干净数据，活动结束后各自的旧数据也要收掉。
|
||||
|
||||
---
|
||||
|
||||
## 四、第三部分：接头选择条件生成方式之争（核心议题）
|
||||
|
||||
### 4.1 两条技术路线
|
||||
|
||||
| | 路线一：模板模式（现行） | 路线二：弹性元件模式（郑工方案） |
|
||||
| --- | --- | --- |
|
||||
| 原理 | 参照既有物料模板复制选择条件 | 不依赖模板；为接头指定弹簧管类型（C 型管/螺旋管），自动读取 BOM 页中对应弹性元件的全部选择条件（量程范围等）融合生成 |
|
||||
| 优点 | 前期轻松、马上可用 | 后期一劳永逸，无需逐条人工甄别"哪个模板量程最全" |
|
||||
| 缺点 | 每个型号首次都要人工甄别模板，型号多、条件多时"看不全、判不准"，长期工作量大 | 前期基础数据准备痛苦；若 BOM 页弹性元件量程本身做少了（如标准 20 个只做了 18 个），会跟着做错；且界面只呈现最终结果，看不到全部接头明细 |
|
||||
|
||||
### 4.2 配套问题讨论
|
||||
|
||||
- **选型约束归属**：采用路线二意味着 BOM 数据做大，必须依靠选型端（平台配置方法参数值表中的约束条件）收口。会议统一思想：**选型数据收紧约束、BOM 数据放大**——BOM 端不管"配不配得起"，能否下单由选型约束决定，选得出即推得出。
|
||||
- 现已出现绕过选型乱选的问题（如 1.6 级被直接选成 2.5 级），虽 BOM 推出的料是对的，但产品做不出来，更说明选型端约束必须维护起来；目前各人已陆续在选型中补加约束。
|
||||
- 螺纹与量程的匹配约束（某螺纹可用到多少压力以下）此前普遍未做。难点在选型页量程单位多达五六种，人工换算工作量大。说话人 4 提出后续配套：建立**量程代码—实际值对照表**并统一单位（如统一换算为 MPa），再做小工具按范围自动筛出符合的量程代码、生成约束条件；同时提供**BOM 与选型匹配性、量程完整性验证工具**。
|
||||
- **径向/轴向问题（说话人 3 提出）**：部分物料只有径向、没有轴向（径向件可代用于轴向、仅偏长，反之不可），库房现状即如此，需补数据；常规 C 型管量程是全的，特殊量程"以现有为准"。决议：在弹簧管**物料名称中加入识别标识**，分三类——不分径轴向的统一叫"弹簧管"，特殊的分别叫"径向弹簧管""轴向弹簧管"，正常不分的保持原样。
|
||||
- 对于按安装/设计形式（如 A0/B0 流程代码）约束管子的既有做法，存在同类型号用管不同、仅靠名称分径轴向区分不开（还有一体式/两体式之别）的特殊情况。
|
||||
|
||||
### 4.3 决议
|
||||
|
||||
经举手表决并综合讨论，**两种模式同时保留、由用户在工具中自行点选**：常规情形推荐采用弹性元件（弹簧管）模式以求长效；特殊型号（径轴向不全等）继续可用模板模式，不强求统一，"目的是把新物料加进去，而不是为了用某个工具"。
|
||||
|
||||
---
|
||||
|
||||
## 五、第四部分：量程范围新增功能讨论
|
||||
|
||||
1. 量程范围新增的处理分两部分，其中弹性元件物料的新增逻辑与过程连接类似；前提是**量程对照表必须先行规定完善**。
|
||||
2. **管子类型改为手动指定**：同一量程在不同型号下管型不同（如电接点型号 4 MPa 用 C 型管、常规 4 MPa 用螺旋管），自动判断不可靠，恢复早期版本的手动指定方式。
|
||||
3. **高压/低压同样改为手动指定**。现行自动逻辑为：统计该物料选择条件内各量程落在高/低压区间的"得分"，多者胜出（不按名称判断，可避开"中压接头"等命名问题；机芯尤其依赖此逻辑，因机芯名称分不出高低压且量程跨高低压两区）。判断错误时用户可手动改正并刷新模板，且只需选择一次。考虑到特殊情况（如部分订单指定用低压机芯出高压数据，需用户自行选择），最终仍决定以手动指定为准。
|
||||
4. **输入框拆分**：现版本中过程连接规格与量程规格共用同一输入单元格、靠类型选择区分；决定改为**两个独立输入框**，各管各的类型，程序上无实现障碍。
|
||||
5. **组合接头处理原则（主持人要求）**：类似 VCR 这种一个接头带三四个小零件的，今后新增时尽量做成**部件**——在 BIP 中将部件设为虚项、搭好部件 BOM，推出来即为散件，报表中以部件形式存在，避免逐零件维护的麻烦。
|
||||
|
||||
---
|
||||
|
||||
## 六、第五部分：线下 BOM 制作方法与经验分享（主持人）
|
||||
|
||||
- 线下 BOM 已基本做好，将发给大家参考；其制作远比线上 BOM 简单，工作量不大。
|
||||
- **一表多型号**：只要弹性元件（及机芯）通用，多个型号可合在一张表内做（示例：YT7-100 一张表覆盖 531、541、631、533、B0533、BT533、ZT521 及特殊单位、深圳地铁等型号），在同一表内为各型号划分独立区域并各自约束即可。
|
||||
- **关键技能与规则**：
|
||||
- 熟练使用 **VLOOKUP** 查询函数；通过增加辅助列等手段保证查询条件**唯一**，结果才正确；
|
||||
- 量程配接头建议采用简洁形式（参照主持人做法，避免过于复杂的搭法），便于后续取条件；
|
||||
- 设置**低压/高压分界线**，相同接头段可直接引用公式（如 MAX 段接头与 M01 接头一致时公式直接等于对应单元格），减少重复维护；
|
||||
- 注意**按量程段拆分**：如 10–60 MPa 段管坯用 F5 材料、100–250 MPa 段用 F7，理论上接头亦应随段不同而区分，目前虽未细分，编表时务必分段处理，不能混在一段；
|
||||
- 电接点型号因 10/16/25 MPa 用细管、与其他管不同，须单独建表，不能并入通用表；
|
||||
- **公用物料放变量库，专用物料做进各自报表**（如各型号配套的壳、罩、玻璃等在变量库中填写完整），读取结果即可保证正确，且便于维护；
|
||||
- YXH 之所以单独拆出，是因其 C 型管制造工艺及机芯与通用型号不同。
|
||||
- 第二版 BOM 的更新节拍理论上不应很多；若更新仍然频繁，应反思是否弹簧管等基础物料没有配全。
|
||||
|
||||
---
|
||||
|
||||
## 七、会议总结与决议汇总
|
||||
|
||||
主持人总结：本次会议历时约两小时，完成了信息化"BOM 新增物料小工具"（含新增接头物料、新增弹簧管物料两大功能）的培训，并形成以下整改决议：
|
||||
|
||||
1. 接头物料新增**增加"以 BOM 页弹性元件作为配用条件"的新模式**，同时**保留点选模板模式**，两种模式由用户自行选择；
|
||||
2. **螺纹规格新增增加查重功能**（已有则不再添加，防止 BOM 推出重复螺纹）；
|
||||
3. **量程范围新增同样增加查重功能**；
|
||||
4. 量程范围与过程连接的**输入框改为两个独立输入框**；
|
||||
5. **管子类型（C 型管/螺旋管）改为手动指定**；
|
||||
6. **高压/低压属性改为手动指定**；
|
||||
7. 研究在工具页面**增加一键打开导入上传小程序（exe）的按钮**（VBA 可操作 exe，此前已有实验基础，评估可实现）；
|
||||
8. 弹簧管物料名称增加**径向/轴向标识**（弹簧管 / 径向弹簧管 / 轴向弹簧管三类）；
|
||||
9. 选型端约束条件持续维护收紧，BOM 端放大；信息化后续配套量程对照、约束生成及 **BOM—选型匹配验证工具**。
|
||||
|
||||
---
|
||||
|
||||
## 八、待办事项
|
||||
|
||||
| # | 事项 | 责任人 | 时限/备注 |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | 按第七节决议 1–6 完成小工具改造并交付 | 说话人 4（信息化） | **本周内** |
|
||||
| 2 | 研究"一键打开导入上传小程序"按钮的可行性并实现 | 说话人 4 | 会后研究 |
|
||||
| 3 | 重新复制干净的演示/基础数据，清理早期实验数据 | 说话人 4 | 已当场处理，会后复核 |
|
||||
| 4 | 将量程映射关系表发给说话人 4，用于工具自动核对 | 小朱 | 尽快 |
|
||||
| 5 | 完成各自剩余线上 BOM（刘凯 7 个、主持人 20 余个等） | 相关维护人员 | 下月初至中旬前（高勇检查节点前） |
|
||||
| 6 | 线上 BOM 补充指针、包装物，核对物料，耐震表油品按 1:1 配置 | 全体 BOM 维护人员 | 编制过程中执行 |
|
||||
| 7 | 弹簧管名称按三类标识补充径向/轴向；径轴向缺失物料补数据 | 相关维护人员 | 持续 |
|
||||
| 8 | 新增量程前对照映射表核查，确需新增的同步通知技术中心录入 | 各维护人员/技术中心 | 长期规则 |
|
||||
| 9 | 选型端约束条件（含螺纹—量程匹配）持续补充维护 | 各维护人员 | 长期 |
|
||||
| 10 | 工具下发后投入使用，统计使用前后耗时、量化提效成果 | 全体使用人员 | 持续；成果用于向北京等单位推广 |
|
||||
| 11 | 建立各人现场 BOM 责任清单 | 全体维护人员 | 尽快 |
|
||||
| 12 | 将线下 BOM 样表发给大家，各自参照制作线下 BOM | 主持人 / 全体 | 尽快 |
|
||||
105
skills/meeting-minutes/references/format-template.md
Normal file
105
skills/meeting-minutes/references/format-template.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# 会议纪要逐节模板与写法要点
|
||||
|
||||
以下骨架中，第 2、4、5 节为固定结构；第 3 节（正文）的章节数量和标题完全跟随会议实际内容。占位符用 `<>` 表示。
|
||||
|
||||
---
|
||||
|
||||
## 1. 标题
|
||||
|
||||
```markdown
|
||||
# 会议纪要：<会议主题，一句话，体现"做了什么事">
|
||||
```
|
||||
|
||||
主题要具体：写"BOM 维护工作部署暨'BOM 新增物料小工具'培训会"，不写"工作会议"。若会议有两条主线（如"部署 + 培训"），标题用"暨"连接。
|
||||
|
||||
## 2. 会议基本信息 + 转写质量声明
|
||||
|
||||
```markdown
|
||||
## 一、会议基本信息
|
||||
|
||||
| 项目 | 内容 |
|
||||
| --- | --- |
|
||||
| 会议时间 | <日期 + 开始时间>，时长约 <N> 分钟 |
|
||||
| 记录来源 | 会议录音转写（源文件：<文件名>，共 <N> 条分句） |
|
||||
| 参会人数 | <N> 人（转写标记为说话人 1–N） |
|
||||
| 主持人 | 说话人 <N>（<身份证据，如：会中被称"郑工"，负责工作部署与总结>） |
|
||||
| 主讲/演示 | 说话人 <N>（<角色描述>） |
|
||||
| 会议主题 | <一到两句话> |
|
||||
|
||||
> 说明：原始转写因<方言口音/专业术语>存在大量同音误写（如"<误写1/误写2>"实为 **<正确词>**、……），本纪要已按上下文统一校正。个别人名、工具名称无法完全确认处已作模糊处理。
|
||||
```
|
||||
|
||||
要点：
|
||||
|
||||
- 表格行可按实际情况增删（没有明显主讲人就删掉该行）。
|
||||
- 校正声明列举**最高频的 5–8 组**误写即可，不必穷举；这段话同时起到免责和提示核对的作用。
|
||||
|
||||
## 3. 正文：按会议脉络分部分
|
||||
|
||||
```markdown
|
||||
## 二、第一部分：<议题名>（<主导者角色>）
|
||||
|
||||
## 三、第二部分：<议题名>（<主导者> 主讲）
|
||||
|
||||
### 3.1 <子议题>
|
||||
### 3.2 <子议题>
|
||||
...
|
||||
```
|
||||
|
||||
划分原则与写法：
|
||||
|
||||
- **跟随会议的真实脉络**：开场部署 → 培训演示 → 争论 → 决议 → 经验分享……有几段写几段，常见 3–6 个部分。每部分标题点明议题，括号注明主导者。
|
||||
- 部署/要求类内容：用编号列表，一条一个要求，**要求 + 理由**写在同一条里。
|
||||
- 培训/演示类内容：分"定位与基本操作 / 机制说明 / 提出的问题与讨论"等小节；操作流程压缩成一条箭头链（打开 → 加载型号 → 填规格 → 数据处理 → 提交）。
|
||||
- **核心争论单独成章**，内部结构固定为三段：
|
||||
1. 各方案对比表（行：原理 / 优点 / 缺点；列：各方案，并注明提出人）
|
||||
2. 配套问题讨论（编号或加粗短语开头的列表）
|
||||
3. `### 决议`（明确写出表决方式和最终结论；若结论是折中或两案并存，写清楚适用边界）
|
||||
- 经验分享类内容：提炼成可执行的规则清单（"关键技能与规则"），保留具体例子作括号内示例。
|
||||
- 每个实质观点尽量带上下文中的**具体例子**（"如 1.6 级被直接选成 2.5 级"），例子是纪要可读性的来源。
|
||||
|
||||
## 4. 会议总结与决议汇总
|
||||
|
||||
```markdown
|
||||
## <N>、会议总结与决议汇总
|
||||
|
||||
<主持人总结时怎么说就怎么概括>：本次会议历时约 <X>，完成了 <主线一>，并形成以下整改决议：
|
||||
|
||||
1. <决议一，关键词加粗>；
|
||||
2. <决议二>；
|
||||
...
|
||||
```
|
||||
|
||||
要点：
|
||||
|
||||
- 如果主持人会末做了口头总结，**以他的总结为骨架**，再用全文信息补全他漏掉的决议。
|
||||
- 每条决议自包含：读这一条就知道改什么、改成什么样，不需要回看正文。
|
||||
|
||||
## 5. 待办事项
|
||||
|
||||
```markdown
|
||||
## <N>、待办事项
|
||||
|
||||
| # | 事项 | 责任人 | 时限/备注 |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | <事项> | <人名/角色> | <明确时限，或"持续""长期规则""会后研究"> |
|
||||
```
|
||||
|
||||
要点：
|
||||
|
||||
- **逐条扫一遍全文**收集待办，不止收集结尾总结里的——会议中途随口指派的事项（"到时候喊小朱发一份给你"）最容易丢。
|
||||
- 责任人写转写中能确定的最具体指称（人名 > 角色 > "全体维护人员"）。
|
||||
- 时限没有明说就写性质（"尽快""持续；成果用于推广"），不要编造日期。
|
||||
- 待办按性质排序：一次性交付（带明确期限的在前）→ 规则性/长期事项 → 全员事项。
|
||||
|
||||
---
|
||||
|
||||
## 颗粒度自检
|
||||
|
||||
成文后通读一遍，逐项检查：
|
||||
|
||||
- [ ] 没参会的人能否复述出每条最终决议及其理由？
|
||||
- [ ] 会中被推翻的中间结论，是否已写成"经讨论改为"，而不是被当成最终决议？
|
||||
- [ ] 所有数字、人名、时限是否与转写一致？
|
||||
- [ ] 还能找到一句删掉也不损失信息的话吗？能找到就删。
|
||||
- [ ] 所有"疑似/待确认"标注是否如实保留？
|
||||
150
skills/mermaid-fixer/SKILL.md
Normal file
150
skills/mermaid-fixer/SKILL.md
Normal file
@@ -0,0 +1,150 @@
|
||||
---
|
||||
name: mermaid-fixer
|
||||
description: Fix Mermaid diagram syntax errors in Markdown files. Use when: (1) Mermaid diagrams aren't rendering or show parse errors, (2) User mentions "mermaid syntax error", "mermaid not working", "diagram broken", (3) CI/CD fails on Mermaid validation, (4) Documentation contains Mermaid code blocks that need fixing.
|
||||
---
|
||||
|
||||
# Mermaid Diagram Fixer
|
||||
|
||||
Automatically detects and fixes Mermaid diagram syntax errors in Markdown files using the `check-mermaid.js` validation script.
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Run `check-mermaid.js`** to get detailed error reports
|
||||
2. **Parse error information** (line numbers, error types, code snippets)
|
||||
3. **Apply intelligent fixes** based on the specific error
|
||||
4. **Verify** by running the check again
|
||||
|
||||
## Core Script
|
||||
|
||||
The `scripts/check-mermaid.js` script is the heart of this skill. It:
|
||||
- Scans Markdown files for ```` ```mermaid ```` code blocks
|
||||
- Validates each block using the Mermaid CLI (`mmdc`)
|
||||
- Returns detailed error reports with line numbers and error messages
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Run the Check Script
|
||||
|
||||
```bash
|
||||
node scripts/check-mermaid.js <path/to/file.md>
|
||||
```
|
||||
|
||||
**Expected output format:**
|
||||
|
||||
- **No errors**: Returns exit code 0 with success message
|
||||
- **Has errors**: Returns exit code 1 with structured error report:
|
||||
|
||||
```markdown
|
||||
## 🚨 Mermaid 语法检查报告
|
||||
|
||||
**检查文件：** `example.md`
|
||||
**检查结果：** ❌ 发现 2 处语法错误 (共检测到 5 个代码块)
|
||||
|
||||
---
|
||||
|
||||
### ❌ 错误 #1 (代码块 #2)
|
||||
|
||||
- **文档位置：** 第 `51` 行至第 `86` 行
|
||||
|
||||
#### 核心错误详情
|
||||
\`\`\`text
|
||||
Error: Parse error on line 13: ...
|
||||
Expecting 'SQE', 'DOUBLECIRCLEEND', ... got 'PS'
|
||||
\`\`\`
|
||||
|
||||
#### 代码内容片段
|
||||
\`\`\`text
|
||||
[shows the problematic code]
|
||||
\`\`\`
|
||||
```
|
||||
|
||||
### Step 2: Parse and Understand Errors
|
||||
|
||||
Key information from each error:
|
||||
- **文档位置**: Which line numbers contain the error
|
||||
- **核心错误详情**: The parser error message
|
||||
- **代码内容片段**: The actual Mermaid code causing the issue
|
||||
|
||||
### Step 3: Apply Fixes Based on Error Type
|
||||
|
||||
#### Error Type: "got 'PS'" (Parentheses Issue)
|
||||
|
||||
**Cause**: Parentheses `()` in node labels or arrow labels within subgraphs
|
||||
|
||||
**Fix**: Quote the label
|
||||
```mermaid
|
||||
# Before (causes error)
|
||||
NodeA[Label (with parens)]
|
||||
-->|Label (parens)| NodeB
|
||||
|
||||
# After (fixed)
|
||||
NodeA["Label (with parens)"]
|
||||
-->"|Label (parens)|" NodeB
|
||||
```
|
||||
|
||||
#### Error Type: "got 'SQS'" (Square Brackets Issue)
|
||||
|
||||
**Cause**: Square brackets `[]` in node labels
|
||||
|
||||
**Fix**: Quote the label
|
||||
```mermaid
|
||||
# Before
|
||||
NodeA[Result[]<br/>text]
|
||||
|
||||
# After
|
||||
NodeA["Result[]<br/>text"]
|
||||
```
|
||||
|
||||
#### Error Type: "got 'DIAMOND_START'" or similar
|
||||
|
||||
**Cause**: Curly braces `{}` or other special characters in labels
|
||||
|
||||
**Fix**: Quote the label
|
||||
```mermaid
|
||||
# Before
|
||||
NodeA[Label {variable}]
|
||||
|
||||
# After
|
||||
NodeA["Label {variable}"]
|
||||
```
|
||||
|
||||
### Step 4: Apply the Fix to the File
|
||||
|
||||
1. Read the Markdown file
|
||||
2. Locate the problematic code block using the reported line numbers
|
||||
3. Apply the appropriate fix (quote labels with special characters)
|
||||
4. Save the file
|
||||
|
||||
### Step 5: Verify
|
||||
|
||||
Run `check-mermaid.js` again to confirm all errors are resolved.
|
||||
|
||||
## Common Fix Patterns
|
||||
|
||||
| Error Pattern | Fix Strategy |
|
||||
|---------------|--------------|
|
||||
| `NodeID[label (text)]` | Change to `NodeID["label (text)"]` |
|
||||
| `-->|label (text)|` | Change to `-->|"label (text)"\|` |
|
||||
| `NodeID[label[]text]` | Change to `NodeID["label[]text"]` |
|
||||
| `NodeID[label{text}]` | Change to `NodeID["label{text}"]` |
|
||||
|
||||
## Important Notes
|
||||
|
||||
1. **Always quote labels** containing: `()`, `[]`, `{}`, `<`, `>`, `#`, or Chinese characters
|
||||
2. **Preserve the original structure** - only modify the problematic labels
|
||||
3. **Check all reported errors** - don't stop after fixing just one
|
||||
4. **Re-verify** after each fix round
|
||||
|
||||
## Example Usage
|
||||
|
||||
```
|
||||
User: "My Mermaid diagrams in docs/architecture.md aren't rendering"
|
||||
|
||||
Claude:
|
||||
1. Run: node scripts/check-mermaid.js docs/architecture.md
|
||||
2. Parse error output
|
||||
3. Identify errors (e.g., "got 'PS'" on line 45)
|
||||
4. Read file, locate line 45, find the Mermaid block
|
||||
5. Apply fix: quote labels with parentheses
|
||||
6. Save and verify
|
||||
```
|
||||
23
skills/mermaid-fixer/evals/evals.json
Normal file
23
skills/mermaid-fixer/evals/evals.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"skill_name": "mermaid-fixer",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"prompt": "The Mermaid diagrams in demo_scripts/demo.md have syntax errors. Please fix them.",
|
||||
"expected_output": "The skill should run check-mermaid.js, identify the 3 syntax errors, and apply fixes by quoting labels with special characters. After fixes, all 9 Mermaid blocks should pass validation.",
|
||||
"files": ["demo_scripts/demo.md"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"prompt": "My documentation file README.md has broken Mermaid diagrams. Can you check and fix them?",
|
||||
"expected_output": "The skill runs check-mermaid.js on README.md, parses any error reports, applies appropriate fixes based on error types, and verifies the fixes by running the check again.",
|
||||
"files": []
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"prompt": "CI/CD is failing with Mermaid validation errors on docs/api.md. Please fix the diagrams.",
|
||||
"expected_output": "The skill identifies all Mermaid syntax errors using check-mermaid.js, applies fixes for each error type (parentheses, brackets, braces in labels), and ensures the file passes validation.",
|
||||
"files": []
|
||||
}
|
||||
]
|
||||
}
|
||||
128
skills/mermaid-fixer/scripts/check-mermaid.js
Normal file
128
skills/mermaid-fixer/scripts/check-mermaid.js
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const { execSync } = require('child_process');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
// 获取命令行传入的 Markdown 文件路径
|
||||
const fileArg = process.argv[2];
|
||||
if (!fileArg) {
|
||||
console.error('❌ 请提供 Markdown 文件路径。用法: node check-mermaid.js <file.md>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const filePath = path.resolve(fileArg);
|
||||
const fileName = path.basename(filePath);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`❌ 文件不存在: ${filePath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
|
||||
// 正则匹配 ```mermaid ... ``` 代码块，兼容 Windows(\r\n) 和 Linux(\n)
|
||||
const mermaidRegex = /```mermaid\r?\n([\s\S]*?)```/g;
|
||||
let match;
|
||||
let blockCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
// 用于收集所有格式化后的错误报告块
|
||||
const errorReports = [];
|
||||
|
||||
while ((match = mermaidRegex.exec(content)) !== null) {
|
||||
blockCount++;
|
||||
|
||||
const fullMatch = match[0];
|
||||
const code = match[1];
|
||||
|
||||
// 1. 计算在 Markdown 文件中的绝对行号
|
||||
const textBeforeMatch = content.substring(0, match.index);
|
||||
const startLine = textBeforeMatch.split(/\r?\n/).length;
|
||||
const endLine = startLine + fullMatch.split(/\r?\n/).length - 1;
|
||||
|
||||
// 创建临时文件存放单个 Mermaid 代码
|
||||
const tmpFile = path.join(os.tmpdir(), `mermaid-check-${Date.now()}-${blockCount}.mmd`);
|
||||
fs.writeFileSync(tmpFile, code.trim());
|
||||
|
||||
try {
|
||||
// 调用 mmdc 进行静默渲染检查
|
||||
execSync(`npx mmdc -i "${tmpFile}" -o "${tmpFile}.svg" -q`, { stdio: 'pipe' });
|
||||
} catch (error) {
|
||||
errorCount++;
|
||||
|
||||
// 2. 净化报错信息：剔除底层执行堆栈
|
||||
const stderr = error.stderr ? error.stderr.toString() : error.message;
|
||||
const errorLines = stderr.split(/\r?\n/);
|
||||
const cleanErrorLines = [];
|
||||
|
||||
for (const line of errorLines) {
|
||||
if (line.trim().startsWith('at ') ||
|
||||
line.includes('Parser3.parseError') ||
|
||||
line.includes('fromText')) {
|
||||
break;
|
||||
}
|
||||
cleanErrorLines.push(line);
|
||||
}
|
||||
const cleanErrorOutput = cleanErrorLines.join('\n').trim();
|
||||
|
||||
// 3. 截断代码内容：最少3行，最多6行
|
||||
const codeLines = code.split(/\r?\n/).filter(l => l.trim() !== '');
|
||||
let displayCode = '';
|
||||
|
||||
if (codeLines.length <= 6) {
|
||||
displayCode = codeLines.join('\n');
|
||||
} else {
|
||||
const head = codeLines.slice(0, 3).join('\n');
|
||||
const tail = codeLines.slice(-3).join('\n');
|
||||
displayCode = `${head}\n ...\n ... (中间省略 ${codeLines.length - 6} 行) ...\n ...\n${tail}`;
|
||||
}
|
||||
|
||||
// 4. 组装单个错误的 Markdown 块
|
||||
errorReports.push(`### ❌ 错误 #${errorCount} (代码块 #${blockCount})
|
||||
|
||||
- **文档位置：** 第 \`${startLine}\` 行至第 \`${endLine}\` 行
|
||||
|
||||
#### 核心错误详情
|
||||
\`\`\`text
|
||||
${cleanErrorOutput}
|
||||
\`\`\`
|
||||
|
||||
#### 代码内容片段
|
||||
\`\`\`text
|
||||
${displayCode}
|
||||
\`\`\``);
|
||||
|
||||
} finally {
|
||||
// 清理临时文件
|
||||
if (fs.existsSync(tmpFile)) fs.unlinkSync(tmpFile);
|
||||
if (fs.existsSync(`${tmpFile}.svg`)) fs.unlinkSync(`${tmpFile}.svg`);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 生成最终的 Markdown 报告输出
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
if (errorCount > 0) {
|
||||
// 存在错误的情况
|
||||
console.log(`## 🚨 Mermaid 语法检查报告\n`);
|
||||
console.log(`**检查文件：** \`${fileName}\``);
|
||||
console.log(`**检查结果：** ❌ 发现 ${errorCount} 处语法错误 (共检测到 ${blockCount} 个代码块)\n`);
|
||||
console.log(`---\n`);
|
||||
console.log(errorReports.join('\n\n---\n\n'));
|
||||
|
||||
// 返回非零状态码，确保在 CI/CD 或 Git Hook 中能够阻断流程
|
||||
process.exit(1);
|
||||
} else {
|
||||
// 全部正确或没有代码块的情况
|
||||
console.log(`## 🎉 Mermaid 语法检查报告\n`);
|
||||
console.log(`**检查文件：** \`${fileName}\``);
|
||||
if (blockCount === 0) {
|
||||
console.log(`**检查结果：** ⚠️ 未检测到 Mermaid 代码块`);
|
||||
} else {
|
||||
console.log(`**检查结果：** ✅ 全部通过 (共检测到 ${blockCount} 个代码块)`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
212
skills/obsidian-cli/SKILL.md
Normal file
212
skills/obsidian-cli/SKILL.md
Normal file
@@ -0,0 +1,212 @@
|
||||
---
|
||||
name: obsidian-cli
|
||||
description: >-
|
||||
Obsidian CLI skill for reading, creating, editing, searching, and managing note content
|
||||
in Obsidian vaults from the command line. Supports daily notes, tasks, properties, tags,
|
||||
links, file history, templates, and full file CRUD operations.
|
||||
Use this skill whenever the user wants to read or edit Obsidian notes, search vault content,
|
||||
manage tasks or properties, append/prepend content, or perform any vault note operation.
|
||||
Also triggers when the user mentions "obsidian cli", "obsidian command", "vault" operations,
|
||||
or "obsidian" in a terminal/CLI context. Always invoke this skill before attempting
|
||||
Obsidian-related operations to ensure correct command syntax and available parameters.
|
||||
Requires Obsidian 1.12+ with CLI enabled and the Obsidian app running.
|
||||
---
|
||||
|
||||
# Obsidian CLI
|
||||
|
||||
Control Obsidian from the command line. The Obsidian desktop app must be running — the CLI connects to it over a local bridge.
|
||||
|
||||
> **Prerequisites:** Obsidian installer 1.12.7+ with CLI enabled (Settings -> General -> Command line interface).
|
||||
|
||||
## Syntax
|
||||
|
||||
**Parameters** use `key=value`. Quote values containing spaces:
|
||||
|
||||
```bash
|
||||
obsidian create name="My Note" content="Hello world"
|
||||
```
|
||||
|
||||
**Flags** are boolean switches with no value:
|
||||
|
||||
```bash
|
||||
obsidian create name="My Note" open overwrite
|
||||
```
|
||||
|
||||
**Multiline content:** Use `\n` for newline, `\t` for tab.
|
||||
|
||||
**Copy output:** Append `--copy` to any command to copy the result to clipboard.
|
||||
|
||||
## Targeting
|
||||
|
||||
### File targeting
|
||||
|
||||
Most file commands accept `file` and/or `path`. If neither is given, the active file is used.
|
||||
|
||||
| Parameter | Behavior |
|
||||
|-----------|----------|
|
||||
| `file=<name>` | Resolves like a wikilink — name only, no path or extension needed |
|
||||
| `path=<path>` | Exact vault-relative path, e.g. `folder/note.md` |
|
||||
|
||||
### Vault targeting
|
||||
|
||||
If your terminal CWD is a vault folder, that vault is used. Otherwise the active vault is used. Override with `vault=<name>` or `vault=<id>` as the first parameter:
|
||||
|
||||
```bash
|
||||
obsidian vault=Notes daily
|
||||
obsidian vault="My Vault" search query="test"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Commands
|
||||
|
||||
For the full parameter reference of every command, read `references/commands-reference.md`.
|
||||
|
||||
### Daily notes
|
||||
|
||||
```bash
|
||||
obsidian daily # Open today's daily note
|
||||
obsidian daily:read # Read daily note content
|
||||
obsidian daily:append content="- [ ] Buy groceries" # Append to daily note
|
||||
obsidian daily:prepend content="# Summary" # Prepend to daily note
|
||||
obsidian daily:path # Get daily note file path
|
||||
```
|
||||
|
||||
### Files and folders
|
||||
|
||||
```bash
|
||||
obsidian file file=Recipe # Show file info
|
||||
obsidian files folder="Projects" ext=md # List files (filter by folder/extension)
|
||||
obsidian folders # List all folders
|
||||
obsidian open file=Recipe # Open a file
|
||||
obsidian read file=Recipe # Read file contents
|
||||
obsidian create name="New Note" content="# Hello" # Create a note
|
||||
obsidian create name="Trip" template=Travel open # Create from template and open
|
||||
obsidian append file=Recipe content="New line" # Append to file
|
||||
obsidian prepend file=Recipe content="Header text" # Prepend after frontmatter
|
||||
obsidian move file=Recipe to="Archive/Recipe.md" # Move/rename file
|
||||
obsidian rename file=Recipe name="New Recipe" # Rename file
|
||||
obsidian delete file=Recipe # Delete file (to trash)
|
||||
obsidian delete file=Recipe permanent # Delete permanently
|
||||
```
|
||||
|
||||
### Search
|
||||
|
||||
```bash
|
||||
obsidian search query="meeting notes" # Search vault (returns file paths)
|
||||
obsidian search query="TODO" path="Projects" limit=10 # Search within a folder
|
||||
obsidian search:context query="error" # Search with line context (grep-style)
|
||||
obsidian search:open query="initial query" # Open search in Obsidian UI
|
||||
```
|
||||
|
||||
### Tasks
|
||||
|
||||
```bash
|
||||
obsidian tasks # List all tasks
|
||||
obsidian tasks todo # Incomplete tasks only
|
||||
obsidian tasks daily # Tasks from today's daily note
|
||||
obsidian tasks file=Recipe done verbose # Completed tasks from a file, with line numbers
|
||||
obsidian tasks daily total # Count tasks in daily note
|
||||
obsidian task ref="Recipe.md:8" toggle # Toggle task completion
|
||||
obsidian task daily line=3 done # Mark daily note task as done
|
||||
obsidian task file=Recipe line=8 status=- # Set custom status character
|
||||
```
|
||||
|
||||
### Properties
|
||||
|
||||
```bash
|
||||
obsidian properties # List all properties in vault
|
||||
obsidian properties active # Properties of active file
|
||||
obsidian properties name=status counts # Count occurrences of a property
|
||||
obsidian property:set name=status value=done file=Recipe # Set a property
|
||||
obsidian property:read name=status file=Recipe # Read a property value
|
||||
obsidian property:remove name=draft file=Recipe # Remove a property
|
||||
obsidian aliases file=Recipe # List aliases for a file
|
||||
```
|
||||
|
||||
### Tags
|
||||
|
||||
```bash
|
||||
obsidian tags # List all tags
|
||||
obsidian tags sort=count counts # Tags sorted by frequency with counts
|
||||
obsidian tags active # Tags of active file
|
||||
obsidian tag name=project verbose # Tag info with file list
|
||||
```
|
||||
|
||||
### Links
|
||||
|
||||
```bash
|
||||
obsidian backlinks file=Recipe # List backlinks to a file
|
||||
obsidian backlinks file=Recipe counts # With link counts
|
||||
obsidian links file=Recipe # Outgoing links from a file
|
||||
obsidian unresolved # List unresolved links
|
||||
obsidian orphans # Files with no incoming links
|
||||
obsidian deadends # Files with no outgoing links
|
||||
```
|
||||
|
||||
### Outline
|
||||
|
||||
```bash
|
||||
obsidian outline file=Recipe # Show headings (tree format)
|
||||
obsidian outline file=Recipe format=json total # JSON format with heading count
|
||||
```
|
||||
|
||||
### File history
|
||||
|
||||
```bash
|
||||
obsidian diff # List versions of active file
|
||||
obsidian diff file=Recipe from=1 # Compare latest version to current
|
||||
obsidian diff file=Recipe from=3 to=1 # Compare two versions
|
||||
obsidian history file=Recipe # Local history versions
|
||||
obsidian history:read file=Recipe version=2 # Read a history version
|
||||
obsidian history:restore file=Recipe version=3 # Restore a history version
|
||||
```
|
||||
|
||||
### Templates
|
||||
|
||||
```bash
|
||||
obsidian templates # List templates
|
||||
obsidian template:read name=Meeting resolve # Read template with variables resolved
|
||||
obsidian template:insert name=Meeting # Insert template into active file
|
||||
```
|
||||
|
||||
### Vault info
|
||||
|
||||
```bash
|
||||
obsidian vault # Show vault info
|
||||
obsidian vault info=files # File count only
|
||||
obsidian vaults verbose # List all known vaults with paths
|
||||
obsidian wordcount file=Recipe # Word and character count
|
||||
obsidian wordcount words # Word count only
|
||||
```
|
||||
|
||||
### General
|
||||
|
||||
```bash
|
||||
obsidian help # List all commands
|
||||
obsidian version # Show Obsidian version
|
||||
obsidian reload # Reload app window
|
||||
obsidian restart # Restart app
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting: CLI Connection Issues
|
||||
|
||||
When `obsidian` commands return empty output (with exit code 0) or are unresponsive, the CLI bridge connection is likely broken. Follow this procedure:
|
||||
|
||||
1. **Detect the issue**: If a command produces empty output with exit code 0, the bridge connection is broken
|
||||
2. **Ask the user**: Use AskUserQuestion to ask the user whether they allow running `obsidian reload` to reload the Obsidian window and restore the connection
|
||||
3. **If the user agrees**: Run `obsidian reload`, wait about 3 seconds, then retry the original command
|
||||
4. **If the user declines**: Fall back to reading/writing vault files directly via the filesystem (locate the vault path from `%APPDATA%/obsidian/obsidian.json`, then use Read/Edit/Write tools to operate on .md files directly)
|
||||
|
||||
> **Note**: `obsidian reload` reloads the Obsidian window. It does not affect unsaved data, but will briefly interrupt the current editing state.
|
||||
|
||||
---
|
||||
|
||||
## Tips
|
||||
|
||||
- **Clipboard:** Add `--copy` to any command to copy output.
|
||||
- **Counts:** Add `total` to list commands to get a count instead of full output.
|
||||
- **Output formats:** Many list commands accept `format=json|tsv|csv` for structured output.
|
||||
- **Help:** Run `obsidian help` for an always-up-to-date command list. Run `obsidian help <command>` for command-specific help.
|
||||
568
skills/obsidian-cli/references/commands-reference.md
Normal file
568
skills/obsidian-cli/references/commands-reference.md
Normal file
@@ -0,0 +1,568 @@
|
||||
# Commands Reference
|
||||
|
||||
Full parameter reference for Obsidian CLI note content commands. Required parameters are marked `(required)`.
|
||||
|
||||
## Table of contents
|
||||
|
||||
- [General](#general)
|
||||
- [Daily notes](#daily-notes)
|
||||
- [Files and folders](#files-and-folders)
|
||||
- [Search](#search)
|
||||
- [Tasks](#tasks)
|
||||
- [Properties](#properties)
|
||||
- [Tags](#tags)
|
||||
- [Links](#links)
|
||||
- [Outline](#outline)
|
||||
- [File history](#file-history)
|
||||
- [Templates](#templates)
|
||||
- [Vault](#vault)
|
||||
- [Wordcount](#wordcount)
|
||||
|
||||
---
|
||||
|
||||
## General
|
||||
|
||||
### `help`
|
||||
|
||||
Show available commands or help for a specific command.
|
||||
|
||||
| Parameter | Description |
|
||||
| ----------- | --------------------------------- |
|
||||
| `<command>` | Show help for a specific command. |
|
||||
|
||||
### `version`
|
||||
|
||||
Show Obsidian version.
|
||||
|
||||
### `reload`
|
||||
|
||||
Reload the app window.
|
||||
|
||||
### `restart`
|
||||
|
||||
Restart the app.
|
||||
|
||||
---
|
||||
|
||||
## Daily notes
|
||||
|
||||
### `daily`
|
||||
|
||||
Open daily note.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------------- | ---------------------- |
|
||||
| `paneType=tab\|split\|window` | Pane type to open in |
|
||||
|
||||
### `daily:path`
|
||||
|
||||
Get daily note path. Returns the expected path even if the file hasn't been created yet.
|
||||
|
||||
### `daily:read`
|
||||
|
||||
Read daily note contents.
|
||||
|
||||
### `daily:append`
|
||||
|
||||
Append content to daily note.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------------- | ---------------------------- |
|
||||
| `content=<text>` | (required) Content to append |
|
||||
| `paneType=tab\|split\|window` | Pane type to open in |
|
||||
| `inline` | Append without newline |
|
||||
| `open` | Open file after adding |
|
||||
|
||||
### `daily:prepend`
|
||||
|
||||
Prepend content to daily note.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------------- | ---------------------------- |
|
||||
| `content=<text>` | (required) Content to prepend|
|
||||
| `paneType=tab\|split\|window` | Pane type to open in |
|
||||
| `inline` | Prepend without newline |
|
||||
| `open` | Open file after adding |
|
||||
|
||||
---
|
||||
|
||||
## Files and folders
|
||||
|
||||
### `file`
|
||||
|
||||
Show file info (default: active file). Returns path, name, extension, size, created/modified timestamps.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ----------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
### `files`
|
||||
|
||||
List files in the vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| ----------------- | ------------------------ |
|
||||
| `folder=<path>` | Filter by folder |
|
||||
| `ext=<extension>` | Filter by extension |
|
||||
| `total` | Return file count |
|
||||
|
||||
### `folder`
|
||||
|
||||
Show folder info.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------------ | -------------------------- |
|
||||
| `path=<path>` | (required) Folder path |
|
||||
| `info=files\|folders\|size` | Return specific info only |
|
||||
|
||||
### `folders`
|
||||
|
||||
List folders in the vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------------- | ------------------------ |
|
||||
| `folder=<path>` | Filter by parent folder |
|
||||
| `total` | Return folder count |
|
||||
|
||||
### `open`
|
||||
|
||||
Open a file.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ---------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `newtab` | Open in new tab |
|
||||
|
||||
### `create`
|
||||
|
||||
Create or overwrite a file.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------ | ------------------------ |
|
||||
| `name=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `content=<text>` | Initial content |
|
||||
| `template=<name>` | Template to use |
|
||||
| `overwrite` | Overwrite if exists |
|
||||
| `open` | Open after creating |
|
||||
| `newtab` | Open in new tab |
|
||||
|
||||
### `read`
|
||||
|
||||
Read file contents (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ----------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
### `append`
|
||||
|
||||
Append content to a file (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------- | ---------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `content=<text>` | (required) Content to append |
|
||||
| `inline` | Append without newline |
|
||||
|
||||
### `prepend`
|
||||
|
||||
Prepend content after frontmatter (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------- | ----------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `content=<text>` | (required) Content to prepend |
|
||||
| `inline` | Prepend without newline |
|
||||
|
||||
### `move`
|
||||
|
||||
Move or rename a file (default: active file). Auto-updates internal links if enabled.
|
||||
|
||||
| Parameter | Description |
|
||||
| -------------- | ----------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `to=<path>` | (required) Destination path |
|
||||
|
||||
### `rename`
|
||||
|
||||
Rename a file (default: active file). Extension preserved if omitted. Use `move` to rename and move simultaneously.
|
||||
|
||||
| Parameter | Description |
|
||||
| -------------- | ------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `name=<name>` | (required) New file name |
|
||||
|
||||
### `delete`
|
||||
|
||||
Delete a file (default: active file, trash by default).
|
||||
|
||||
| Parameter | Description |
|
||||
| -------------- | ------------------------------ |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `permanent` | Skip trash, delete permanently |
|
||||
|
||||
---
|
||||
|
||||
## Search
|
||||
|
||||
### `search`
|
||||
|
||||
Search vault for text. Returns matching file paths.
|
||||
|
||||
| Parameter | Description |
|
||||
| -------------------- | ------------------------------ |
|
||||
| `query=<text>` | (required) Search query |
|
||||
| `path=<folder>` | Limit to folder |
|
||||
| `limit=<n>` | Max files |
|
||||
| `format=text\|json` | Output format (default: text) |
|
||||
| `total` | Return match count |
|
||||
| `case` | Case sensitive |
|
||||
|
||||
### `search:context`
|
||||
|
||||
Search with matching line context. Returns grep-style `path:line: text` output.
|
||||
|
||||
| Parameter | Description |
|
||||
| -------------------- | ------------------------------ |
|
||||
| `query=<text>` | (required) Search query |
|
||||
| `path=<folder>` | Limit to folder |
|
||||
| `limit=<n>` | Max files |
|
||||
| `format=text\|json` | Output format (default: text) |
|
||||
| `case` | Case sensitive |
|
||||
|
||||
### `search:open`
|
||||
|
||||
Open search view in Obsidian.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------------- | ---------------------- |
|
||||
| `query=<text>` | Initial search query |
|
||||
|
||||
---
|
||||
|
||||
## Tasks
|
||||
|
||||
### `tasks`
|
||||
|
||||
List tasks in the vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------ | --------------------------------- |
|
||||
| `file=<name>` | Filter by file name |
|
||||
| `path=<path>` | Filter by file path |
|
||||
| `status="<char>"` | Filter by status character |
|
||||
| `total` | Return task count |
|
||||
| `done` | Show completed tasks |
|
||||
| `todo` | Show incomplete tasks |
|
||||
| `verbose` | Group by file with line numbers |
|
||||
| `format=json\|tsv\|csv` | Output format (default: text) |
|
||||
| `active` | Show tasks for active file |
|
||||
| `daily` | Show tasks from daily note |
|
||||
|
||||
### `task`
|
||||
|
||||
Show or update a task.
|
||||
|
||||
| Parameter | Description |
|
||||
| -------------------- | -------------------------- |
|
||||
| `ref=<path:line>` | Task reference (path:line) |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `line=<n>` | Line number |
|
||||
| `status="<char>"` | Set status character |
|
||||
| `toggle` | Toggle task status |
|
||||
| `daily` | Target daily note |
|
||||
| `done` | Mark as done |
|
||||
| `todo` | Mark as todo |
|
||||
|
||||
---
|
||||
|
||||
## Properties
|
||||
|
||||
### `aliases`
|
||||
|
||||
List aliases in the vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | -------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `total` | Return alias count |
|
||||
| `verbose` | Include file paths |
|
||||
| `active` | Show for active file |
|
||||
|
||||
### `properties`
|
||||
|
||||
List properties in the vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------ | ----------------------------- |
|
||||
| `file=<name>` | Show properties for file |
|
||||
| `path=<path>` | Show properties for path |
|
||||
| `name=<name>` | Get specific property count |
|
||||
| `sort=count` | Sort by count (default: name) |
|
||||
| `format=yaml\|json\|tsv` | Output format (default: yaml) |
|
||||
| `total` | Return property count |
|
||||
| `counts` | Include occurrence counts |
|
||||
| `active` | Show for active file |
|
||||
|
||||
### `property:set`
|
||||
|
||||
Set a property on a file (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------------------------------------------- | -------------------- |
|
||||
| `name=<name>` | (required) Property name |
|
||||
| `value=<value>` | (required) Property value |
|
||||
| `type=text\|list\|number\|checkbox\|date\|datetime` | Property type |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
### `property:remove`
|
||||
|
||||
Remove a property from a file (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ------------------------ |
|
||||
| `name=<name>` | (required) Property name |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
### `property:read`
|
||||
|
||||
Read a property value from a file (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ------------------------ |
|
||||
| `name=<name>` | (required) Property name |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
---
|
||||
|
||||
## Tags
|
||||
|
||||
### `tags`
|
||||
|
||||
List tags in the vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------ | ----------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `sort=count` | Sort by count (default: name) |
|
||||
| `total` | Return tag count |
|
||||
| `counts` | Include tag counts |
|
||||
| `format=json\|tsv\|csv` | Output format (default: tsv) |
|
||||
| `active` | Show for active file |
|
||||
|
||||
### `tag`
|
||||
|
||||
Get tag info.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------ | --------------------------- |
|
||||
| `name=<tag>` | (required) Tag name |
|
||||
| `total` | Return occurrence count |
|
||||
| `verbose` | Include file list and count |
|
||||
|
||||
---
|
||||
|
||||
## Links
|
||||
|
||||
### `backlinks`
|
||||
|
||||
List backlinks to a file (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------ | ----------------------------- |
|
||||
| `file=<name>` | Target file name |
|
||||
| `path=<path>` | Target file path |
|
||||
| `counts` | Include link counts |
|
||||
| `total` | Return backlink count |
|
||||
| `format=json\|tsv\|csv` | Output format (default: tsv) |
|
||||
|
||||
### `links`
|
||||
|
||||
List outgoing links from a file (default: active file).
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ------------------ |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `total` | Return link count |
|
||||
|
||||
### `unresolved`
|
||||
|
||||
List unresolved links in vault.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------ | ----------------------------- |
|
||||
| `total` | Return unresolved link count |
|
||||
| `counts` | Include link counts |
|
||||
| `verbose` | Include source files |
|
||||
| `format=json\|tsv\|csv` | Output format (default: tsv) |
|
||||
|
||||
### `orphans`
|
||||
|
||||
List files with no incoming links.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------- | ------------------- |
|
||||
| `total` | Return orphan count |
|
||||
|
||||
### `deadends`
|
||||
|
||||
List files with no outgoing links.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------- | ---------------------- |
|
||||
| `total` | Return dead-end count |
|
||||
|
||||
---
|
||||
|
||||
## Outline
|
||||
|
||||
### `outline`
|
||||
|
||||
Show headings for a file.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------------ | ------------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `format=tree\|md\|json` | Output format (default: tree) |
|
||||
| `total` | Return heading count |
|
||||
|
||||
---
|
||||
|
||||
## File history
|
||||
|
||||
### `diff`
|
||||
|
||||
List or compare versions from local File recovery and Sync. Versions numbered newest to oldest.
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------------- | ------------------------------ |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `from=<n>` | Version number to diff from |
|
||||
| `to=<n>` | Version number to diff to |
|
||||
| `filter=local\|sync` | Filter by version source |
|
||||
|
||||
### `history`
|
||||
|
||||
List versions from File recovery only.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ----------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
### `history:list`
|
||||
|
||||
List all files with local history.
|
||||
|
||||
### `history:read`
|
||||
|
||||
Read a local history version.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------------- | ------------------------------ |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `version=<n>` | Version number (default: 1) |
|
||||
|
||||
### `history:restore`
|
||||
|
||||
Restore a local history version.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------------- | ------------------------ |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `version=<n>` | (required) Version number|
|
||||
|
||||
### `history:open`
|
||||
|
||||
Open file recovery.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------- | ----------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
|
||||
---
|
||||
|
||||
## Templates
|
||||
|
||||
### `templates`
|
||||
|
||||
List templates.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------- | ---------------------- |
|
||||
| `total` | Return template count |
|
||||
|
||||
### `template:read`
|
||||
|
||||
Read template content.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------ | --------------------------------------------------- |
|
||||
| `name=<template>` | (required) Template name |
|
||||
| `title=<title>` | Title for variable resolution |
|
||||
| `resolve` | Resolve `{{date}}`, `{{time}}`, `{{title}}` variables |
|
||||
|
||||
### `template:insert`
|
||||
|
||||
Insert template into active file.
|
||||
|
||||
| Parameter | Description |
|
||||
| ------------------ | ------------------------ |
|
||||
| `name=<template>` | (required) Template name |
|
||||
|
||||
---
|
||||
|
||||
## Vault
|
||||
|
||||
### `vault`
|
||||
|
||||
Show vault info.
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------------------------------- | -------------------------- |
|
||||
| `info=name\|path\|files\|folders\|size` | Return specific info only |
|
||||
|
||||
### `vaults`
|
||||
|
||||
List known vaults.
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------- | -------------------- |
|
||||
| `total` | Return vault count |
|
||||
| `verbose` | Include vault paths |
|
||||
|
||||
---
|
||||
|
||||
## Wordcount
|
||||
|
||||
### `wordcount`
|
||||
|
||||
Count words and characters.
|
||||
|
||||
| Parameter | Description |
|
||||
| --------------- | ---------------------------- |
|
||||
| `file=<name>` | File name |
|
||||
| `path=<path>` | File path |
|
||||
| `words` | Return word count only |
|
||||
| `characters` | Return character count only |
|
||||
201
skills/seq-api/SKILL.md
Normal file
201
skills/seq-api/SKILL.md
Normal file
@@ -0,0 +1,201 @@
|
||||
---
|
||||
name: seq-api
|
||||
description: Guide for making HTTP API calls to the Seq structured log server (by Datalust). Use this skill whenever the user wants to interact with Seq programmatically — including ingesting events/logs/traces, querying events, managing API keys, signals, dashboards, alerts, users, workspaces, retention policies, app instances, backups, diagnostics, or any other Seq server resource. Also trigger when the user mentions Seq API, CLEF ingestion, Seq health checks, Seq SQL queries via API, or building scripts/automation that talk to a Seq server. Even if the user just says "send logs to Seq" or "query Seq" or "check Seq health", use this skill.
|
||||
---
|
||||
|
||||
# Seq HTTP API Skill
|
||||
|
||||
This skill provides guidance for calling the Seq server HTTP API. Seq is a centralized structured log server by Datalust. Its full API is REST-like, JSON-based, and navigable from the `/api/` root resource.
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Base URL & Discovery
|
||||
|
||||
The API root is at `{SEQ_SERVER_URL}/api/`. A GET to this endpoint returns a JSON object listing links to all resource groups (e.g., `ApiKeysResources`, `EventsResources`).
|
||||
|
||||
### Authentication
|
||||
|
||||
Two methods (only needed when auth is enabled on the server):
|
||||
|
||||
1. **Query string**: `?apiKey={YOUR_API_KEY}`
|
||||
2. **Header**: `X-Seq-ApiKey: {YOUR_API_KEY}`
|
||||
|
||||
API keys are the recommended method. Each key has specific permissions — make sure the key has the required permission level for the endpoint you're calling (see permission levels below).
|
||||
|
||||
### Permission Levels
|
||||
|
||||
Seq endpoints require one of these permission levels (from lowest to highest):
|
||||
|
||||
- **Public** — No auth needed (health checks, resource listings, auth settings)
|
||||
- **Ingest** — Required for writing events when `RequireApiKeyForWritingEvents` is enabled
|
||||
- **Read** — View events, signals, dashboards, alerts, queries
|
||||
- **Write** — Create/update/delete signals, dashboards, alerts, queries
|
||||
- **Project** — Administrative project-level access (retention policies, indexes, alert state, diagnostics metrics)
|
||||
- **Organization** — Manage users
|
||||
- **System** — Full server administration (apps, backups, cluster, feeds, settings, licenses)
|
||||
|
||||
### Common Patterns
|
||||
|
||||
All resource endpoints follow a consistent pattern:
|
||||
|
||||
| Operation | Method | Path |
|
||||
|-----------|--------|------|
|
||||
| List all | GET | `api/{resource}` |
|
||||
| Create | POST | `api/{resource}` |
|
||||
| Get one | GET | `api/{resource}/{id}` |
|
||||
| Update | PUT | `api/{resource}/{id}` |
|
||||
| Delete | DELETE | `api/{resource}/{id}` |
|
||||
| Get template | GET | `api/{resource}/template` |
|
||||
| Resource links | GET | `api/{resource}/resources` |
|
||||
|
||||
Use the `template` endpoint to get a blank object with the correct structure before creating a new resource.
|
||||
|
||||
### Content Type
|
||||
|
||||
All API requests and responses use `application/json`, except:
|
||||
- CLEF ingestion uses `application/vnd.serilog.clef`
|
||||
- OpenTelemetry ingestion uses protocol-specific content types
|
||||
|
||||
---
|
||||
|
||||
## Key Tasks
|
||||
|
||||
### 1. Health Check
|
||||
|
||||
```
|
||||
GET {SEQ_URL}/health
|
||||
```
|
||||
|
||||
Returns 200 (healthy) or 503 (unhealthy) with JSON body `{"status": "..."}`. No authentication needed. For clusters, use `/health/cluster`.
|
||||
|
||||
### 2. Ingest Events (CLEF Format)
|
||||
|
||||
**Read `references/ingestion.md` for full CLEF format details and all reified properties.**
|
||||
|
||||
```
|
||||
POST {SEQ_URL}/ingest/clef
|
||||
Content-Type: application/vnd.serilog.clef
|
||||
X-Seq-ApiKey: {API_KEY}
|
||||
|
||||
{"@t":"2024-01-15T10:30:00Z","@mt":"User {User} logged in","User":"alice","@l":"Information"}
|
||||
{"@t":"2024-01-15T10:30:01Z","@mt":"Error processing {OrderId}","OrderId":123,"@l":"Error","@x":"System.Exception: ..."}
|
||||
```
|
||||
|
||||
Response: `201 Created` with `{"MinimumLevelAccepted": null}` on success.
|
||||
|
||||
### 3. Ingest via OpenTelemetry
|
||||
|
||||
```
|
||||
POST {SEQ_URL}/ingest/otlp/v1/logs
|
||||
POST {SEQ_URL}/ingest/otlp/v1/traces
|
||||
POST {SEQ_URL}/ingest/otlp/v1/metrics
|
||||
```
|
||||
|
||||
### 4. Query Events
|
||||
|
||||
```
|
||||
GET {SEQ_URL}/api/events?count=30&filter=StatusCode%20%3E%20399
|
||||
```
|
||||
|
||||
Key query parameters: `count`, `filter` (Seq filter expression), `signal` (signal ID), `fromDateUtc`, `toDateUtc`.
|
||||
|
||||
For SQL-style queries:
|
||||
|
||||
```
|
||||
POST {SEQ_URL}/api/data
|
||||
Content-Type: application/json
|
||||
X-Seq-ApiKey: {API_KEY}
|
||||
|
||||
{
|
||||
"Query": "select count(*) from stream group by RequestPath",
|
||||
"RangeStartUtc": "2024-01-01T00:00:00Z",
|
||||
"RangeEndUtc": "2024-01-02T00:00:00Z",
|
||||
"SignalExpression": null
|
||||
}
|
||||
```
|
||||
|
||||
Also available via GET: `GET /api/data?q={query}&rangeStartUtc=...&rangeEndUtc=...`
|
||||
|
||||
### 5. Manage API Keys
|
||||
|
||||
```bash
|
||||
# List keys (need Read or Project permission)
|
||||
GET /api/apikeys
|
||||
|
||||
# Create a key
|
||||
POST /api/apikeys
|
||||
{"Title": "My App Key", "AssignedPermissions": ["Ingest"]}
|
||||
|
||||
# Delete a key
|
||||
DELETE /api/apikeys/{id}
|
||||
```
|
||||
|
||||
### 6. Manage Signals
|
||||
|
||||
```bash
|
||||
# List signals
|
||||
GET /api/signals
|
||||
|
||||
# Create a signal
|
||||
POST /api/signals
|
||||
{"Title": "Errors", "Filters": [{"Filter": "@Level = 'Error'"}]}
|
||||
```
|
||||
|
||||
### 7. Manage Dashboards & Alerts
|
||||
|
||||
Follow the same CRUD pattern at `/api/dashboards` and `/api/alerts`. Use the `/template` endpoint to get the correct JSON structure.
|
||||
|
||||
### 8. Server Diagnostics
|
||||
|
||||
```bash
|
||||
GET /api/diagnostics/status # Read permission
|
||||
GET /api/diagnostics/metrics # Project permission
|
||||
GET /api/diagnostics/ingestion # System permission
|
||||
GET /api/diagnostics/storage # Project permission
|
||||
GET /api/diagnostics/report # System permission — full diagnostic report
|
||||
```
|
||||
|
||||
### 9. User Management
|
||||
|
||||
```bash
|
||||
GET /api/users/current # Get logged-in user
|
||||
POST /api/users/login # Authenticate
|
||||
GET /api/users # List users (Project permission)
|
||||
POST /api/users # Create user (Organization permission)
|
||||
```
|
||||
|
||||
### 10. Backup
|
||||
|
||||
```bash
|
||||
POST /api/backups/immediate # Trigger immediate backup (System permission)
|
||||
GET /api/backups # List backups
|
||||
GET /api/backups/files/{name} # Download a backup file
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ownership & Sharing Rules
|
||||
|
||||
For signals, dashboards, alerts, SQL queries, and workspaces:
|
||||
- Users can only read/modify **shared** resources and **their own** resources
|
||||
- Creating/modifying **protected** resources requires **Project** permission
|
||||
- Resources are either shared (visible to everyone), personal (owner-only), or protected (admin-managed)
|
||||
|
||||
---
|
||||
|
||||
## Response Conventions
|
||||
|
||||
- Successful list requests return a JSON array
|
||||
- Successful single-resource requests return a JSON object
|
||||
- Error responses include `{"Error": "message"}`
|
||||
- Ingestion success returns `{"MinimumLevelAccepted": null}` (or a level string if filtering is applied)
|
||||
|
||||
---
|
||||
|
||||
## Reference Files
|
||||
|
||||
For the complete endpoint listing with all paths, HTTP methods, and permission requirements, read:
|
||||
- `references/endpoints.md` — Full API endpoint table for every resource
|
||||
- `references/ingestion.md` — CLEF format specification, reified properties, status codes, and batch formatting
|
||||
|
||||
Read the appropriate reference file when you need the specific details for an endpoint or ingestion format.
|
||||
488
skills/seq-api/references/endpoints.md
Normal file
488
skills/seq-api/references/endpoints.md
Normal file
@@ -0,0 +1,488 @@
|
||||
# Seq Server API — Complete Endpoint Reference
|
||||
|
||||
This file lists every API endpoint provided by the Seq server, organized by resource group.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [api (root)](#api-root)
|
||||
2. [alerts](#alerts)
|
||||
3. [alertstate](#alertstate)
|
||||
4. [apikeys](#apikeys)
|
||||
5. [appinstances](#appinstances)
|
||||
6. [apps](#apps)
|
||||
7. [backups](#backups)
|
||||
8. [cluster](#cluster)
|
||||
9. [dashboards](#dashboards)
|
||||
10. [data (queries)](#data-queries)
|
||||
11. [deferred](#deferred)
|
||||
12. [diagnostics](#diagnostics)
|
||||
13. [events](#events)
|
||||
14. [expressionindexes](#expressionindexes)
|
||||
15. [expressions](#expressions)
|
||||
16. [feeds](#feeds)
|
||||
17. [indexes](#indexes)
|
||||
18. [licenses](#licenses)
|
||||
19. [permalinks](#permalinks)
|
||||
20. [retentionpolicies](#retentionpolicies)
|
||||
21. [roles](#roles)
|
||||
22. [runningtasks](#runningtasks)
|
||||
23. [settings](#settings)
|
||||
24. [signals](#signals)
|
||||
25. [sqlqueries](#sqlqueries)
|
||||
26. [updates](#updates)
|
||||
27. [users](#users)
|
||||
28. [workspaces](#workspaces)
|
||||
29. [health](#health)
|
||||
30. [ingestion](#ingestion)
|
||||
31. [other](#other)
|
||||
|
||||
---
|
||||
|
||||
## api (root)
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api` | GET | Public |
|
||||
|
||||
Returns the root resource with links to all API resource groups.
|
||||
|
||||
---
|
||||
|
||||
## alerts
|
||||
|
||||
Manage alert definitions. Users can only access shared alerts and their own. Protected alerts require `Project` permission.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/alerts` | GET | Read | Shared + own only |
|
||||
| `api/alerts` | POST | Write | Project for protected |
|
||||
| `api/alerts/{id}` | GET | Read | Shared + own only |
|
||||
| `api/alerts/{id}` | PUT | Write | Project for protected |
|
||||
| `api/alerts/{id}` | DELETE | Write | Project for protected |
|
||||
| `api/alerts/resources` | GET | Public | |
|
||||
| `api/alerts/template` | GET | Write | |
|
||||
|
||||
---
|
||||
|
||||
## alertstate
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/alertstate` | GET | Project |
|
||||
| `api/alertstate/{id}` | GET | Project |
|
||||
| `api/alertstate/{id}` | DELETE | Project |
|
||||
| `api/alertstate/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## apikeys
|
||||
|
||||
Manage API keys. Non-Project principals can only view/manage their own keys.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/apikeys` | GET | Read | Project sees all; others own only |
|
||||
| `api/apikeys` | POST | Write | Can only delegate own permissions |
|
||||
| `api/apikeys/{id}` | GET | Read | Project sees all; others own only |
|
||||
| `api/apikeys/{id}` | PUT | Write | Can only delegate own permissions |
|
||||
| `api/apikeys/{id}` | DELETE | Write | Project removes any; others own only |
|
||||
| `api/apikeys/{id}/metrics/{measurement}` | GET | Read | Own or Project |
|
||||
| `api/apikeys/metrics/{measurement}` | GET | Project | |
|
||||
| `api/apikeys/resources` | GET | Public | |
|
||||
| `api/apikeys/template` | GET | Read | |
|
||||
|
||||
---
|
||||
|
||||
## appinstances
|
||||
|
||||
Manage installed Seq app instances. Non-Project principals see basic details only.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/appinstances` | GET | Read | Basic details without Project |
|
||||
| `api/appinstances` | POST | System | |
|
||||
| `api/appinstances/{id}` | GET | Read | Basic details without Project |
|
||||
| `api/appinstances/{id}` | PUT | System | |
|
||||
| `api/appinstances/{id}` | DELETE | System | |
|
||||
| `api/appinstances/{id}/icon` | GET | Read | |
|
||||
| `api/appinstances/{id}/invoke` | POST | Write | Must be an output app; System for non-direct-invocation |
|
||||
| `api/appinstances/{id}/metrics/{measurement}` | GET | Project | |
|
||||
| `api/appinstances/resources` | GET | Public | |
|
||||
| `api/appinstances/template` | GET | System | |
|
||||
|
||||
---
|
||||
|
||||
## apps
|
||||
|
||||
Manage app packages (install, update, remove). All require System permission.
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/apps` | GET | System |
|
||||
| `api/apps/{id}` | GET | System |
|
||||
| `api/apps/{id}` | DELETE | System |
|
||||
| `api/apps/{id}/icon` | GET | System |
|
||||
| `api/apps/{id}/update` | POST | System |
|
||||
| `api/apps/install` | POST | System |
|
||||
| `api/apps/resources` | GET | Public |
|
||||
| `api/apps/template` | GET | System |
|
||||
|
||||
---
|
||||
|
||||
## backups
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/backups` | GET | System | |
|
||||
| `api/backups/{id}` | GET | System | |
|
||||
| `api/backups/files/{filename}` | GET | System | Download backup file |
|
||||
| `api/backups/immediate` | POST | System | Allows cross-site POSTs |
|
||||
| `api/backups/resources` | GET | Public | |
|
||||
|
||||
---
|
||||
|
||||
## cluster
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/cluster` | GET | System |
|
||||
| `api/cluster/{id}` | GET | System |
|
||||
| `api/cluster/{id}/drain` | POST | System |
|
||||
| `api/cluster/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## dashboards
|
||||
|
||||
Manage dashboards. Users can only access shared dashboards and their own. Protected dashboards require `Project` permission.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/dashboards` | GET | Read | Shared + own only |
|
||||
| `api/dashboards` | POST | Write | Project for protected |
|
||||
| `api/dashboards/{id}` | GET | Read | Shared + own only |
|
||||
| `api/dashboards/{id}` | PUT | Write | Project for protected |
|
||||
| `api/dashboards/{id}` | DELETE | Write | Project for protected |
|
||||
| `api/dashboards/query/template` | GET | Write | |
|
||||
| `api/dashboards/resources` | GET | Public | |
|
||||
| `api/dashboards/template` | GET | Write | |
|
||||
|
||||
---
|
||||
|
||||
## data (queries)
|
||||
|
||||
Execute SQL-style queries against the event stream.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/data` | GET | Read | Query via query params |
|
||||
| `api/data` | POST | Read | Query via JSON body |
|
||||
| `api/data/{signalId}` | GET | Read | **Obsolete** |
|
||||
| `api/data/resources` | GET | Public | |
|
||||
|
||||
---
|
||||
|
||||
## deferred
|
||||
|
||||
Retrieve results of long-running/deferred operations.
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/deferred/{deferredId}` | GET | Read |
|
||||
|
||||
---
|
||||
|
||||
## diagnostics
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/diagnostics/status` | GET | Read | Basic server status |
|
||||
| `api/diagnostics/metrics` | GET | Project | |
|
||||
| `api/diagnostics/metrics/{measurement}` | GET | Project | |
|
||||
| `api/diagnostics/ingestion` | GET | System | |
|
||||
| `api/diagnostics/storage` | GET | Project | |
|
||||
| `api/diagnostics/report` | GET | System | Full diagnostic report |
|
||||
| `api/diagnostics/cluster/metrics` | GET | System | |
|
||||
| `api/diagnostics/usage-telemetry` | POST | Read | |
|
||||
| `api/diagnostics/resources` | GET | Public | |
|
||||
|
||||
---
|
||||
|
||||
## events
|
||||
|
||||
Core event operations — retrieve, search, stream, delete by signal, and raw ingestion.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/events` | GET | Read | List/search events |
|
||||
| `api/events/{id}` | GET | Read | Get single event |
|
||||
| `api/events/raw` | POST | Public* | Raw event ingestion; cross-site allowed. *Ingest required if RequireApiKeyForWritingEvents is on |
|
||||
| `api/events/scan` | GET | Read | |
|
||||
| `api/events/scan` | POST | Read | |
|
||||
| `api/events/signal` | GET | Read | |
|
||||
| `api/events/signal` | POST | Read | |
|
||||
| `api/events/signal` | DELETE | Project | Delete events matching signal |
|
||||
| `api/events/signal/{signalId}` | GET | Read | **Obsolete** |
|
||||
| `api/events/stream` | GET | Read | Live event stream (Server-Sent Events) |
|
||||
| `api/events/tabulate` | POST | Read | |
|
||||
| `api/events/tabulate/{signalId}` | GET | Read | |
|
||||
| `api/events/resources` | GET | Public | |
|
||||
|
||||
---
|
||||
|
||||
## expressionindexes
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/expressionindexes` | GET | Read |
|
||||
| `api/expressionindexes` | POST | Write |
|
||||
| `api/expressionindexes/{id}` | GET | Read |
|
||||
| `api/expressionindexes/{id}` | DELETE | Write |
|
||||
| `api/expressionindexes/resources` | GET | Public |
|
||||
| `api/expressionindexes/template` | GET | Write |
|
||||
|
||||
---
|
||||
|
||||
## expressions
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/expressions/sql` | GET | Read |
|
||||
| `api/expressions/strict` | GET | Read |
|
||||
| `api/expressions/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## feeds
|
||||
|
||||
App package feeds. All require System permission.
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/feeds` | GET | System |
|
||||
| `api/feeds` | POST | System |
|
||||
| `api/feeds/{id}` | GET | System |
|
||||
| `api/feeds/{id}` | PUT | System |
|
||||
| `api/feeds/{id}` | DELETE | System |
|
||||
| `api/feeds/resources` | GET | Public |
|
||||
| `api/feeds/template` | GET | System |
|
||||
|
||||
---
|
||||
|
||||
## indexes
|
||||
|
||||
Signal indexes. Require Project permission.
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/indexes` | GET | Project |
|
||||
| `api/indexes/{id}` | GET | Project |
|
||||
| `api/indexes/{id}` | DELETE | Project |
|
||||
| `api/indexes/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## licenses
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/licenses` | GET | System | |
|
||||
| `api/licenses/{id}` | GET | Read | Read sees status; System sees certificate details |
|
||||
| `api/licenses/{id}` | PUT | System | |
|
||||
| `api/licenses/downgrade` | POST | System | |
|
||||
| `api/licenses/resources` | GET | Public | |
|
||||
|
||||
---
|
||||
|
||||
## permalinks
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/permalinks` | GET | Read | Non-Project: own only |
|
||||
| `api/permalinks` | POST | Write | Non-Project: own only |
|
||||
| `api/permalinks/{id}` | GET | Read | Non-Project: own only |
|
||||
| `api/permalinks/{id}` | DELETE | Write | Non-Project: own only |
|
||||
| `api/permalinks/resources` | GET | Public | |
|
||||
| `api/permalinks/template` | GET | Write | |
|
||||
|
||||
---
|
||||
|
||||
## retentionpolicies
|
||||
|
||||
All require Project permission.
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/retentionpolicies` | GET | Project |
|
||||
| `api/retentionpolicies` | POST | Project |
|
||||
| `api/retentionpolicies/{id}` | GET | Project |
|
||||
| `api/retentionpolicies/{id}` | PUT | Project |
|
||||
| `api/retentionpolicies/{id}` | DELETE | Project |
|
||||
| `api/retentionpolicies/resources` | GET | Public |
|
||||
| `api/retentionpolicies/template` | GET | Project |
|
||||
|
||||
---
|
||||
|
||||
## roles
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/roles` | GET | Read |
|
||||
| `api/roles/{id}` | GET | Read |
|
||||
| `api/roles/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## runningtasks
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/runningtasks` | GET | System |
|
||||
| `api/runningtasks/{id}` | GET | System |
|
||||
| `api/runningtasks/{id}` | DELETE | System |
|
||||
| `api/runningtasks/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## settings
|
||||
|
||||
Server settings. Most require System permission. Notable publicly accessible settings:
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/settings/{id}` | GET | System | Generic setting |
|
||||
| `api/settings/{id}` | PUT | System | |
|
||||
| `api/settings/setting-authenticationprovider` | GET | Public | |
|
||||
| `api/settings/setting-instancetitle` | GET | Public | |
|
||||
| `api/settings/setting-isauthenticationenabled` | GET | Public | |
|
||||
| `api/settings/setting-isactivedirectoryauthentication` | GET | Public | |
|
||||
| `api/settings/setting-isusagetelemetryenabled` | GET | Read | |
|
||||
| `api/settings/setting-searchdurationseconds` | GET | Read | |
|
||||
| `api/settings/setting-searchdurationseconds` | PUT | System | |
|
||||
| `api/settings/setting-servicenameexpression` | GET | Read | |
|
||||
| `api/settings/setting-servicenameexpression` | PUT | Project | |
|
||||
| `api/settings/setting-requireapikeyforwritingevents` | GET | Project | |
|
||||
| `api/settings/setting-requireapikeyforwritingevents` | PUT | Project | |
|
||||
| `api/settings/setting-newusershowdashboardids` | GET/PUT | Organization | |
|
||||
| `api/settings/setting-newusershowqueryids` | GET/PUT | Organization | |
|
||||
| `api/settings/setting-newusershowsignalids` | GET/PUT | Organization | |
|
||||
| `api/settings/setting-checkforupdates` | GET/PUT | System | |
|
||||
| `api/settings/setting-minimumfreestoragespace` | GET/PUT | System | |
|
||||
| `api/settings/setting-raweventmaximumcontentlength` | GET/PUT | System | |
|
||||
| `api/settings/setting-rawpayloadmaximumcontentlength` | GET/PUT | System | |
|
||||
| `api/settings/setting-themestyles` | GET/PUT | System | |
|
||||
| `api/settings/internal-error-reporting` | GET/PUT | System | |
|
||||
| `api/settings/resources` | GET | Public | |
|
||||
|
||||
---
|
||||
|
||||
## signals
|
||||
|
||||
Saved signals. Users can only access shared signals and their own. Protected signals require `Project` permission.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/signals` | GET | Read | Shared + own only |
|
||||
| `api/signals` | POST | Write | Project for protected |
|
||||
| `api/signals/{id}` | GET | Read | Shared + own only |
|
||||
| `api/signals/{id}` | PUT | Write | Project for protected |
|
||||
| `api/signals/{id}` | DELETE | Write | Project for protected |
|
||||
| `api/signals/resources` | GET | Public | |
|
||||
| `api/signals/template` | GET | Write | |
|
||||
|
||||
---
|
||||
|
||||
## sqlqueries
|
||||
|
||||
Saved SQL queries. Same ownership/sharing rules as signals.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/sqlqueries` | GET | Read | Shared + own only |
|
||||
| `api/sqlqueries` | POST | Write | Project for protected |
|
||||
| `api/sqlqueries/{id}` | GET | Read | Shared + own only |
|
||||
| `api/sqlqueries/{id}` | PUT | Write | Project for protected |
|
||||
| `api/sqlqueries/{id}` | DELETE | Write | Project for protected |
|
||||
| `api/sqlqueries/resources` | GET | Public | |
|
||||
| `api/sqlqueries/template` | GET | Write | |
|
||||
|
||||
---
|
||||
|
||||
## updates
|
||||
|
||||
| Path | Method | Permission |
|
||||
|------|--------|------------|
|
||||
| `api/updates` | GET | System |
|
||||
| `api/updates/{id}` | GET | System |
|
||||
| `api/updates/resources` | GET | Public |
|
||||
|
||||
---
|
||||
|
||||
## users
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/users` | GET | Project | System for auth provider info |
|
||||
| `api/users` | POST | Organization | Cannot grant permissions you don't have |
|
||||
| `api/users/{id}` | GET | Public | Own record; Project for others |
|
||||
| `api/users/{id}` | PUT | Public | Own limited fields; Organization for others |
|
||||
| `api/users/{id}` | DELETE | Organization | |
|
||||
| `api/users/{id}/searches` | GET | Read | Own search history only |
|
||||
| `api/users/{id}/searches` | DELETE | Write | Own search history only |
|
||||
| `api/users/{id}/searches/update` | POST | Write | Own search history only |
|
||||
| `api/users/{id}/unlinkauthenticationprovider` | POST | System | |
|
||||
| `api/users/current` | GET | Public | Logged-in user only |
|
||||
| `api/users/login` | POST | Public | |
|
||||
| `api/users/logout` | POST | Public | Allows cross-site POSTs |
|
||||
| `api/users/providers` | GET | Public | |
|
||||
| `api/users/resources` | GET | Public | |
|
||||
| `api/users/template` | GET | Organization | |
|
||||
|
||||
---
|
||||
|
||||
## workspaces
|
||||
|
||||
Same ownership/sharing rules as signals, dashboards, etc.
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `api/workspaces` | GET | Read | Shared + own only |
|
||||
| `api/workspaces` | POST | Write | Project for protected |
|
||||
| `api/workspaces/{id}` | GET | Read | Shared + own only |
|
||||
| `api/workspaces/{id}` | PUT | Write | Project for protected |
|
||||
| `api/workspaces/{id}` | DELETE | Write | Project for protected |
|
||||
| `api/workspaces/resources` | GET | Public | |
|
||||
| `api/workspaces/template` | GET | Write | |
|
||||
|
||||
---
|
||||
|
||||
## health
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `health` | GET | Public | Returns 200 or 503 |
|
||||
| `health/cluster` | GET | Public | Cluster health |
|
||||
|
||||
---
|
||||
|
||||
## ingestion
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `ingest/clef` | POST | Public* | CLEF format; cross-site allowed |
|
||||
| `ingest/otlp/v1/logs` | POST | Public* | OpenTelemetry logs |
|
||||
| `ingest/otlp/v1/traces` | POST | Public* | OpenTelemetry traces |
|
||||
| `ingest/otlp/v1/metrics` | POST | Public* | OpenTelemetry metrics |
|
||||
|
||||
*If `RequireApiKeyForWritingEvents` is enabled, Ingest permission is required.
|
||||
|
||||
---
|
||||
|
||||
## other
|
||||
|
||||
| Path | Method | Permission | Notes |
|
||||
|------|--------|------------|-------|
|
||||
| `integrated` | GET | Public | Windows integrated auth |
|
||||
| `oidc/challenge` | GET | Public | OpenID Connect |
|
||||
| `oidc/challenge` | POST | Public | OpenID Connect |
|
||||
| `theme/styles.css` | GET | Public | Custom theme CSS |
|
||||
144
skills/seq-api/references/ingestion.md
Normal file
144
skills/seq-api/references/ingestion.md
Normal file
@@ -0,0 +1,144 @@
|
||||
# Seq Ingestion Reference — CLEF Format & HTTP Details
|
||||
|
||||
## Endpoint
|
||||
|
||||
```
|
||||
POST {SEQ_URL}/ingest/clef
|
||||
```
|
||||
|
||||
## Headers
|
||||
|
||||
| Header | Value | Required |
|
||||
|--------|-------|----------|
|
||||
| `Content-Type` | `application/vnd.serilog.clef` (batch) or `application/json` (single event) | Yes |
|
||||
| `X-Seq-ApiKey` | Your API key | Only if `RequireApiKeyForWritingEvents` is enabled |
|
||||
|
||||
Alternatively, the API key can be passed as a query parameter: `?apiKey={key}`
|
||||
|
||||
## CLEF Format
|
||||
|
||||
Events are newline-delimited JSON documents (one JSON object per line). Each object represents one log event.
|
||||
|
||||
### Batch Example
|
||||
|
||||
```
|
||||
{"@t":"2024-01-15T10:30:00.000Z","@mt":"Hello, {User}","User":"alice"}
|
||||
{"@t":"2024-01-15T10:30:01.123Z","@mt":"Processing order {OrderId}","OrderId":42,"@l":"Information"}
|
||||
{"@t":"2024-01-15T10:30:02.456Z","@mt":"Failed to process {OrderId}","OrderId":42,"@l":"Error","@x":"System.Exception: Something went wrong\n at MyApp.OrderProcessor.Process()"}
|
||||
```
|
||||
|
||||
### Reified Properties (Special @ Properties)
|
||||
|
||||
Any JSON property at the top level is treated as a regular event property, **except** the following special properties:
|
||||
|
||||
| Property | Name | Description | Required? |
|
||||
|----------|------|-------------|-----------|
|
||||
| `@t` | Timestamp | ISO 8601 timestamp | **Yes** |
|
||||
| `@m` | Message | Fully-rendered message text | No (use `@mt` or `@m`) |
|
||||
| `@mt` | Message Template | [Message template](http://messagetemplates.org) with named holes like `{User}` | No (alternative to `@m`) |
|
||||
| `@l` | Level | Log level string: `Verbose`, `Debug`, `Information`, `Warning`, `Error`, `Fatal` | No (defaults to Information) |
|
||||
| `@x` | Exception | Error/backtrace as a string | No |
|
||||
| `@i` | Event ID | Event type identifier (numeric or hex string) | No |
|
||||
| `@r` | Renderings | Pre-rendered values for format tokens in `@mt` | No |
|
||||
| `@tr` | Trace ID | Groups spans/logs in the same trace | Required for spans |
|
||||
| `@sp` | Span ID | Unique span identifier | Required for spans |
|
||||
| `@ps` | Parent Span ID | Parent span's ID; absent = root span | No |
|
||||
| `@st` | Span Start | ISO 8601 start timestamp of the span | Required for spans |
|
||||
| `@sc` | Instrumentation Scope | App-local component name | No |
|
||||
| `@ra` | Resource Attributes | System-level component descriptor | No |
|
||||
| `@sk` | Span Kind | `Client`, `Server`, `Internal`, `Producer`, or `Consumer` | No |
|
||||
|
||||
### Escaping @ in Property Names
|
||||
|
||||
To use a property name starting with `@`, double it: `@@myProp` becomes `@myProp` in Seq.
|
||||
|
||||
### Batch Delimiters
|
||||
|
||||
Use `\n` or `\r\n` between JSON objects. No trailing delimiter is required but is harmless.
|
||||
|
||||
## Status Codes
|
||||
|
||||
| Code | Meaning |
|
||||
|------|---------|
|
||||
| **201** Created | Events ingested successfully |
|
||||
| **400** Bad Request | Malformed payload or event exceeds max size |
|
||||
| **401** Unauthorized | API key missing or invalid |
|
||||
| **403** Forbidden | API key lacks Ingest permission |
|
||||
| **413** Request Entity Too Large | Payload exceeds configured max size |
|
||||
| **500** Internal Server Error | Server-side error; check Seq diagnostics |
|
||||
| **503** Service Unavailable | Server starting up, or storage space below threshold |
|
||||
|
||||
## Response Format
|
||||
|
||||
### Success (201)
|
||||
|
||||
```json
|
||||
{"MinimumLevelAccepted": null}
|
||||
```
|
||||
|
||||
`MinimumLevelAccepted` will be one of `Verbose`, `Debug`, `Information`, `Warning`, `Error`, `Fatal` if a level filter is applied to the API key, or `null` if no filtering. Clients can use this to pre-filter events and reduce bandwidth.
|
||||
|
||||
### Error (4xx/5xx)
|
||||
|
||||
```json
|
||||
{"Error": "Description of what went wrong"}
|
||||
```
|
||||
|
||||
## OpenTelemetry Ingestion
|
||||
|
||||
Seq also accepts OpenTelemetry Protocol (OTLP) payloads:
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `ingest/otlp/v1/logs` | OTLP logs |
|
||||
| `ingest/otlp/v1/traces` | OTLP traces |
|
||||
| `ingest/otlp/v1/metrics` | OTLP metrics |
|
||||
|
||||
These follow the standard OTLP HTTP specification. The same API key authentication rules apply.
|
||||
|
||||
## Raw Events Endpoint (Legacy)
|
||||
|
||||
The older `api/events/raw` endpoint also accepts event ingestion with cross-site POST support. The `/ingest/clef` endpoint is preferred for new integrations.
|
||||
|
||||
## Common curl Examples
|
||||
|
||||
### Send a single event
|
||||
|
||||
```bash
|
||||
curl -X POST "https://seq.example.com/ingest/clef" \
|
||||
-H "Content-Type: application/vnd.serilog.clef" \
|
||||
-H "X-Seq-ApiKey: YOUR_API_KEY" \
|
||||
-d '{"@t":"2024-01-15T10:30:00Z","@mt":"Deployment started for {App}","App":"myservice","@l":"Information"}'
|
||||
```
|
||||
|
||||
### Send a batch
|
||||
|
||||
```bash
|
||||
curl -X POST "https://seq.example.com/ingest/clef" \
|
||||
-H "Content-Type: application/vnd.serilog.clef" \
|
||||
-H "X-Seq-ApiKey: YOUR_API_KEY" \
|
||||
-d '{"@t":"2024-01-15T10:30:00Z","@mt":"Step 1 complete","@l":"Information"}
|
||||
{"@t":"2024-01-15T10:30:01Z","@mt":"Step 2 complete","@l":"Information"}
|
||||
{"@t":"2024-01-15T10:30:02Z","@mt":"All steps done","@l":"Information"}'
|
||||
```
|
||||
|
||||
### Send with an API key in the query string
|
||||
|
||||
```bash
|
||||
curl -X POST "https://seq.example.com/ingest/clef?apiKey=YOUR_API_KEY" \
|
||||
-H "Content-Type: application/vnd.serilog.clef" \
|
||||
-d '{"@t":"2024-01-15T10:30:00Z","@mt":"Hello from curl"}'
|
||||
```
|
||||
|
||||
### Query events via the data API
|
||||
|
||||
```bash
|
||||
curl "https://seq.example.com/api/data?q=select%20count(*)%20from%20stream%20group%20by%20%40Level&rangeStartUtc=2024-01-01&rangeEndUtc=2024-01-02" \
|
||||
-H "X-Seq-ApiKey: YOUR_API_KEY"
|
||||
```
|
||||
|
||||
### Check server health
|
||||
|
||||
```bash
|
||||
curl https://seq.example.com/health
|
||||
```
|
||||
202
skills/skill-creator/LICENSE.txt
Normal file
202
skills/skill-creator/LICENSE.txt
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
485
skills/skill-creator/SKILL.md
Normal file
485
skills/skill-creator/SKILL.md
Normal file
@@ -0,0 +1,485 @@
|
||||
---
|
||||
name: skill-creator
|
||||
description: Create new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.
|
||||
---
|
||||
|
||||
# Skill Creator
|
||||
|
||||
A skill for creating new skills and iteratively improving them.
|
||||
|
||||
At a high level, the process of creating a skill goes like this:
|
||||
|
||||
- Decide what you want the skill to do and roughly how it should do it
|
||||
- Write a draft of the skill
|
||||
- Create a few test prompts and run claude-with-access-to-the-skill on them
|
||||
- Help the user evaluate the results both qualitatively and quantitatively
|
||||
- While the runs happen in the background, draft some quantitative evals if there aren't any (if there are some, you can either use as is or modify if you feel something needs to change about them). Then explain them to the user (or if they already existed, explain the ones that already exist)
|
||||
- Use the `eval-viewer/generate_review.py` script to show the user the results for them to look at, and also let them look at the quantitative metrics
|
||||
- Rewrite the skill based on feedback from the user's evaluation of the results (and also if there are any glaring flaws that become apparent from the quantitative benchmarks)
|
||||
- Repeat until you're satisfied
|
||||
- Expand the test set and try again at larger scale
|
||||
|
||||
Your job when using this skill is to figure out where the user is in this process and then jump in and help them progress through these stages. So for instance, maybe they're like "I want to make a skill for X". You can help narrow down what they mean, write a draft, write the test cases, figure out how they want to evaluate, run all the prompts, and repeat.
|
||||
|
||||
On the other hand, maybe they already have a draft of the skill. In this case you can go straight to the eval/iterate part of the loop.
|
||||
|
||||
Of course, you should always be flexible and if the user is like "I don't need to run a bunch of evaluations, just vibe with me", you can do that instead.
|
||||
|
||||
Then after the skill is done (but again, the order is flexible), you can also run the skill description improver, which we have a whole separate script for, to optimize the triggering of the skill.
|
||||
|
||||
Cool? Cool.
|
||||
|
||||
## Communicating with the user
|
||||
|
||||
The skill creator is liable to be used by people across a wide range of familiarity with coding jargon. If you haven't heard (and how could you, it's only very recently that it started), there's a trend now where the power of Claude is inspiring plumbers to open up their terminals, parents and grandparents to google "how to install npm". On the other hand, the bulk of users are probably fairly computer-literate.
|
||||
|
||||
So please pay attention to context cues to understand how to phrase your communication! In the default case, just to give you some idea:
|
||||
|
||||
- "evaluation" and "benchmark" are borderline, but OK
|
||||
- for "JSON" and "assertion" you want to see serious cues from the user that they know what those things are before using them without explaining them
|
||||
|
||||
It's OK to briefly explain terms if you're in doubt, and feel free to clarify terms with a short definition if you're unsure if the user will get it.
|
||||
|
||||
---
|
||||
|
||||
## Creating a skill
|
||||
|
||||
### Capture Intent
|
||||
|
||||
Start by understanding the user's intent. The current conversation might already contain a workflow the user wants to capture (e.g., they say "turn this into a skill"). If so, extract answers from the conversation history first — the tools used, the sequence of steps, corrections the user made, input/output formats observed. The user may need to fill the gaps, and should confirm before proceeding to the next step.
|
||||
|
||||
1. What should this skill enable Claude to do?
|
||||
2. When should this skill trigger? (what user phrases/contexts)
|
||||
3. What's the expected output format?
|
||||
4. Should we set up test cases to verify the skill works? Skills with objectively verifiable outputs (file transforms, data extraction, code generation, fixed workflow steps) benefit from test cases. Skills with subjective outputs (writing style, art) often don't need them. Suggest the appropriate default based on the skill type, but let the user decide.
|
||||
|
||||
### Interview and Research
|
||||
|
||||
Proactively ask questions about edge cases, input/output formats, example files, success criteria, and dependencies. Wait to write test prompts until you've got this part ironed out.
|
||||
|
||||
Check available MCPs - if useful for research (searching docs, finding similar skills, looking up best practices), research in parallel via subagents if available, otherwise inline. Come prepared with context to reduce burden on the user.
|
||||
|
||||
### Write the SKILL.md
|
||||
|
||||
Based on the user interview, fill in these components:
|
||||
|
||||
- **name**: Skill identifier
|
||||
- **description**: When to trigger, what it does. This is the primary triggering mechanism - include both what the skill does AND specific contexts for when to use it. All "when to use" info goes here, not in the body. Note: currently Claude has a tendency to "undertrigger" skills -- to not use them when they'd be useful. To combat this, please make the skill descriptions a little bit "pushy". So for instance, instead of "How to build a simple fast dashboard to display internal Anthropic data.", you might write "How to build a simple fast dashboard to display internal Anthropic data. Make sure to use this skill whenever the user mentions dashboards, data visualization, internal metrics, or wants to display any kind of company data, even if they don't explicitly ask for a 'dashboard.'"
|
||||
- **compatibility**: Required tools, dependencies (optional, rarely needed)
|
||||
- **the rest of the skill :)**
|
||||
|
||||
### Skill Writing Guide
|
||||
|
||||
#### Anatomy of a Skill
|
||||
|
||||
```
|
||||
skill-name/
|
||||
├── SKILL.md (required)
|
||||
│ ├── YAML frontmatter (name, description required)
|
||||
│ └── Markdown instructions
|
||||
└── Bundled Resources (optional)
|
||||
├── scripts/ - Executable code for deterministic/repetitive tasks
|
||||
├── references/ - Docs loaded into context as needed
|
||||
└── assets/ - Files used in output (templates, icons, fonts)
|
||||
```
|
||||
|
||||
#### Progressive Disclosure
|
||||
|
||||
Skills use a three-level loading system:
|
||||
1. **Metadata** (name + description) - Always in context (~100 words)
|
||||
2. **SKILL.md body** - In context whenever skill triggers (<500 lines ideal)
|
||||
3. **Bundled resources** - As needed (unlimited, scripts can execute without loading)
|
||||
|
||||
These word counts are approximate and you can feel free to go longer if needed.
|
||||
|
||||
**Key patterns:**
|
||||
- Keep SKILL.md under 500 lines; if you're approaching this limit, add an additional layer of hierarchy along with clear pointers about where the model using the skill should go next to follow up.
|
||||
- Reference files clearly from SKILL.md with guidance on when to read them
|
||||
- For large reference files (>300 lines), include a table of contents
|
||||
|
||||
**Domain organization**: When a skill supports multiple domains/frameworks, organize by variant:
|
||||
```
|
||||
cloud-deploy/
|
||||
├── SKILL.md (workflow + selection)
|
||||
└── references/
|
||||
├── aws.md
|
||||
├── gcp.md
|
||||
└── azure.md
|
||||
```
|
||||
Claude reads only the relevant reference file.
|
||||
|
||||
#### Principle of Lack of Surprise
|
||||
|
||||
This goes without saying, but skills must not contain malware, exploit code, or any content that could compromise system security. A skill's contents should not surprise the user in their intent if described. Don't go along with requests to create misleading skills or skills designed to facilitate unauthorized access, data exfiltration, or other malicious activities. Things like a "roleplay as an XYZ" are OK though.
|
||||
|
||||
#### Writing Patterns
|
||||
|
||||
Prefer using the imperative form in instructions.
|
||||
|
||||
**Defining output formats** - You can do it like this:
|
||||
```markdown
|
||||
## Report structure
|
||||
ALWAYS use this exact template:
|
||||
# [Title]
|
||||
## Executive summary
|
||||
## Key findings
|
||||
## Recommendations
|
||||
```
|
||||
|
||||
**Examples pattern** - It's useful to include examples. You can format them like this (but if "Input" and "Output" are in the examples you might want to deviate a little):
|
||||
```markdown
|
||||
## Commit message format
|
||||
**Example 1:**
|
||||
Input: Added user authentication with JWT tokens
|
||||
Output: feat(auth): implement JWT-based authentication
|
||||
```
|
||||
|
||||
### Writing Style
|
||||
|
||||
Try to explain to the model why things are important in lieu of heavy-handed musty MUSTs. Use theory of mind and try to make the skill general and not super-narrow to specific examples. Start by writing a draft and then look at it with fresh eyes and improve it.
|
||||
|
||||
### Test Cases
|
||||
|
||||
After writing the skill draft, come up with 2-3 realistic test prompts — the kind of thing a real user would actually say. Share them with the user: [you don't have to use this exact language] "Here are a few test cases I'd like to try. Do these look right, or do you want to add more?" Then run them.
|
||||
|
||||
Save test cases to `evals/evals.json`. Don't write assertions yet — just the prompts. You'll draft assertions in the next step while the runs are in progress.
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "example-skill",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"prompt": "User's task prompt",
|
||||
"expected_output": "Description of expected result",
|
||||
"files": []
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
See `references/schemas.md` for the full schema (including the `assertions` field, which you'll add later).
|
||||
|
||||
## Running and evaluating test cases
|
||||
|
||||
This section is one continuous sequence — don't stop partway through. Do NOT use `/skill-test` or any other testing skill.
|
||||
|
||||
Put results in `<skill-name>-workspace/` as a sibling to the skill directory. Within the workspace, organize results by iteration (`iteration-1/`, `iteration-2/`, etc.) and within that, each test case gets a directory (`eval-0/`, `eval-1/`, etc.). Don't create all of this upfront — just create directories as you go.
|
||||
|
||||
### Step 1: Spawn all runs (with-skill AND baseline) in the same turn
|
||||
|
||||
For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time.
|
||||
|
||||
**With-skill run:**
|
||||
|
||||
```
|
||||
Execute this task:
|
||||
- Skill path: <path-to-skill>
|
||||
- Task: <eval prompt>
|
||||
- Input files: <eval files if any, or "none">
|
||||
- Save outputs to: <workspace>/iteration-<N>/eval-<ID>/with_skill/outputs/
|
||||
- Outputs to save: <what the user cares about — e.g., "the .docx file", "the final CSV">
|
||||
```
|
||||
|
||||
**Baseline run** (same prompt, but the baseline depends on context):
|
||||
- **Creating a new skill**: no skill at all. Same prompt, no skill path, save to `without_skill/outputs/`.
|
||||
- **Improving an existing skill**: the old version. Before editing, snapshot the skill (`cp -r <skill-path> <workspace>/skill-snapshot/`), then point the baseline subagent at the snapshot. Save to `old_skill/outputs/`.
|
||||
|
||||
Write an `eval_metadata.json` for each test case (assertions can be empty for now). Give each eval a descriptive name based on what it's testing — not just "eval-0". Use this name for the directory too. If this iteration uses new or modified eval prompts, create these files for each new eval directory — don't assume they carry over from previous iterations.
|
||||
|
||||
```json
|
||||
{
|
||||
"eval_id": 0,
|
||||
"eval_name": "descriptive-name-here",
|
||||
"prompt": "The user's task prompt",
|
||||
"assertions": []
|
||||
}
|
||||
```
|
||||
|
||||
### Step 2: While runs are in progress, draft assertions
|
||||
|
||||
Don't just wait for the runs to finish — you can use this time productively. Draft quantitative assertions for each test case and explain them to the user. If assertions already exist in `evals/evals.json`, review them and explain what they check.
|
||||
|
||||
Good assertions are objectively verifiable and have descriptive names — they should read clearly in the benchmark viewer so someone glancing at the results immediately understands what each one checks. Subjective skills (writing style, design quality) are better evaluated qualitatively — don't force assertions onto things that need human judgment.
|
||||
|
||||
Update the `eval_metadata.json` files and `evals/evals.json` with the assertions once drafted. Also explain to the user what they'll see in the viewer — both the qualitative outputs and the quantitative benchmark.
|
||||
|
||||
### Step 3: As runs complete, capture timing data
|
||||
|
||||
When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms`. Save this data immediately to `timing.json` in the run directory:
|
||||
|
||||
```json
|
||||
{
|
||||
"total_tokens": 84852,
|
||||
"duration_ms": 23332,
|
||||
"total_duration_seconds": 23.3
|
||||
}
|
||||
```
|
||||
|
||||
This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives rather than trying to batch them.
|
||||
|
||||
### Step 4: Grade, aggregate, and launch the viewer
|
||||
|
||||
Once all runs are done:
|
||||
|
||||
1. **Grade each run** — spawn a grader subagent (or grade inline) that reads `agents/grader.md` and evaluates each assertion against the outputs. Save results to `grading.json` in each run directory. The grading.json expectations array must use the fields `text`, `passed`, and `evidence` (not `name`/`met`/`details` or other variants) — the viewer depends on these exact field names. For assertions that can be checked programmatically, write and run a script rather than eyeballing it — scripts are faster, more reliable, and can be reused across iterations.
|
||||
|
||||
2. **Aggregate into benchmark** — run the aggregation script from the skill-creator directory:
|
||||
```bash
|
||||
python -m scripts.aggregate_benchmark <workspace>/iteration-N --skill-name <name>
|
||||
```
|
||||
This produces `benchmark.json` and `benchmark.md` with pass_rate, time, and tokens for each configuration, with mean ± stddev and the delta. If generating benchmark.json manually, see `references/schemas.md` for the exact schema the viewer expects.
|
||||
Put each with_skill version before its baseline counterpart.
|
||||
|
||||
3. **Do an analyst pass** — read the benchmark data and surface patterns the aggregate stats might hide. See `agents/analyzer.md` (the "Analyzing Benchmark Results" section) for what to look for — things like assertions that always pass regardless of skill (non-discriminating), high-variance evals (possibly flaky), and time/token tradeoffs.
|
||||
|
||||
4. **Launch the viewer** with both qualitative outputs and quantitative data:
|
||||
```bash
|
||||
nohup python <skill-creator-path>/eval-viewer/generate_review.py \
|
||||
<workspace>/iteration-N \
|
||||
--skill-name "my-skill" \
|
||||
--benchmark <workspace>/iteration-N/benchmark.json \
|
||||
> /dev/null 2>&1 &
|
||||
VIEWER_PID=$!
|
||||
```
|
||||
For iteration 2+, also pass `--previous-workspace <workspace>/iteration-<N-1>`.
|
||||
|
||||
**Cowork / headless environments:** If `webbrowser.open()` is not available or the environment has no display, use `--static <output_path>` to write a standalone HTML file instead of starting a server. Feedback will be downloaded as a `feedback.json` file when the user clicks "Submit All Reviews". After download, copy `feedback.json` into the workspace directory for the next iteration to pick up.
|
||||
|
||||
Note: please use generate_review.py to create the viewer; there's no need to write custom HTML.
|
||||
|
||||
5. **Tell the user** something like: "I've opened the results in your browser. There are two tabs — 'Outputs' lets you click through each test case and leave feedback, 'Benchmark' shows the quantitative comparison. When you're done, come back here and let me know."
|
||||
|
||||
### What the user sees in the viewer
|
||||
|
||||
The "Outputs" tab shows one test case at a time:
|
||||
- **Prompt**: the task that was given
|
||||
- **Output**: the files the skill produced, rendered inline where possible
|
||||
- **Previous Output** (iteration 2+): collapsed section showing last iteration's output
|
||||
- **Formal Grades** (if grading was run): collapsed section showing assertion pass/fail
|
||||
- **Feedback**: a textbox that auto-saves as they type
|
||||
- **Previous Feedback** (iteration 2+): their comments from last time, shown below the textbox
|
||||
|
||||
The "Benchmark" tab shows the stats summary: pass rates, timing, and token usage for each configuration, with per-eval breakdowns and analyst observations.
|
||||
|
||||
Navigation is via prev/next buttons or arrow keys. When done, they click "Submit All Reviews" which saves all feedback to `feedback.json`.
|
||||
|
||||
### Step 5: Read the feedback
|
||||
|
||||
When the user tells you they're done, read `feedback.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"reviews": [
|
||||
{"run_id": "eval-0-with_skill", "feedback": "the chart is missing axis labels", "timestamp": "..."},
|
||||
{"run_id": "eval-1-with_skill", "feedback": "", "timestamp": "..."},
|
||||
{"run_id": "eval-2-with_skill", "feedback": "perfect, love this", "timestamp": "..."}
|
||||
],
|
||||
"status": "complete"
|
||||
}
|
||||
```
|
||||
|
||||
Empty feedback means the user thought it was fine. Focus your improvements on the test cases where the user had specific complaints.
|
||||
|
||||
Kill the viewer server when you're done with it:
|
||||
|
||||
```bash
|
||||
kill $VIEWER_PID 2>/dev/null
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Improving the skill
|
||||
|
||||
This is the heart of the loop. You've run the test cases, the user has reviewed the results, and now you need to make the skill better based on their feedback.
|
||||
|
||||
### How to think about improvements
|
||||
|
||||
1. **Generalize from the feedback.** The big picture thing that's happening here is that we're trying to create skills that can be used a million times (maybe literally, maybe even more who knows) across many different prompts. Here you and the user are iterating on only a few examples over and over again because it helps move faster. The user knows these examples in and out and it's quick for them to assess new outputs. But if the skill you and the user are codeveloping works only for those examples, it's useless. Rather than put in fiddly overfitty changes, or oppressively constrictive MUSTs, if there's some stubborn issue, you might try branching out and using different metaphors, or recommending different patterns of working. It's relatively cheap to try and maybe you'll land on something great.
|
||||
|
||||
2. **Keep the prompt lean.** Remove things that aren't pulling their weight. Make sure to read the transcripts, not just the final outputs — if it looks like the skill is making the model waste a bunch of time doing things that are unproductive, you can try getting rid of the parts of the skill that are making it do that and seeing what happens.
|
||||
|
||||
3. **Explain the why.** Try hard to explain the **why** behind everything you're asking the model to do. Today's LLMs are *smart*. They have good theory of mind and when given a good harness can go beyond rote instructions and really make things happen. Even if the feedback from the user is terse or frustrated, try to actually understand the task and why the user is writing what they wrote, and what they actually wrote, and then transmit this understanding into the instructions. If you find yourself writing ALWAYS or NEVER in all caps, or using super rigid structures, that's a yellow flag — if possible, reframe and explain the reasoning so that the model understands why the thing you're asking for is important. That's a more humane, powerful, and effective approach.
|
||||
|
||||
4. **Look for repeated work across test cases.** Read the transcripts from the test runs and notice if the subagents all independently wrote similar helper scripts or took the same multi-step approach to something. If all 3 test cases resulted in the subagent writing a `create_docx.py` or a `build_chart.py`, that's a strong signal the skill should bundle that script. Write it once, put it in `scripts/`, and tell the skill to use it. This saves every future invocation from reinventing the wheel.
|
||||
|
||||
This task is pretty important (we are trying to create billions a year in economic value here!) and your thinking time is not the blocker; take your time and really mull things over. I'd suggest writing a draft revision and then looking at it anew and making improvements. Really do your best to get into the head of the user and understand what they want and need.
|
||||
|
||||
### The iteration loop
|
||||
|
||||
After improving the skill:
|
||||
|
||||
1. Apply your improvements to the skill
|
||||
2. Rerun all test cases into a new `iteration-<N+1>/` directory, including baseline runs. If you're creating a new skill, the baseline is always `without_skill` (no skill) — that stays the same across iterations. If you're improving an existing skill, use your judgment on what makes sense as the baseline: the original version the user came in with, or the previous iteration.
|
||||
3. Launch the reviewer with `--previous-workspace` pointing at the previous iteration
|
||||
4. Wait for the user to review and tell you they're done
|
||||
5. Read the new feedback, improve again, repeat
|
||||
|
||||
Keep going until:
|
||||
- The user says they're happy
|
||||
- The feedback is all empty (everything looks good)
|
||||
- You're not making meaningful progress
|
||||
|
||||
---
|
||||
|
||||
## Advanced: Blind comparison
|
||||
|
||||
For situations where you want a more rigorous comparison between two versions of a skill (e.g., the user asks "is the new version actually better?"), there's a blind comparison system. Read `agents/comparator.md` and `agents/analyzer.md` for the details. The basic idea is: give two outputs to an independent agent without telling it which is which, and let it judge quality. Then analyze why the winner won.
|
||||
|
||||
This is optional, requires subagents, and most users won't need it. The human review loop is usually sufficient.
|
||||
|
||||
---
|
||||
|
||||
## Description Optimization
|
||||
|
||||
The description field in SKILL.md frontmatter is the primary mechanism that determines whether Claude invokes a skill. After creating or improving a skill, offer to optimize the description for better triggering accuracy.
|
||||
|
||||
### Step 1: Generate trigger eval queries
|
||||
|
||||
Create 20 eval queries — a mix of should-trigger and should-not-trigger. Save as JSON:
|
||||
|
||||
```json
|
||||
[
|
||||
{"query": "the user prompt", "should_trigger": true},
|
||||
{"query": "another prompt", "should_trigger": false}
|
||||
]
|
||||
```
|
||||
|
||||
The queries must be realistic and something a Claude Code or Claude.ai user would actually type. Not abstract requests, but requests that are concrete and specific and have a good amount of detail. For instance, file paths, personal context about the user's job or situation, column names and values, company names, URLs. A little bit of backstory. Some might be in lowercase or contain abbreviations or typos or casual speech. Use a mix of different lengths, and focus on edge cases rather than making them clear-cut (the user will get a chance to sign off on them).
|
||||
|
||||
Bad: `"Format this data"`, `"Extract text from PDF"`, `"Create a chart"`
|
||||
|
||||
Good: `"ok so my boss just sent me this xlsx file (its in my downloads, called something like 'Q4 sales final FINAL v2.xlsx') and she wants me to add a column that shows the profit margin as a percentage. The revenue is in column C and costs are in column D i think"`
|
||||
|
||||
For the **should-trigger** queries (8-10), think about coverage. You want different phrasings of the same intent — some formal, some casual. Include cases where the user doesn't explicitly name the skill or file type but clearly needs it. Throw in some uncommon use cases and cases where this skill competes with another but should win.
|
||||
|
||||
For the **should-not-trigger** queries (8-10), the most valuable ones are the near-misses — queries that share keywords or concepts with the skill but actually need something different. Think adjacent domains, ambiguous phrasing where a naive keyword match would trigger but shouldn't, and cases where the query touches on something the skill does but in a context where another tool is more appropriate.
|
||||
|
||||
The key thing to avoid: don't make should-not-trigger queries obviously irrelevant. "Write a fibonacci function" as a negative test for a PDF skill is too easy — it doesn't test anything. The negative cases should be genuinely tricky.
|
||||
|
||||
### Step 2: Review with user
|
||||
|
||||
Present the eval set to the user for review using the HTML template:
|
||||
|
||||
1. Read the template from `assets/eval_review.html`
|
||||
2. Replace the placeholders:
|
||||
- `__EVAL_DATA_PLACEHOLDER__` → the JSON array of eval items (no quotes around it — it's a JS variable assignment)
|
||||
- `__SKILL_NAME_PLACEHOLDER__` → the skill's name
|
||||
- `__SKILL_DESCRIPTION_PLACEHOLDER__` → the skill's current description
|
||||
3. Write to a temp file (e.g., `/tmp/eval_review_<skill-name>.html`) and open it: `open /tmp/eval_review_<skill-name>.html`
|
||||
4. The user can edit queries, toggle should-trigger, add/remove entries, then click "Export Eval Set"
|
||||
5. The file downloads to `~/Downloads/eval_set.json` — check the Downloads folder for the most recent version in case there are multiple (e.g., `eval_set (1).json`)
|
||||
|
||||
This step matters — bad eval queries lead to bad descriptions.
|
||||
|
||||
### Step 3: Run the optimization loop
|
||||
|
||||
Tell the user: "This will take some time — I'll run the optimization loop in the background and check on it periodically."
|
||||
|
||||
Save the eval set to the workspace, then run in the background:
|
||||
|
||||
```bash
|
||||
python -m scripts.run_loop \
|
||||
--eval-set <path-to-trigger-eval.json> \
|
||||
--skill-path <path-to-skill> \
|
||||
--model <model-id-powering-this-session> \
|
||||
--max-iterations 5 \
|
||||
--verbose
|
||||
```
|
||||
|
||||
Use the model ID from your system prompt (the one powering the current session) so the triggering test matches what the user actually experiences.
|
||||
|
||||
While it runs, periodically tail the output to give the user updates on which iteration it's on and what the scores look like.
|
||||
|
||||
This handles the full optimization loop automatically. It splits the eval set into 60% train and 40% held-out test, evaluates the current description (running each query 3 times to get a reliable trigger rate), then calls Claude to propose improvements based on what failed. It re-evaluates each new description on both train and test, iterating up to 5 times. When it's done, it opens an HTML report in the browser showing the results per iteration and returns JSON with `best_description` — selected by test score rather than train score to avoid overfitting.
|
||||
|
||||
### How skill triggering works
|
||||
|
||||
Understanding the triggering mechanism helps design better eval queries. Skills appear in Claude's `available_skills` list with their name + description, and Claude decides whether to consult a skill based on that description. The important thing to know is that Claude only consults skills for tasks it can't easily handle on its own — simple, one-step queries like "read this PDF" may not trigger a skill even if the description matches perfectly, because Claude can handle them directly with basic tools. Complex, multi-step, or specialized queries reliably trigger skills when the description matches.
|
||||
|
||||
This means your eval queries should be substantive enough that Claude would actually benefit from consulting a skill. Simple queries like "read file X" are poor test cases — they won't trigger skills regardless of description quality.
|
||||
|
||||
### Step 4: Apply the result
|
||||
|
||||
Take `best_description` from the JSON output and update the skill's SKILL.md frontmatter. Show the user before/after and report the scores.
|
||||
|
||||
---
|
||||
|
||||
### Package and Present (only if `present_files` tool is available)
|
||||
|
||||
Check whether you have access to the `present_files` tool. If you don't, skip this step. If you do, package the skill and present the .skill file to the user:
|
||||
|
||||
```bash
|
||||
python -m scripts.package_skill <path/to/skill-folder>
|
||||
```
|
||||
|
||||
After packaging, direct the user to the resulting `.skill` file path so they can install it.
|
||||
|
||||
---
|
||||
|
||||
## Claude.ai-specific instructions
|
||||
|
||||
In Claude.ai, the core workflow is the same (draft → test → review → improve → repeat), but because Claude.ai doesn't have subagents, some mechanics change. Here's what to adapt:
|
||||
|
||||
**Running test cases**: No subagents means no parallel execution. For each test case, read the skill's SKILL.md, then follow its instructions to accomplish the test prompt yourself. Do them one at a time. This is less rigorous than independent subagents (you wrote the skill and you're also running it, so you have full context), but it's a useful sanity check — and the human review step compensates. Skip the baseline runs — just use the skill to complete the task as requested.
|
||||
|
||||
**Reviewing results**: If you can't open a browser (e.g., Claude.ai's VM has no display, or you're on a remote server), skip the browser reviewer entirely. Instead, present results directly in the conversation. For each test case, show the prompt and the output. If the output is a file the user needs to see (like a .docx or .xlsx), save it to the filesystem and tell them where it is so they can download and inspect it. Ask for feedback inline: "How does this look? Anything you'd change?"
|
||||
|
||||
**Benchmarking**: Skip the quantitative benchmarking — it relies on baseline comparisons which aren't meaningful without subagents. Focus on qualitative feedback from the user.
|
||||
|
||||
**The iteration loop**: Same as before — improve the skill, rerun the test cases, ask for feedback — just without the browser reviewer in the middle. You can still organize results into iteration directories on the filesystem if you have one.
|
||||
|
||||
**Description optimization**: This section requires the `claude` CLI tool (specifically `claude -p`) which is only available in Claude Code. Skip it if you're on Claude.ai.
|
||||
|
||||
**Blind comparison**: Requires subagents. Skip it.
|
||||
|
||||
**Packaging**: The `package_skill.py` script works anywhere with Python and a filesystem. On Claude.ai, you can run it and the user can download the resulting `.skill` file.
|
||||
|
||||
**Updating an existing skill**: The user might be asking you to update an existing skill, not create a new one. In this case:
|
||||
- **Preserve the original name.** Note the skill's directory name and `name` frontmatter field -- use them unchanged. E.g., if the installed skill is `research-helper`, output `research-helper.skill` (not `research-helper-v2`).
|
||||
- **Copy to a writeable location before editing.** The installed skill path may be read-only. Copy to `/tmp/skill-name/`, edit there, and package from the copy.
|
||||
- **If packaging manually, stage in `/tmp/` first**, then copy to the output directory -- direct writes may fail due to permissions.
|
||||
|
||||
---
|
||||
|
||||
## Cowork-Specific Instructions
|
||||
|
||||
If you're in Cowork, the main things to know are:
|
||||
|
||||
- You have subagents, so the main workflow (spawn test cases in parallel, run baselines, grade, etc.) all works. (However, if you run into severe problems with timeouts, it's OK to run the test prompts in series rather than parallel.)
|
||||
- You don't have a browser or display, so when generating the eval viewer, use `--static <output_path>` to write a standalone HTML file instead of starting a server. Then proffer a link that the user can click to open the HTML in their browser.
|
||||
- For whatever reason, the Cowork setup seems to disincline Claude from generating the eval viewer after running the tests, so just to reiterate: whether you're in Cowork or in Claude Code, after running tests, you should always generate the eval viewer for the human to look at examples before revising the skill yourself and trying to make corrections, using `generate_review.py` (not writing your own boutique html code). Sorry in advance but I'm gonna go all caps here: GENERATE THE EVAL VIEWER *BEFORE* evaluating inputs yourself. You want to get them in front of the human ASAP!
|
||||
- Feedback works differently: since there's no running server, the viewer's "Submit All Reviews" button will download `feedback.json` as a file. You can then read it from there (you may have to request access first).
|
||||
- Packaging works — `package_skill.py` just needs Python and a filesystem.
|
||||
- Description optimization (`run_loop.py` / `run_eval.py`) should work in Cowork just fine since it uses `claude -p` via subprocess, not a browser, but please save it until you've fully finished making the skill and the user agrees it's in good shape.
|
||||
- **Updating an existing skill**: The user might be asking you to update an existing skill, not create a new one. Follow the update guidance in the claude.ai section above.
|
||||
|
||||
---
|
||||
|
||||
## Reference files
|
||||
|
||||
The agents/ directory contains instructions for specialized subagents. Read them when you need to spawn the relevant subagent.
|
||||
|
||||
- `agents/grader.md` — How to evaluate assertions against outputs
|
||||
- `agents/comparator.md` — How to do blind A/B comparison between two outputs
|
||||
- `agents/analyzer.md` — How to analyze why one version beat another
|
||||
|
||||
The references/ directory has additional documentation:
|
||||
- `references/schemas.md` — JSON structures for evals.json, grading.json, etc.
|
||||
|
||||
---
|
||||
|
||||
Repeating one more time the core loop here for emphasis:
|
||||
|
||||
- Figure out what the skill is about
|
||||
- Draft or edit the skill
|
||||
- Run claude-with-access-to-the-skill on test prompts
|
||||
- With the user, evaluate the outputs:
|
||||
- Create benchmark.json and run `eval-viewer/generate_review.py` to help the user review them
|
||||
- Run quantitative evals
|
||||
- Repeat until you and the user are satisfied
|
||||
- Package the final skill and return it to the user.
|
||||
|
||||
Please add steps to your TodoList, if you have such a thing, to make sure you don't forget. If you're in Cowork, please specifically put "Create evals JSON and run `eval-viewer/generate_review.py` so human can review test cases" in your TodoList to make sure it happens.
|
||||
|
||||
Good luck!
|
||||
274
skills/skill-creator/agents/analyzer.md
Normal file
274
skills/skill-creator/agents/analyzer.md
Normal file
@@ -0,0 +1,274 @@
|
||||
# Post-hoc Analyzer Agent
|
||||
|
||||
Analyze blind comparison results to understand WHY the winner won and generate improvement suggestions.
|
||||
|
||||
## Role
|
||||
|
||||
After the blind comparator determines a winner, the Post-hoc Analyzer "unblids" the results by examining the skills and transcripts. The goal is to extract actionable insights: what made the winner better, and how can the loser be improved?
|
||||
|
||||
## Inputs
|
||||
|
||||
You receive these parameters in your prompt:
|
||||
|
||||
- **winner**: "A" or "B" (from blind comparison)
|
||||
- **winner_skill_path**: Path to the skill that produced the winning output
|
||||
- **winner_transcript_path**: Path to the execution transcript for the winner
|
||||
- **loser_skill_path**: Path to the skill that produced the losing output
|
||||
- **loser_transcript_path**: Path to the execution transcript for the loser
|
||||
- **comparison_result_path**: Path to the blind comparator's output JSON
|
||||
- **output_path**: Where to save the analysis results
|
||||
|
||||
## Process
|
||||
|
||||
### Step 1: Read Comparison Result
|
||||
|
||||
1. Read the blind comparator's output at comparison_result_path
|
||||
2. Note the winning side (A or B), the reasoning, and any scores
|
||||
3. Understand what the comparator valued in the winning output
|
||||
|
||||
### Step 2: Read Both Skills
|
||||
|
||||
1. Read the winner skill's SKILL.md and key referenced files
|
||||
2. Read the loser skill's SKILL.md and key referenced files
|
||||
3. Identify structural differences:
|
||||
- Instructions clarity and specificity
|
||||
- Script/tool usage patterns
|
||||
- Example coverage
|
||||
- Edge case handling
|
||||
|
||||
### Step 3: Read Both Transcripts
|
||||
|
||||
1. Read the winner's transcript
|
||||
2. Read the loser's transcript
|
||||
3. Compare execution patterns:
|
||||
- How closely did each follow their skill's instructions?
|
||||
- What tools were used differently?
|
||||
- Where did the loser diverge from optimal behavior?
|
||||
- Did either encounter errors or make recovery attempts?
|
||||
|
||||
### Step 4: Analyze Instruction Following
|
||||
|
||||
For each transcript, evaluate:
|
||||
- Did the agent follow the skill's explicit instructions?
|
||||
- Did the agent use the skill's provided tools/scripts?
|
||||
- Were there missed opportunities to leverage skill content?
|
||||
- Did the agent add unnecessary steps not in the skill?
|
||||
|
||||
Score instruction following 1-10 and note specific issues.
|
||||
|
||||
### Step 5: Identify Winner Strengths
|
||||
|
||||
Determine what made the winner better:
|
||||
- Clearer instructions that led to better behavior?
|
||||
- Better scripts/tools that produced better output?
|
||||
- More comprehensive examples that guided edge cases?
|
||||
- Better error handling guidance?
|
||||
|
||||
Be specific. Quote from skills/transcripts where relevant.
|
||||
|
||||
### Step 6: Identify Loser Weaknesses
|
||||
|
||||
Determine what held the loser back:
|
||||
- Ambiguous instructions that led to suboptimal choices?
|
||||
- Missing tools/scripts that forced workarounds?
|
||||
- Gaps in edge case coverage?
|
||||
- Poor error handling that caused failures?
|
||||
|
||||
### Step 7: Generate Improvement Suggestions
|
||||
|
||||
Based on the analysis, produce actionable suggestions for improving the loser skill:
|
||||
- Specific instruction changes to make
|
||||
- Tools/scripts to add or modify
|
||||
- Examples to include
|
||||
- Edge cases to address
|
||||
|
||||
Prioritize by impact. Focus on changes that would have changed the outcome.
|
||||
|
||||
### Step 8: Write Analysis Results
|
||||
|
||||
Save structured analysis to `{output_path}`.
|
||||
|
||||
## Output Format
|
||||
|
||||
Write a JSON file with this structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"comparison_summary": {
|
||||
"winner": "A",
|
||||
"winner_skill": "path/to/winner/skill",
|
||||
"loser_skill": "path/to/loser/skill",
|
||||
"comparator_reasoning": "Brief summary of why comparator chose winner"
|
||||
},
|
||||
"winner_strengths": [
|
||||
"Clear step-by-step instructions for handling multi-page documents",
|
||||
"Included validation script that caught formatting errors",
|
||||
"Explicit guidance on fallback behavior when OCR fails"
|
||||
],
|
||||
"loser_weaknesses": [
|
||||
"Vague instruction 'process the document appropriately' led to inconsistent behavior",
|
||||
"No script for validation, agent had to improvise and made errors",
|
||||
"No guidance on OCR failure, agent gave up instead of trying alternatives"
|
||||
],
|
||||
"instruction_following": {
|
||||
"winner": {
|
||||
"score": 9,
|
||||
"issues": [
|
||||
"Minor: skipped optional logging step"
|
||||
]
|
||||
},
|
||||
"loser": {
|
||||
"score": 6,
|
||||
"issues": [
|
||||
"Did not use the skill's formatting template",
|
||||
"Invented own approach instead of following step 3",
|
||||
"Missed the 'always validate output' instruction"
|
||||
]
|
||||
}
|
||||
},
|
||||
"improvement_suggestions": [
|
||||
{
|
||||
"priority": "high",
|
||||
"category": "instructions",
|
||||
"suggestion": "Replace 'process the document appropriately' with explicit steps: 1) Extract text, 2) Identify sections, 3) Format per template",
|
||||
"expected_impact": "Would eliminate ambiguity that caused inconsistent behavior"
|
||||
},
|
||||
{
|
||||
"priority": "high",
|
||||
"category": "tools",
|
||||
"suggestion": "Add validate_output.py script similar to winner skill's validation approach",
|
||||
"expected_impact": "Would catch formatting errors before final output"
|
||||
},
|
||||
{
|
||||
"priority": "medium",
|
||||
"category": "error_handling",
|
||||
"suggestion": "Add fallback instructions: 'If OCR fails, try: 1) different resolution, 2) image preprocessing, 3) manual extraction'",
|
||||
"expected_impact": "Would prevent early failure on difficult documents"
|
||||
}
|
||||
],
|
||||
"transcript_insights": {
|
||||
"winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script -> Fixed 2 issues -> Produced output",
|
||||
"loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods -> No validation -> Output had errors"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
|
||||
- **Be specific**: Quote from skills and transcripts, don't just say "instructions were unclear"
|
||||
- **Be actionable**: Suggestions should be concrete changes, not vague advice
|
||||
- **Focus on skill improvements**: The goal is to improve the losing skill, not critique the agent
|
||||
- **Prioritize by impact**: Which changes would most likely have changed the outcome?
|
||||
- **Consider causation**: Did the skill weakness actually cause the worse output, or is it incidental?
|
||||
- **Stay objective**: Analyze what happened, don't editorialize
|
||||
- **Think about generalization**: Would this improvement help on other evals too?
|
||||
|
||||
## Categories for Suggestions
|
||||
|
||||
Use these categories to organize improvement suggestions:
|
||||
|
||||
| Category | Description |
|
||||
|----------|-------------|
|
||||
| `instructions` | Changes to the skill's prose instructions |
|
||||
| `tools` | Scripts, templates, or utilities to add/modify |
|
||||
| `examples` | Example inputs/outputs to include |
|
||||
| `error_handling` | Guidance for handling failures |
|
||||
| `structure` | Reorganization of skill content |
|
||||
| `references` | External docs or resources to add |
|
||||
|
||||
## Priority Levels
|
||||
|
||||
- **high**: Would likely change the outcome of this comparison
|
||||
- **medium**: Would improve quality but may not change win/loss
|
||||
- **low**: Nice to have, marginal improvement
|
||||
|
||||
---
|
||||
|
||||
# Analyzing Benchmark Results
|
||||
|
||||
When analyzing benchmark results, the analyzer's purpose is to **surface patterns and anomalies** across multiple runs, not suggest skill improvements.
|
||||
|
||||
## Role
|
||||
|
||||
Review all benchmark run results and generate freeform notes that help the user understand skill performance. Focus on patterns that wouldn't be visible from aggregate metrics alone.
|
||||
|
||||
## Inputs
|
||||
|
||||
You receive these parameters in your prompt:
|
||||
|
||||
- **benchmark_data_path**: Path to the in-progress benchmark.json with all run results
|
||||
- **skill_path**: Path to the skill being benchmarked
|
||||
- **output_path**: Where to save the notes (as JSON array of strings)
|
||||
|
||||
## Process
|
||||
|
||||
### Step 1: Read Benchmark Data
|
||||
|
||||
1. Read the benchmark.json containing all run results
|
||||
2. Note the configurations tested (with_skill, without_skill)
|
||||
3. Understand the run_summary aggregates already calculated
|
||||
|
||||
### Step 2: Analyze Per-Assertion Patterns
|
||||
|
||||
For each expectation across all runs:
|
||||
- Does it **always pass** in both configurations? (may not differentiate skill value)
|
||||
- Does it **always fail** in both configurations? (may be broken or beyond capability)
|
||||
- Does it **always pass with skill but fail without**? (skill clearly adds value here)
|
||||
- Does it **always fail with skill but pass without**? (skill may be hurting)
|
||||
- Is it **highly variable**? (flaky expectation or non-deterministic behavior)
|
||||
|
||||
### Step 3: Analyze Cross-Eval Patterns
|
||||
|
||||
Look for patterns across evals:
|
||||
- Are certain eval types consistently harder/easier?
|
||||
- Do some evals show high variance while others are stable?
|
||||
- Are there surprising results that contradict expectations?
|
||||
|
||||
### Step 4: Analyze Metrics Patterns
|
||||
|
||||
Look at time_seconds, tokens, tool_calls:
|
||||
- Does the skill significantly increase execution time?
|
||||
- Is there high variance in resource usage?
|
||||
- Are there outlier runs that skew the aggregates?
|
||||
|
||||
### Step 5: Generate Notes
|
||||
|
||||
Write freeform observations as a list of strings. Each note should:
|
||||
- State a specific observation
|
||||
- Be grounded in the data (not speculation)
|
||||
- Help the user understand something the aggregate metrics don't show
|
||||
|
||||
Examples:
|
||||
- "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value"
|
||||
- "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure that may be flaky"
|
||||
- "Without-skill runs consistently fail on table extraction expectations (0% pass rate)"
|
||||
- "Skill adds 13s average execution time but improves pass rate by 50%"
|
||||
- "Token usage is 80% higher with skill, primarily due to script output parsing"
|
||||
- "All 3 without-skill runs for eval 1 produced empty output"
|
||||
|
||||
### Step 6: Write Notes
|
||||
|
||||
Save notes to `{output_path}` as a JSON array of strings:
|
||||
|
||||
```json
|
||||
[
|
||||
"Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value",
|
||||
"Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure",
|
||||
"Without-skill runs consistently fail on table extraction expectations",
|
||||
"Skill adds 13s average execution time but improves pass rate by 50%"
|
||||
]
|
||||
```
|
||||
|
||||
## Guidelines
|
||||
|
||||
**DO:**
|
||||
- Report what you observe in the data
|
||||
- Be specific about which evals, expectations, or runs you're referring to
|
||||
- Note patterns that aggregate metrics would hide
|
||||
- Provide context that helps interpret the numbers
|
||||
|
||||
**DO NOT:**
|
||||
- Suggest improvements to the skill (that's for the improvement step, not benchmarking)
|
||||
- Make subjective quality judgments ("the output was good/bad")
|
||||
- Speculate about causes without evidence
|
||||
- Repeat information already in the run_summary aggregates
|
||||
202
skills/skill-creator/agents/comparator.md
Normal file
202
skills/skill-creator/agents/comparator.md
Normal file
@@ -0,0 +1,202 @@
|
||||
# Blind Comparator Agent
|
||||
|
||||
Compare two outputs WITHOUT knowing which skill produced them.
|
||||
|
||||
## Role
|
||||
|
||||
The Blind Comparator judges which output better accomplishes the eval task. You receive two outputs labeled A and B, but you do NOT know which skill produced which. This prevents bias toward a particular skill or approach.
|
||||
|
||||
Your judgment is based purely on output quality and task completion.
|
||||
|
||||
## Inputs
|
||||
|
||||
You receive these parameters in your prompt:
|
||||
|
||||
- **output_a_path**: Path to the first output file or directory
|
||||
- **output_b_path**: Path to the second output file or directory
|
||||
- **eval_prompt**: The original task/prompt that was executed
|
||||
- **expectations**: List of expectations to check (optional - may be empty)
|
||||
|
||||
## Process
|
||||
|
||||
### Step 1: Read Both Outputs
|
||||
|
||||
1. Examine output A (file or directory)
|
||||
2. Examine output B (file or directory)
|
||||
3. Note the type, structure, and content of each
|
||||
4. If outputs are directories, examine all relevant files inside
|
||||
|
||||
### Step 2: Understand the Task
|
||||
|
||||
1. Read the eval_prompt carefully
|
||||
2. Identify what the task requires:
|
||||
- What should be produced?
|
||||
- What qualities matter (accuracy, completeness, format)?
|
||||
- What would distinguish a good output from a poor one?
|
||||
|
||||
### Step 3: Generate Evaluation Rubric
|
||||
|
||||
Based on the task, generate a rubric with two dimensions:
|
||||
|
||||
**Content Rubric** (what the output contains):
|
||||
| Criterion | 1 (Poor) | 3 (Acceptable) | 5 (Excellent) |
|
||||
|-----------|----------|----------------|---------------|
|
||||
| Correctness | Major errors | Minor errors | Fully correct |
|
||||
| Completeness | Missing key elements | Mostly complete | All elements present |
|
||||
| Accuracy | Significant inaccuracies | Minor inaccuracies | Accurate throughout |
|
||||
|
||||
**Structure Rubric** (how the output is organized):
|
||||
| Criterion | 1 (Poor) | 3 (Acceptable) | 5 (Excellent) |
|
||||
|-----------|----------|----------------|---------------|
|
||||
| Organization | Disorganized | Reasonably organized | Clear, logical structure |
|
||||
| Formatting | Inconsistent/broken | Mostly consistent | Professional, polished |
|
||||
| Usability | Difficult to use | Usable with effort | Easy to use |
|
||||
|
||||
Adapt criteria to the specific task. For example:
|
||||
- PDF form → "Field alignment", "Text readability", "Data placement"
|
||||
- Document → "Section structure", "Heading hierarchy", "Paragraph flow"
|
||||
- Data output → "Schema correctness", "Data types", "Completeness"
|
||||
|
||||
### Step 4: Evaluate Each Output Against the Rubric
|
||||
|
||||
For each output (A and B):
|
||||
|
||||
1. **Score each criterion** on the rubric (1-5 scale)
|
||||
2. **Calculate dimension totals**: Content score, Structure score
|
||||
3. **Calculate overall score**: Average of dimension scores, scaled to 1-10
|
||||
|
||||
### Step 5: Check Assertions (if provided)
|
||||
|
||||
If expectations are provided:
|
||||
|
||||
1. Check each expectation against output A
|
||||
2. Check each expectation against output B
|
||||
3. Count pass rates for each output
|
||||
4. Use expectation scores as secondary evidence (not the primary decision factor)
|
||||
|
||||
### Step 6: Determine the Winner
|
||||
|
||||
Compare A and B based on (in priority order):
|
||||
|
||||
1. **Primary**: Overall rubric score (content + structure)
|
||||
2. **Secondary**: Assertion pass rates (if applicable)
|
||||
3. **Tiebreaker**: If truly equal, declare a TIE
|
||||
|
||||
Be decisive - ties should be rare. One output is usually better, even if marginally.
|
||||
|
||||
### Step 7: Write Comparison Results
|
||||
|
||||
Save results to a JSON file at the path specified (or `comparison.json` if not specified).
|
||||
|
||||
## Output Format
|
||||
|
||||
Write a JSON file with this structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"winner": "A",
|
||||
"reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.",
|
||||
"rubric": {
|
||||
"A": {
|
||||
"content": {
|
||||
"correctness": 5,
|
||||
"completeness": 5,
|
||||
"accuracy": 4
|
||||
},
|
||||
"structure": {
|
||||
"organization": 4,
|
||||
"formatting": 5,
|
||||
"usability": 4
|
||||
},
|
||||
"content_score": 4.7,
|
||||
"structure_score": 4.3,
|
||||
"overall_score": 9.0
|
||||
},
|
||||
"B": {
|
||||
"content": {
|
||||
"correctness": 3,
|
||||
"completeness": 2,
|
||||
"accuracy": 3
|
||||
},
|
||||
"structure": {
|
||||
"organization": 3,
|
||||
"formatting": 2,
|
||||
"usability": 3
|
||||
},
|
||||
"content_score": 2.7,
|
||||
"structure_score": 2.7,
|
||||
"overall_score": 5.4
|
||||
}
|
||||
},
|
||||
"output_quality": {
|
||||
"A": {
|
||||
"score": 9,
|
||||
"strengths": ["Complete solution", "Well-formatted", "All fields present"],
|
||||
"weaknesses": ["Minor style inconsistency in header"]
|
||||
},
|
||||
"B": {
|
||||
"score": 5,
|
||||
"strengths": ["Readable output", "Correct basic structure"],
|
||||
"weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"]
|
||||
}
|
||||
},
|
||||
"expectation_results": {
|
||||
"A": {
|
||||
"passed": 4,
|
||||
"total": 5,
|
||||
"pass_rate": 0.80,
|
||||
"details": [
|
||||
{"text": "Output includes name", "passed": true},
|
||||
{"text": "Output includes date", "passed": true},
|
||||
{"text": "Format is PDF", "passed": true},
|
||||
{"text": "Contains signature", "passed": false},
|
||||
{"text": "Readable text", "passed": true}
|
||||
]
|
||||
},
|
||||
"B": {
|
||||
"passed": 3,
|
||||
"total": 5,
|
||||
"pass_rate": 0.60,
|
||||
"details": [
|
||||
{"text": "Output includes name", "passed": true},
|
||||
{"text": "Output includes date", "passed": false},
|
||||
{"text": "Format is PDF", "passed": true},
|
||||
{"text": "Contains signature", "passed": false},
|
||||
{"text": "Readable text", "passed": true}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If no expectations were provided, omit the `expectation_results` field entirely.
|
||||
|
||||
## Field Descriptions
|
||||
|
||||
- **winner**: "A", "B", or "TIE"
|
||||
- **reasoning**: Clear explanation of why the winner was chosen (or why it's a tie)
|
||||
- **rubric**: Structured rubric evaluation for each output
|
||||
- **content**: Scores for content criteria (correctness, completeness, accuracy)
|
||||
- **structure**: Scores for structure criteria (organization, formatting, usability)
|
||||
- **content_score**: Average of content criteria (1-5)
|
||||
- **structure_score**: Average of structure criteria (1-5)
|
||||
- **overall_score**: Combined score scaled to 1-10
|
||||
- **output_quality**: Summary quality assessment
|
||||
- **score**: 1-10 rating (should match rubric overall_score)
|
||||
- **strengths**: List of positive aspects
|
||||
- **weaknesses**: List of issues or shortcomings
|
||||
- **expectation_results**: (Only if expectations provided)
|
||||
- **passed**: Number of expectations that passed
|
||||
- **total**: Total number of expectations
|
||||
- **pass_rate**: Fraction passed (0.0 to 1.0)
|
||||
- **details**: Individual expectation results
|
||||
|
||||
## Guidelines
|
||||
|
||||
- **Stay blind**: DO NOT try to infer which skill produced which output. Judge purely on output quality.
|
||||
- **Be specific**: Cite specific examples when explaining strengths and weaknesses.
|
||||
- **Be decisive**: Choose a winner unless outputs are genuinely equivalent.
|
||||
- **Output quality first**: Assertion scores are secondary to overall task completion.
|
||||
- **Be objective**: Don't favor outputs based on style preferences; focus on correctness and completeness.
|
||||
- **Explain your reasoning**: The reasoning field should make it clear why you chose the winner.
|
||||
- **Handle edge cases**: If both outputs fail, pick the one that fails less badly. If both are excellent, pick the one that's marginally better.
|
||||
223
skills/skill-creator/agents/grader.md
Normal file
223
skills/skill-creator/agents/grader.md
Normal file
@@ -0,0 +1,223 @@
|
||||
# Grader Agent
|
||||
|
||||
Evaluate expectations against an execution transcript and outputs.
|
||||
|
||||
## Role
|
||||
|
||||
The Grader reviews a transcript and output files, then determines whether each expectation passes or fails. Provide clear evidence for each judgment.
|
||||
|
||||
You have two jobs: grade the outputs, and critique the evals themselves. A passing grade on a weak assertion is worse than useless — it creates false confidence. When you notice an assertion that's trivially satisfied, or an important outcome that no assertion checks, say so.
|
||||
|
||||
## Inputs
|
||||
|
||||
You receive these parameters in your prompt:
|
||||
|
||||
- **expectations**: List of expectations to evaluate (strings)
|
||||
- **transcript_path**: Path to the execution transcript (markdown file)
|
||||
- **outputs_dir**: Directory containing output files from execution
|
||||
|
||||
## Process
|
||||
|
||||
### Step 1: Read the Transcript
|
||||
|
||||
1. Read the transcript file completely
|
||||
2. Note the eval prompt, execution steps, and final result
|
||||
3. Identify any issues or errors documented
|
||||
|
||||
### Step 2: Examine Output Files
|
||||
|
||||
1. List files in outputs_dir
|
||||
2. Read/examine each file relevant to the expectations. If outputs aren't plain text, use the inspection tools provided in your prompt — don't rely solely on what the transcript says the executor produced.
|
||||
3. Note contents, structure, and quality
|
||||
|
||||
### Step 3: Evaluate Each Assertion
|
||||
|
||||
For each expectation:
|
||||
|
||||
1. **Search for evidence** in the transcript and outputs
|
||||
2. **Determine verdict**:
|
||||
- **PASS**: Clear evidence the expectation is true AND the evidence reflects genuine task completion, not just surface-level compliance
|
||||
- **FAIL**: No evidence, or evidence contradicts the expectation, or the evidence is superficial (e.g., correct filename but empty/wrong content)
|
||||
3. **Cite the evidence**: Quote the specific text or describe what you found
|
||||
|
||||
### Step 4: Extract and Verify Claims
|
||||
|
||||
Beyond the predefined expectations, extract implicit claims from the outputs and verify them:
|
||||
|
||||
1. **Extract claims** from the transcript and outputs:
|
||||
- Factual statements ("The form has 12 fields")
|
||||
- Process claims ("Used pypdf to fill the form")
|
||||
- Quality claims ("All fields were filled correctly")
|
||||
|
||||
2. **Verify each claim**:
|
||||
- **Factual claims**: Can be checked against the outputs or external sources
|
||||
- **Process claims**: Can be verified from the transcript
|
||||
- **Quality claims**: Evaluate whether the claim is justified
|
||||
|
||||
3. **Flag unverifiable claims**: Note claims that cannot be verified with available information
|
||||
|
||||
This catches issues that predefined expectations might miss.
|
||||
|
||||
### Step 5: Read User Notes
|
||||
|
||||
If `{outputs_dir}/user_notes.md` exists:
|
||||
1. Read it and note any uncertainties or issues flagged by the executor
|
||||
2. Include relevant concerns in the grading output
|
||||
3. These may reveal problems even when expectations pass
|
||||
|
||||
### Step 6: Critique the Evals
|
||||
|
||||
After grading, consider whether the evals themselves could be improved. Only surface suggestions when there's a clear gap.
|
||||
|
||||
Good suggestions test meaningful outcomes — assertions that are hard to satisfy without actually doing the work correctly. Think about what makes an assertion *discriminating*: it passes when the skill genuinely succeeds and fails when it doesn't.
|
||||
|
||||
Suggestions worth raising:
|
||||
- An assertion that passed but would also pass for a clearly wrong output (e.g., checking filename existence but not file content)
|
||||
- An important outcome you observed — good or bad — that no assertion covers at all
|
||||
- An assertion that can't actually be verified from the available outputs
|
||||
|
||||
Keep the bar high. The goal is to flag things the eval author would say "good catch" about, not to nitpick every assertion.
|
||||
|
||||
### Step 7: Write Grading Results
|
||||
|
||||
Save results to `{outputs_dir}/../grading.json` (sibling to outputs_dir).
|
||||
|
||||
## Grading Criteria
|
||||
|
||||
**PASS when**:
|
||||
- The transcript or outputs clearly demonstrate the expectation is true
|
||||
- Specific evidence can be cited
|
||||
- The evidence reflects genuine substance, not just surface compliance (e.g., a file exists AND contains correct content, not just the right filename)
|
||||
|
||||
**FAIL when**:
|
||||
- No evidence found for the expectation
|
||||
- Evidence contradicts the expectation
|
||||
- The expectation cannot be verified from available information
|
||||
- The evidence is superficial — the assertion is technically satisfied but the underlying task outcome is wrong or incomplete
|
||||
- The output appears to meet the assertion by coincidence rather than by actually doing the work
|
||||
|
||||
**When uncertain**: The burden of proof to pass is on the expectation.
|
||||
|
||||
### Step 8: Read Executor Metrics and Timing
|
||||
|
||||
1. If `{outputs_dir}/metrics.json` exists, read it and include in grading output
|
||||
2. If `{outputs_dir}/../timing.json` exists, read it and include timing data
|
||||
|
||||
## Output Format
|
||||
|
||||
Write a JSON file with this structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"expectations": [
|
||||
{
|
||||
"text": "The output includes the name 'John Smith'",
|
||||
"passed": true,
|
||||
"evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'"
|
||||
},
|
||||
{
|
||||
"text": "The spreadsheet has a SUM formula in cell B10",
|
||||
"passed": false,
|
||||
"evidence": "No spreadsheet was created. The output was a text file."
|
||||
},
|
||||
{
|
||||
"text": "The assistant used the skill's OCR script",
|
||||
"passed": true,
|
||||
"evidence": "Transcript Step 2 shows: 'Tool: Bash - python ocr_script.py image.png'"
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"passed": 2,
|
||||
"failed": 1,
|
||||
"total": 3,
|
||||
"pass_rate": 0.67
|
||||
},
|
||||
"execution_metrics": {
|
||||
"tool_calls": {
|
||||
"Read": 5,
|
||||
"Write": 2,
|
||||
"Bash": 8
|
||||
},
|
||||
"total_tool_calls": 15,
|
||||
"total_steps": 6,
|
||||
"errors_encountered": 0,
|
||||
"output_chars": 12450,
|
||||
"transcript_chars": 3200
|
||||
},
|
||||
"timing": {
|
||||
"executor_duration_seconds": 165.0,
|
||||
"grader_duration_seconds": 26.0,
|
||||
"total_duration_seconds": 191.0
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"claim": "The form has 12 fillable fields",
|
||||
"type": "factual",
|
||||
"verified": true,
|
||||
"evidence": "Counted 12 fields in field_info.json"
|
||||
},
|
||||
{
|
||||
"claim": "All required fields were populated",
|
||||
"type": "quality",
|
||||
"verified": false,
|
||||
"evidence": "Reference section was left blank despite data being available"
|
||||
}
|
||||
],
|
||||
"user_notes_summary": {
|
||||
"uncertainties": ["Used 2023 data, may be stale"],
|
||||
"needs_review": [],
|
||||
"workarounds": ["Fell back to text overlay for non-fillable fields"]
|
||||
},
|
||||
"eval_feedback": {
|
||||
"suggestions": [
|
||||
{
|
||||
"assertion": "The output includes the name 'John Smith'",
|
||||
"reason": "A hallucinated document that mentions the name would also pass — consider checking it appears as the primary contact with matching phone and email from the input"
|
||||
},
|
||||
{
|
||||
"reason": "No assertion checks whether the extracted phone numbers match the input — I observed incorrect numbers in the output that went uncaught"
|
||||
}
|
||||
],
|
||||
"overall": "Assertions check presence but not correctness. Consider adding content verification."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Field Descriptions
|
||||
|
||||
- **expectations**: Array of graded expectations
|
||||
- **text**: The original expectation text
|
||||
- **passed**: Boolean - true if expectation passes
|
||||
- **evidence**: Specific quote or description supporting the verdict
|
||||
- **summary**: Aggregate statistics
|
||||
- **passed**: Count of passed expectations
|
||||
- **failed**: Count of failed expectations
|
||||
- **total**: Total expectations evaluated
|
||||
- **pass_rate**: Fraction passed (0.0 to 1.0)
|
||||
- **execution_metrics**: Copied from executor's metrics.json (if available)
|
||||
- **output_chars**: Total character count of output files (proxy for tokens)
|
||||
- **transcript_chars**: Character count of transcript
|
||||
- **timing**: Wall clock timing from timing.json (if available)
|
||||
- **executor_duration_seconds**: Time spent in executor subagent
|
||||
- **total_duration_seconds**: Total elapsed time for the run
|
||||
- **claims**: Extracted and verified claims from the output
|
||||
- **claim**: The statement being verified
|
||||
- **type**: "factual", "process", or "quality"
|
||||
- **verified**: Boolean - whether the claim holds
|
||||
- **evidence**: Supporting or contradicting evidence
|
||||
- **user_notes_summary**: Issues flagged by the executor
|
||||
- **uncertainties**: Things the executor wasn't sure about
|
||||
- **needs_review**: Items requiring human attention
|
||||
- **workarounds**: Places where the skill didn't work as expected
|
||||
- **eval_feedback**: Improvement suggestions for the evals (only when warranted)
|
||||
- **suggestions**: List of concrete suggestions, each with a `reason` and optionally an `assertion` it relates to
|
||||
- **overall**: Brief assessment — can be "No suggestions, evals look solid" if nothing to flag
|
||||
|
||||
## Guidelines
|
||||
|
||||
- **Be objective**: Base verdicts on evidence, not assumptions
|
||||
- **Be specific**: Quote the exact text that supports your verdict
|
||||
- **Be thorough**: Check both transcript and output files
|
||||
- **Be consistent**: Apply the same standard to each expectation
|
||||
- **Explain failures**: Make it clear why evidence was insufficient
|
||||
- **No partial credit**: Each expectation is pass or fail, not partial
|
||||
146
skills/skill-creator/assets/eval_review.html
Normal file
146
skills/skill-creator/assets/eval_review.html
Normal file
@@ -0,0 +1,146 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Eval Set Review - __SKILL_NAME_PLACEHOLDER__</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { font-family: 'Lora', Georgia, serif; background: #faf9f5; padding: 2rem; color: #141413; }
|
||||
h1 { font-family: 'Poppins', sans-serif; margin-bottom: 0.5rem; font-size: 1.5rem; }
|
||||
.description { color: #b0aea5; margin-bottom: 1.5rem; font-style: italic; max-width: 900px; }
|
||||
.controls { margin-bottom: 1rem; display: flex; gap: 0.5rem; }
|
||||
.btn { font-family: 'Poppins', sans-serif; padding: 0.5rem 1rem; border: none; border-radius: 6px; cursor: pointer; font-size: 0.875rem; font-weight: 500; }
|
||||
.btn-add { background: #6a9bcc; color: white; }
|
||||
.btn-add:hover { background: #5889b8; }
|
||||
.btn-export { background: #d97757; color: white; }
|
||||
.btn-export:hover { background: #c4613f; }
|
||||
table { width: 100%; max-width: 1100px; border-collapse: collapse; background: white; border-radius: 6px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }
|
||||
th { font-family: 'Poppins', sans-serif; background: #141413; color: #faf9f5; padding: 0.75rem 1rem; text-align: left; font-size: 0.875rem; }
|
||||
td { padding: 0.75rem 1rem; border-bottom: 1px solid #e8e6dc; vertical-align: top; }
|
||||
tr:nth-child(even) td { background: #faf9f5; }
|
||||
tr:hover td { background: #f3f1ea; }
|
||||
.section-header td { background: #e8e6dc; font-family: 'Poppins', sans-serif; font-weight: 500; font-size: 0.8rem; color: #141413; text-transform: uppercase; letter-spacing: 0.05em; }
|
||||
.query-input { width: 100%; padding: 0.4rem; border: 1px solid #e8e6dc; border-radius: 4px; font-size: 0.875rem; font-family: 'Lora', Georgia, serif; resize: vertical; min-height: 60px; }
|
||||
.query-input:focus { outline: none; border-color: #d97757; box-shadow: 0 0 0 2px rgba(217,119,87,0.15); }
|
||||
.toggle { position: relative; display: inline-block; width: 44px; height: 24px; }
|
||||
.toggle input { opacity: 0; width: 0; height: 0; }
|
||||
.toggle .slider { position: absolute; inset: 0; background: #b0aea5; border-radius: 24px; cursor: pointer; transition: 0.2s; }
|
||||
.toggle .slider::before { content: ""; position: absolute; width: 18px; height: 18px; left: 3px; bottom: 3px; background: white; border-radius: 50%; transition: 0.2s; }
|
||||
.toggle input:checked + .slider { background: #d97757; }
|
||||
.toggle input:checked + .slider::before { transform: translateX(20px); }
|
||||
.btn-delete { background: #c44; color: white; padding: 0.3rem 0.6rem; border: none; border-radius: 4px; cursor: pointer; font-size: 0.75rem; font-family: 'Poppins', sans-serif; }
|
||||
.btn-delete:hover { background: #a33; }
|
||||
.summary { margin-top: 1rem; color: #b0aea5; font-size: 0.875rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Eval Set Review: <span id="skill-name">__SKILL_NAME_PLACEHOLDER__</span></h1>
|
||||
<p class="description">Current description: <span id="skill-desc">__SKILL_DESCRIPTION_PLACEHOLDER__</span></p>
|
||||
|
||||
<div class="controls">
|
||||
<button class="btn btn-add" onclick="addRow()">+ Add Query</button>
|
||||
<button class="btn btn-export" onclick="exportEvalSet()">Export Eval Set</button>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width:65%">Query</th>
|
||||
<th style="width:18%">Should Trigger</th>
|
||||
<th style="width:10%">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="eval-body"></tbody>
|
||||
</table>
|
||||
|
||||
<p class="summary" id="summary"></p>
|
||||
|
||||
<script>
|
||||
const EVAL_DATA = __EVAL_DATA_PLACEHOLDER__;
|
||||
|
||||
let evalItems = [...EVAL_DATA];
|
||||
|
||||
function render() {
|
||||
const tbody = document.getElementById('eval-body');
|
||||
tbody.innerHTML = '';
|
||||
|
||||
// Sort: should-trigger first, then should-not-trigger
|
||||
const sorted = evalItems
|
||||
.map((item, origIdx) => ({ ...item, origIdx }))
|
||||
.sort((a, b) => (b.should_trigger ? 1 : 0) - (a.should_trigger ? 1 : 0));
|
||||
|
||||
let lastGroup = null;
|
||||
sorted.forEach(item => {
|
||||
const group = item.should_trigger ? 'trigger' : 'no-trigger';
|
||||
if (group !== lastGroup) {
|
||||
const headerRow = document.createElement('tr');
|
||||
headerRow.className = 'section-header';
|
||||
headerRow.innerHTML = `<td colspan="3">${item.should_trigger ? 'Should Trigger' : 'Should NOT Trigger'}</td>`;
|
||||
tbody.appendChild(headerRow);
|
||||
lastGroup = group;
|
||||
}
|
||||
|
||||
const idx = item.origIdx;
|
||||
const tr = document.createElement('tr');
|
||||
tr.innerHTML = `
|
||||
<td><textarea class="query-input" onchange="updateQuery(${idx}, this.value)">${escapeHtml(item.query)}</textarea></td>
|
||||
<td>
|
||||
<label class="toggle">
|
||||
<input type="checkbox" ${item.should_trigger ? 'checked' : ''} onchange="updateTrigger(${idx}, this.checked)">
|
||||
<span class="slider"></span>
|
||||
</label>
|
||||
<span style="margin-left:8px;font-size:0.8rem;color:#b0aea5">${item.should_trigger ? 'Yes' : 'No'}</span>
|
||||
</td>
|
||||
<td><button class="btn-delete" onclick="deleteRow(${idx})">Delete</button></td>
|
||||
`;
|
||||
tbody.appendChild(tr);
|
||||
});
|
||||
updateSummary();
|
||||
}
|
||||
|
||||
function escapeHtml(text) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function updateQuery(idx, value) { evalItems[idx].query = value; updateSummary(); }
|
||||
function updateTrigger(idx, value) { evalItems[idx].should_trigger = value; render(); }
|
||||
function deleteRow(idx) { evalItems.splice(idx, 1); render(); }
|
||||
|
||||
function addRow() {
|
||||
evalItems.push({ query: '', should_trigger: true });
|
||||
render();
|
||||
const inputs = document.querySelectorAll('.query-input');
|
||||
inputs[inputs.length - 1].focus();
|
||||
}
|
||||
|
||||
function updateSummary() {
|
||||
const trigger = evalItems.filter(i => i.should_trigger).length;
|
||||
const noTrigger = evalItems.filter(i => !i.should_trigger).length;
|
||||
document.getElementById('summary').textContent =
|
||||
`${evalItems.length} queries total: ${trigger} should trigger, ${noTrigger} should not trigger`;
|
||||
}
|
||||
|
||||
function exportEvalSet() {
|
||||
const valid = evalItems.filter(i => i.query.trim() !== '');
|
||||
const data = valid.map(i => ({ query: i.query.trim(), should_trigger: i.should_trigger }));
|
||||
const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'eval_set.json';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
render();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
471
skills/skill-creator/eval-viewer/generate_review.py
Normal file
471
skills/skill-creator/eval-viewer/generate_review.py
Normal file
@@ -0,0 +1,471 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate and serve a review page for eval results.
|
||||
|
||||
Reads the workspace directory, discovers runs (directories with outputs/),
|
||||
embeds all output data into a self-contained HTML page, and serves it via
|
||||
a tiny HTTP server. Feedback auto-saves to feedback.json in the workspace.
|
||||
|
||||
Usage:
|
||||
python generate_review.py <workspace-path> [--port PORT] [--skill-name NAME]
|
||||
python generate_review.py <workspace-path> --previous-feedback /path/to/old/feedback.json
|
||||
|
||||
No dependencies beyond the Python stdlib are required.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import webbrowser
|
||||
from functools import partial
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from pathlib import Path
|
||||
|
||||
# Files to exclude from output listings
|
||||
METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"}
|
||||
|
||||
# Extensions we render as inline text
|
||||
TEXT_EXTENSIONS = {
|
||||
".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx",
|
||||
".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs",
|
||||
".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml",
|
||||
}
|
||||
|
||||
# Extensions we render as inline images
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"}
|
||||
|
||||
# MIME type overrides for common types
|
||||
MIME_OVERRIDES = {
|
||||
".svg": "image/svg+xml",
|
||||
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
}
|
||||
|
||||
|
||||
def get_mime_type(path: Path) -> str:
|
||||
ext = path.suffix.lower()
|
||||
if ext in MIME_OVERRIDES:
|
||||
return MIME_OVERRIDES[ext]
|
||||
mime, _ = mimetypes.guess_type(str(path))
|
||||
return mime or "application/octet-stream"
|
||||
|
||||
|
||||
def find_runs(workspace: Path) -> list[dict]:
|
||||
"""Recursively find directories that contain an outputs/ subdirectory."""
|
||||
runs: list[dict] = []
|
||||
_find_runs_recursive(workspace, workspace, runs)
|
||||
runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r["id"]))
|
||||
return runs
|
||||
|
||||
|
||||
def _find_runs_recursive(root: Path, current: Path, runs: list[dict]) -> None:
|
||||
if not current.is_dir():
|
||||
return
|
||||
|
||||
outputs_dir = current / "outputs"
|
||||
if outputs_dir.is_dir():
|
||||
run = build_run(root, current)
|
||||
if run:
|
||||
runs.append(run)
|
||||
return
|
||||
|
||||
skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"}
|
||||
for child in sorted(current.iterdir()):
|
||||
if child.is_dir() and child.name not in skip:
|
||||
_find_runs_recursive(root, child, runs)
|
||||
|
||||
|
||||
def build_run(root: Path, run_dir: Path) -> dict | None:
|
||||
"""Build a run dict with prompt, outputs, and grading data."""
|
||||
prompt = ""
|
||||
eval_id = None
|
||||
|
||||
# Try eval_metadata.json
|
||||
for candidate in [run_dir / "eval_metadata.json", run_dir.parent / "eval_metadata.json"]:
|
||||
if candidate.exists():
|
||||
try:
|
||||
metadata = json.loads(candidate.read_text())
|
||||
prompt = metadata.get("prompt", "")
|
||||
eval_id = metadata.get("eval_id")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
if prompt:
|
||||
break
|
||||
|
||||
# Fall back to transcript.md
|
||||
if not prompt:
|
||||
for candidate in [run_dir / "transcript.md", run_dir / "outputs" / "transcript.md"]:
|
||||
if candidate.exists():
|
||||
try:
|
||||
text = candidate.read_text()
|
||||
match = re.search(r"## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)", text)
|
||||
if match:
|
||||
prompt = match.group(1).strip()
|
||||
except OSError:
|
||||
pass
|
||||
if prompt:
|
||||
break
|
||||
|
||||
if not prompt:
|
||||
prompt = "(No prompt found)"
|
||||
|
||||
run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-")
|
||||
|
||||
# Collect output files
|
||||
outputs_dir = run_dir / "outputs"
|
||||
output_files: list[dict] = []
|
||||
if outputs_dir.is_dir():
|
||||
for f in sorted(outputs_dir.iterdir()):
|
||||
if f.is_file() and f.name not in METADATA_FILES:
|
||||
output_files.append(embed_file(f))
|
||||
|
||||
# Load grading if present
|
||||
grading = None
|
||||
for candidate in [run_dir / "grading.json", run_dir.parent / "grading.json"]:
|
||||
if candidate.exists():
|
||||
try:
|
||||
grading = json.loads(candidate.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
if grading:
|
||||
break
|
||||
|
||||
return {
|
||||
"id": run_id,
|
||||
"prompt": prompt,
|
||||
"eval_id": eval_id,
|
||||
"outputs": output_files,
|
||||
"grading": grading,
|
||||
}
|
||||
|
||||
|
||||
def embed_file(path: Path) -> dict:
|
||||
"""Read a file and return an embedded representation."""
|
||||
ext = path.suffix.lower()
|
||||
mime = get_mime_type(path)
|
||||
|
||||
if ext in TEXT_EXTENSIONS:
|
||||
try:
|
||||
content = path.read_text(errors="replace")
|
||||
except OSError:
|
||||
content = "(Error reading file)"
|
||||
return {
|
||||
"name": path.name,
|
||||
"type": "text",
|
||||
"content": content,
|
||||
}
|
||||
elif ext in IMAGE_EXTENSIONS:
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
b64 = base64.b64encode(raw).decode("ascii")
|
||||
except OSError:
|
||||
return {"name": path.name, "type": "error", "content": "(Error reading file)"}
|
||||
return {
|
||||
"name": path.name,
|
||||
"type": "image",
|
||||
"mime": mime,
|
||||
"data_uri": f"data:{mime};base64,{b64}",
|
||||
}
|
||||
elif ext == ".pdf":
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
b64 = base64.b64encode(raw).decode("ascii")
|
||||
except OSError:
|
||||
return {"name": path.name, "type": "error", "content": "(Error reading file)"}
|
||||
return {
|
||||
"name": path.name,
|
||||
"type": "pdf",
|
||||
"data_uri": f"data:{mime};base64,{b64}",
|
||||
}
|
||||
elif ext == ".xlsx":
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
b64 = base64.b64encode(raw).decode("ascii")
|
||||
except OSError:
|
||||
return {"name": path.name, "type": "error", "content": "(Error reading file)"}
|
||||
return {
|
||||
"name": path.name,
|
||||
"type": "xlsx",
|
||||
"data_b64": b64,
|
||||
}
|
||||
else:
|
||||
# Binary / unknown — base64 download link
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
b64 = base64.b64encode(raw).decode("ascii")
|
||||
except OSError:
|
||||
return {"name": path.name, "type": "error", "content": "(Error reading file)"}
|
||||
return {
|
||||
"name": path.name,
|
||||
"type": "binary",
|
||||
"mime": mime,
|
||||
"data_uri": f"data:{mime};base64,{b64}",
|
||||
}
|
||||
|
||||
|
||||
def load_previous_iteration(workspace: Path) -> dict[str, dict]:
|
||||
"""Load previous iteration's feedback and outputs.
|
||||
|
||||
Returns a map of run_id -> {"feedback": str, "outputs": list[dict]}.
|
||||
"""
|
||||
result: dict[str, dict] = {}
|
||||
|
||||
# Load feedback
|
||||
feedback_map: dict[str, str] = {}
|
||||
feedback_path = workspace / "feedback.json"
|
||||
if feedback_path.exists():
|
||||
try:
|
||||
data = json.loads(feedback_path.read_text())
|
||||
feedback_map = {
|
||||
r["run_id"]: r["feedback"]
|
||||
for r in data.get("reviews", [])
|
||||
if r.get("feedback", "").strip()
|
||||
}
|
||||
except (json.JSONDecodeError, OSError, KeyError):
|
||||
pass
|
||||
|
||||
# Load runs (to get outputs)
|
||||
prev_runs = find_runs(workspace)
|
||||
for run in prev_runs:
|
||||
result[run["id"]] = {
|
||||
"feedback": feedback_map.get(run["id"], ""),
|
||||
"outputs": run.get("outputs", []),
|
||||
}
|
||||
|
||||
# Also add feedback for run_ids that had feedback but no matching run
|
||||
for run_id, fb in feedback_map.items():
|
||||
if run_id not in result:
|
||||
result[run_id] = {"feedback": fb, "outputs": []}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def generate_html(
|
||||
runs: list[dict],
|
||||
skill_name: str,
|
||||
previous: dict[str, dict] | None = None,
|
||||
benchmark: dict | None = None,
|
||||
) -> str:
|
||||
"""Generate the complete standalone HTML page with embedded data."""
|
||||
template_path = Path(__file__).parent / "viewer.html"
|
||||
template = template_path.read_text()
|
||||
|
||||
# Build previous_feedback and previous_outputs maps for the template
|
||||
previous_feedback: dict[str, str] = {}
|
||||
previous_outputs: dict[str, list[dict]] = {}
|
||||
if previous:
|
||||
for run_id, data in previous.items():
|
||||
if data.get("feedback"):
|
||||
previous_feedback[run_id] = data["feedback"]
|
||||
if data.get("outputs"):
|
||||
previous_outputs[run_id] = data["outputs"]
|
||||
|
||||
embedded = {
|
||||
"skill_name": skill_name,
|
||||
"runs": runs,
|
||||
"previous_feedback": previous_feedback,
|
||||
"previous_outputs": previous_outputs,
|
||||
}
|
||||
if benchmark:
|
||||
embedded["benchmark"] = benchmark
|
||||
|
||||
data_json = json.dumps(embedded)
|
||||
|
||||
return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP server (stdlib only, zero dependencies)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _kill_port(port: int) -> None:
|
||||
"""Kill any process listening on the given port."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["lsof", "-ti", f":{port}"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
for pid_str in result.stdout.strip().split("\n"):
|
||||
if pid_str.strip():
|
||||
try:
|
||||
os.kill(int(pid_str.strip()), signal.SIGTERM)
|
||||
except (ProcessLookupError, ValueError):
|
||||
pass
|
||||
if result.stdout.strip():
|
||||
time.sleep(0.5)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
print("Note: lsof not found, cannot check if port is in use", file=sys.stderr)
|
||||
|
||||
class ReviewHandler(BaseHTTPRequestHandler):
|
||||
"""Serves the review HTML and handles feedback saves.
|
||||
|
||||
Regenerates the HTML on each page load so that refreshing the browser
|
||||
picks up new eval outputs without restarting the server.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
workspace: Path,
|
||||
skill_name: str,
|
||||
feedback_path: Path,
|
||||
previous: dict[str, dict],
|
||||
benchmark_path: Path | None,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
self.workspace = workspace
|
||||
self.skill_name = skill_name
|
||||
self.feedback_path = feedback_path
|
||||
self.previous = previous
|
||||
self.benchmark_path = benchmark_path
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/" or self.path == "/index.html":
|
||||
# Regenerate HTML on each request (re-scans workspace for new outputs)
|
||||
runs = find_runs(self.workspace)
|
||||
benchmark = None
|
||||
if self.benchmark_path and self.benchmark_path.exists():
|
||||
try:
|
||||
benchmark = json.loads(self.benchmark_path.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
html = generate_html(runs, self.skill_name, self.previous, benchmark)
|
||||
content = html.encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(content)))
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
elif self.path == "/api/feedback":
|
||||
data = b"{}"
|
||||
if self.feedback_path.exists():
|
||||
data = self.feedback_path.read_bytes()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if self.path == "/api/feedback":
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
try:
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, dict) or "reviews" not in data:
|
||||
raise ValueError("Expected JSON object with 'reviews' key")
|
||||
self.feedback_path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
resp = b'{"ok":true}'
|
||||
self.send_response(200)
|
||||
except (json.JSONDecodeError, OSError, ValueError) as e:
|
||||
resp = json.dumps({"error": str(e)}).encode()
|
||||
self.send_response(500)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(resp)))
|
||||
self.end_headers()
|
||||
self.wfile.write(resp)
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
# Suppress request logging to keep terminal clean
|
||||
pass
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Generate and serve eval review")
|
||||
parser.add_argument("workspace", type=Path, help="Path to workspace directory")
|
||||
parser.add_argument("--port", "-p", type=int, default=3117, help="Server port (default: 3117)")
|
||||
parser.add_argument("--skill-name", "-n", type=str, default=None, help="Skill name for header")
|
||||
parser.add_argument(
|
||||
"--previous-workspace", type=Path, default=None,
|
||||
help="Path to previous iteration's workspace (shows old outputs and feedback as context)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--benchmark", type=Path, default=None,
|
||||
help="Path to benchmark.json to show in the Benchmark tab",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--static", "-s", type=Path, default=None,
|
||||
help="Write standalone HTML to this path instead of starting a server",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace = args.workspace.resolve()
|
||||
if not workspace.is_dir():
|
||||
print(f"Error: {workspace} is not a directory", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
runs = find_runs(workspace)
|
||||
if not runs:
|
||||
print(f"No runs found in {workspace}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
skill_name = args.skill_name or workspace.name.replace("-workspace", "")
|
||||
feedback_path = workspace / "feedback.json"
|
||||
|
||||
previous: dict[str, dict] = {}
|
||||
if args.previous_workspace:
|
||||
previous = load_previous_iteration(args.previous_workspace.resolve())
|
||||
|
||||
benchmark_path = args.benchmark.resolve() if args.benchmark else None
|
||||
benchmark = None
|
||||
if benchmark_path and benchmark_path.exists():
|
||||
try:
|
||||
benchmark = json.loads(benchmark_path.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
if args.static:
|
||||
html = generate_html(runs, skill_name, previous, benchmark)
|
||||
args.static.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.static.write_text(html)
|
||||
print(f"\n Static viewer written to: {args.static}\n")
|
||||
sys.exit(0)
|
||||
|
||||
# Kill any existing process on the target port
|
||||
port = args.port
|
||||
_kill_port(port)
|
||||
handler = partial(ReviewHandler, workspace, skill_name, feedback_path, previous, benchmark_path)
|
||||
try:
|
||||
server = HTTPServer(("127.0.0.1", port), handler)
|
||||
except OSError:
|
||||
# Port still in use after kill attempt — find a free one
|
||||
server = HTTPServer(("127.0.0.1", 0), handler)
|
||||
port = server.server_address[1]
|
||||
|
||||
url = f"http://localhost:{port}"
|
||||
print(f"\n Eval Viewer")
|
||||
print(f" ─────────────────────────────────")
|
||||
print(f" URL: {url}")
|
||||
print(f" Workspace: {workspace}")
|
||||
print(f" Feedback: {feedback_path}")
|
||||
if previous:
|
||||
print(f" Previous: {args.previous_workspace} ({len(previous)} runs)")
|
||||
if benchmark_path:
|
||||
print(f" Benchmark: {benchmark_path}")
|
||||
print(f"\n Press Ctrl+C to stop.\n")
|
||||
|
||||
webbrowser.open(url)
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopped.")
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1325
skills/skill-creator/eval-viewer/viewer.html
Normal file
1325
skills/skill-creator/eval-viewer/viewer.html
Normal file
File diff suppressed because it is too large
Load Diff
430
skills/skill-creator/references/schemas.md
Normal file
430
skills/skill-creator/references/schemas.md
Normal file
@@ -0,0 +1,430 @@
|
||||
# JSON Schemas
|
||||
|
||||
This document defines the JSON schemas used by skill-creator.
|
||||
|
||||
---
|
||||
|
||||
## evals.json
|
||||
|
||||
Defines the evals for a skill. Located at `evals/evals.json` within the skill directory.
|
||||
|
||||
```json
|
||||
{
|
||||
"skill_name": "example-skill",
|
||||
"evals": [
|
||||
{
|
||||
"id": 1,
|
||||
"prompt": "User's example prompt",
|
||||
"expected_output": "Description of expected result",
|
||||
"files": ["evals/files/sample1.pdf"],
|
||||
"expectations": [
|
||||
"The output includes X",
|
||||
"The skill used script Y"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `skill_name`: Name matching the skill's frontmatter
|
||||
- `evals[].id`: Unique integer identifier
|
||||
- `evals[].prompt`: The task to execute
|
||||
- `evals[].expected_output`: Human-readable description of success
|
||||
- `evals[].files`: Optional list of input file paths (relative to skill root)
|
||||
- `evals[].expectations`: List of verifiable statements
|
||||
|
||||
---
|
||||
|
||||
## history.json
|
||||
|
||||
Tracks version progression in Improve mode. Located at workspace root.
|
||||
|
||||
```json
|
||||
{
|
||||
"started_at": "2026-01-15T10:30:00Z",
|
||||
"skill_name": "pdf",
|
||||
"current_best": "v2",
|
||||
"iterations": [
|
||||
{
|
||||
"version": "v0",
|
||||
"parent": null,
|
||||
"expectation_pass_rate": 0.65,
|
||||
"grading_result": "baseline",
|
||||
"is_current_best": false
|
||||
},
|
||||
{
|
||||
"version": "v1",
|
||||
"parent": "v0",
|
||||
"expectation_pass_rate": 0.75,
|
||||
"grading_result": "won",
|
||||
"is_current_best": false
|
||||
},
|
||||
{
|
||||
"version": "v2",
|
||||
"parent": "v1",
|
||||
"expectation_pass_rate": 0.85,
|
||||
"grading_result": "won",
|
||||
"is_current_best": true
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `started_at`: ISO timestamp of when improvement started
|
||||
- `skill_name`: Name of the skill being improved
|
||||
- `current_best`: Version identifier of the best performer
|
||||
- `iterations[].version`: Version identifier (v0, v1, ...)
|
||||
- `iterations[].parent`: Parent version this was derived from
|
||||
- `iterations[].expectation_pass_rate`: Pass rate from grading
|
||||
- `iterations[].grading_result`: "baseline", "won", "lost", or "tie"
|
||||
- `iterations[].is_current_best`: Whether this is the current best version
|
||||
|
||||
---
|
||||
|
||||
## grading.json
|
||||
|
||||
Output from the grader agent. Located at `<run-dir>/grading.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"expectations": [
|
||||
{
|
||||
"text": "The output includes the name 'John Smith'",
|
||||
"passed": true,
|
||||
"evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'"
|
||||
},
|
||||
{
|
||||
"text": "The spreadsheet has a SUM formula in cell B10",
|
||||
"passed": false,
|
||||
"evidence": "No spreadsheet was created. The output was a text file."
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"passed": 2,
|
||||
"failed": 1,
|
||||
"total": 3,
|
||||
"pass_rate": 0.67
|
||||
},
|
||||
"execution_metrics": {
|
||||
"tool_calls": {
|
||||
"Read": 5,
|
||||
"Write": 2,
|
||||
"Bash": 8
|
||||
},
|
||||
"total_tool_calls": 15,
|
||||
"total_steps": 6,
|
||||
"errors_encountered": 0,
|
||||
"output_chars": 12450,
|
||||
"transcript_chars": 3200
|
||||
},
|
||||
"timing": {
|
||||
"executor_duration_seconds": 165.0,
|
||||
"grader_duration_seconds": 26.0,
|
||||
"total_duration_seconds": 191.0
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"claim": "The form has 12 fillable fields",
|
||||
"type": "factual",
|
||||
"verified": true,
|
||||
"evidence": "Counted 12 fields in field_info.json"
|
||||
}
|
||||
],
|
||||
"user_notes_summary": {
|
||||
"uncertainties": ["Used 2023 data, may be stale"],
|
||||
"needs_review": [],
|
||||
"workarounds": ["Fell back to text overlay for non-fillable fields"]
|
||||
},
|
||||
"eval_feedback": {
|
||||
"suggestions": [
|
||||
{
|
||||
"assertion": "The output includes the name 'John Smith'",
|
||||
"reason": "A hallucinated document that mentions the name would also pass"
|
||||
}
|
||||
],
|
||||
"overall": "Assertions check presence but not correctness."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `expectations[]`: Graded expectations with evidence
|
||||
- `summary`: Aggregate pass/fail counts
|
||||
- `execution_metrics`: Tool usage and output size (from executor's metrics.json)
|
||||
- `timing`: Wall clock timing (from timing.json)
|
||||
- `claims`: Extracted and verified claims from the output
|
||||
- `user_notes_summary`: Issues flagged by the executor
|
||||
- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising
|
||||
|
||||
---
|
||||
|
||||
## metrics.json
|
||||
|
||||
Output from the executor agent. Located at `<run-dir>/outputs/metrics.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"tool_calls": {
|
||||
"Read": 5,
|
||||
"Write": 2,
|
||||
"Bash": 8,
|
||||
"Edit": 1,
|
||||
"Glob": 2,
|
||||
"Grep": 0
|
||||
},
|
||||
"total_tool_calls": 18,
|
||||
"total_steps": 6,
|
||||
"files_created": ["filled_form.pdf", "field_values.json"],
|
||||
"errors_encountered": 0,
|
||||
"output_chars": 12450,
|
||||
"transcript_chars": 3200
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `tool_calls`: Count per tool type
|
||||
- `total_tool_calls`: Sum of all tool calls
|
||||
- `total_steps`: Number of major execution steps
|
||||
- `files_created`: List of output files created
|
||||
- `errors_encountered`: Number of errors during execution
|
||||
- `output_chars`: Total character count of output files
|
||||
- `transcript_chars`: Character count of transcript
|
||||
|
||||
---
|
||||
|
||||
## timing.json
|
||||
|
||||
Wall clock timing for a run. Located at `<run-dir>/timing.json`.
|
||||
|
||||
**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact.
|
||||
|
||||
```json
|
||||
{
|
||||
"total_tokens": 84852,
|
||||
"duration_ms": 23332,
|
||||
"total_duration_seconds": 23.3,
|
||||
"executor_start": "2026-01-15T10:30:00Z",
|
||||
"executor_end": "2026-01-15T10:32:45Z",
|
||||
"executor_duration_seconds": 165.0,
|
||||
"grader_start": "2026-01-15T10:32:46Z",
|
||||
"grader_end": "2026-01-15T10:33:12Z",
|
||||
"grader_duration_seconds": 26.0
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## benchmark.json
|
||||
|
||||
Output from Benchmark mode. Located at `benchmarks/<timestamp>/benchmark.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"skill_name": "pdf",
|
||||
"skill_path": "/path/to/pdf",
|
||||
"executor_model": "claude-sonnet-4-20250514",
|
||||
"analyzer_model": "most-capable-model",
|
||||
"timestamp": "2026-01-15T10:30:00Z",
|
||||
"evals_run": [1, 2, 3],
|
||||
"runs_per_configuration": 3
|
||||
},
|
||||
|
||||
"runs": [
|
||||
{
|
||||
"eval_id": 1,
|
||||
"eval_name": "Ocean",
|
||||
"configuration": "with_skill",
|
||||
"run_number": 1,
|
||||
"result": {
|
||||
"pass_rate": 0.85,
|
||||
"passed": 6,
|
||||
"failed": 1,
|
||||
"total": 7,
|
||||
"time_seconds": 42.5,
|
||||
"tokens": 3800,
|
||||
"tool_calls": 18,
|
||||
"errors": 0
|
||||
},
|
||||
"expectations": [
|
||||
{"text": "...", "passed": true, "evidence": "..."}
|
||||
],
|
||||
"notes": [
|
||||
"Used 2023 data, may be stale",
|
||||
"Fell back to text overlay for non-fillable fields"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
"run_summary": {
|
||||
"with_skill": {
|
||||
"pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90},
|
||||
"time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0},
|
||||
"tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100}
|
||||
},
|
||||
"without_skill": {
|
||||
"pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45},
|
||||
"time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0},
|
||||
"tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500}
|
||||
},
|
||||
"delta": {
|
||||
"pass_rate": "+0.50",
|
||||
"time_seconds": "+13.0",
|
||||
"tokens": "+1700"
|
||||
}
|
||||
},
|
||||
|
||||
"notes": [
|
||||
"Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value",
|
||||
"Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent",
|
||||
"Without-skill runs consistently fail on table extraction expectations",
|
||||
"Skill adds 13s average execution time but improves pass rate by 50%"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
- `metadata`: Information about the benchmark run
|
||||
- `skill_name`: Name of the skill
|
||||
- `timestamp`: When the benchmark was run
|
||||
- `evals_run`: List of eval names or IDs
|
||||
- `runs_per_configuration`: Number of runs per config (e.g. 3)
|
||||
- `runs[]`: Individual run results
|
||||
- `eval_id`: Numeric eval identifier
|
||||
- `eval_name`: Human-readable eval name (used as section header in the viewer)
|
||||
- `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding)
|
||||
- `run_number`: Integer run number (1, 2, 3...)
|
||||
- `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors`
|
||||
- `run_summary`: Statistical aggregates per configuration
|
||||
- `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields
|
||||
- `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"`
|
||||
- `notes`: Freeform observations from the analyzer
|
||||
|
||||
**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually.
|
||||
|
||||
---
|
||||
|
||||
## comparison.json
|
||||
|
||||
Output from blind comparator. Located at `<grading-dir>/comparison-N.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"winner": "A",
|
||||
"reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.",
|
||||
"rubric": {
|
||||
"A": {
|
||||
"content": {
|
||||
"correctness": 5,
|
||||
"completeness": 5,
|
||||
"accuracy": 4
|
||||
},
|
||||
"structure": {
|
||||
"organization": 4,
|
||||
"formatting": 5,
|
||||
"usability": 4
|
||||
},
|
||||
"content_score": 4.7,
|
||||
"structure_score": 4.3,
|
||||
"overall_score": 9.0
|
||||
},
|
||||
"B": {
|
||||
"content": {
|
||||
"correctness": 3,
|
||||
"completeness": 2,
|
||||
"accuracy": 3
|
||||
},
|
||||
"structure": {
|
||||
"organization": 3,
|
||||
"formatting": 2,
|
||||
"usability": 3
|
||||
},
|
||||
"content_score": 2.7,
|
||||
"structure_score": 2.7,
|
||||
"overall_score": 5.4
|
||||
}
|
||||
},
|
||||
"output_quality": {
|
||||
"A": {
|
||||
"score": 9,
|
||||
"strengths": ["Complete solution", "Well-formatted", "All fields present"],
|
||||
"weaknesses": ["Minor style inconsistency in header"]
|
||||
},
|
||||
"B": {
|
||||
"score": 5,
|
||||
"strengths": ["Readable output", "Correct basic structure"],
|
||||
"weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"]
|
||||
}
|
||||
},
|
||||
"expectation_results": {
|
||||
"A": {
|
||||
"passed": 4,
|
||||
"total": 5,
|
||||
"pass_rate": 0.80,
|
||||
"details": [
|
||||
{"text": "Output includes name", "passed": true}
|
||||
]
|
||||
},
|
||||
"B": {
|
||||
"passed": 3,
|
||||
"total": 5,
|
||||
"pass_rate": 0.60,
|
||||
"details": [
|
||||
{"text": "Output includes name", "passed": true}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## analysis.json
|
||||
|
||||
Output from post-hoc analyzer. Located at `<grading-dir>/analysis.json`.
|
||||
|
||||
```json
|
||||
{
|
||||
"comparison_summary": {
|
||||
"winner": "A",
|
||||
"winner_skill": "path/to/winner/skill",
|
||||
"loser_skill": "path/to/loser/skill",
|
||||
"comparator_reasoning": "Brief summary of why comparator chose winner"
|
||||
},
|
||||
"winner_strengths": [
|
||||
"Clear step-by-step instructions for handling multi-page documents",
|
||||
"Included validation script that caught formatting errors"
|
||||
],
|
||||
"loser_weaknesses": [
|
||||
"Vague instruction 'process the document appropriately' led to inconsistent behavior",
|
||||
"No script for validation, agent had to improvise"
|
||||
],
|
||||
"instruction_following": {
|
||||
"winner": {
|
||||
"score": 9,
|
||||
"issues": ["Minor: skipped optional logging step"]
|
||||
},
|
||||
"loser": {
|
||||
"score": 6,
|
||||
"issues": [
|
||||
"Did not use the skill's formatting template",
|
||||
"Invented own approach instead of following step 3"
|
||||
]
|
||||
}
|
||||
},
|
||||
"improvement_suggestions": [
|
||||
{
|
||||
"priority": "high",
|
||||
"category": "instructions",
|
||||
"suggestion": "Replace 'process the document appropriately' with explicit steps",
|
||||
"expected_impact": "Would eliminate ambiguity that caused inconsistent behavior"
|
||||
}
|
||||
],
|
||||
"transcript_insights": {
|
||||
"winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script",
|
||||
"loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods"
|
||||
}
|
||||
}
|
||||
```
|
||||
0
skills/skill-creator/scripts/__init__.py
Normal file
0
skills/skill-creator/scripts/__init__.py
Normal file
401
skills/skill-creator/scripts/aggregate_benchmark.py
Normal file
401
skills/skill-creator/scripts/aggregate_benchmark.py
Normal file
@@ -0,0 +1,401 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate individual run results into benchmark summary statistics.
|
||||
|
||||
Reads grading.json files from run directories and produces:
|
||||
- run_summary with mean, stddev, min, max for each metric
|
||||
- delta between with_skill and without_skill configurations
|
||||
|
||||
Usage:
|
||||
python aggregate_benchmark.py <benchmark_dir>
|
||||
|
||||
Example:
|
||||
python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/
|
||||
|
||||
The script supports two directory layouts:
|
||||
|
||||
Workspace layout (from skill-creator iterations):
|
||||
<benchmark_dir>/
|
||||
└── eval-N/
|
||||
├── with_skill/
|
||||
│ ├── run-1/grading.json
|
||||
│ └── run-2/grading.json
|
||||
└── without_skill/
|
||||
├── run-1/grading.json
|
||||
└── run-2/grading.json
|
||||
|
||||
Legacy layout (with runs/ subdirectory):
|
||||
<benchmark_dir>/
|
||||
└── runs/
|
||||
└── eval-N/
|
||||
├── with_skill/
|
||||
│ └── run-1/grading.json
|
||||
└── without_skill/
|
||||
└── run-1/grading.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def calculate_stats(values: list[float]) -> dict:
|
||||
"""Calculate mean, stddev, min, max for a list of values."""
|
||||
if not values:
|
||||
return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}
|
||||
|
||||
n = len(values)
|
||||
mean = sum(values) / n
|
||||
|
||||
if n > 1:
|
||||
variance = sum((x - mean) ** 2 for x in values) / (n - 1)
|
||||
stddev = math.sqrt(variance)
|
||||
else:
|
||||
stddev = 0.0
|
||||
|
||||
return {
|
||||
"mean": round(mean, 4),
|
||||
"stddev": round(stddev, 4),
|
||||
"min": round(min(values), 4),
|
||||
"max": round(max(values), 4)
|
||||
}
|
||||
|
||||
|
||||
def load_run_results(benchmark_dir: Path) -> dict:
|
||||
"""
|
||||
Load all run results from a benchmark directory.
|
||||
|
||||
Returns dict keyed by config name (e.g. "with_skill"/"without_skill",
|
||||
or "new_skill"/"old_skill"), each containing a list of run results.
|
||||
"""
|
||||
# Support both layouts: eval dirs directly under benchmark_dir, or under runs/
|
||||
runs_dir = benchmark_dir / "runs"
|
||||
if runs_dir.exists():
|
||||
search_dir = runs_dir
|
||||
elif list(benchmark_dir.glob("eval-*")):
|
||||
search_dir = benchmark_dir
|
||||
else:
|
||||
print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}")
|
||||
return {}
|
||||
|
||||
results: dict[str, list] = {}
|
||||
|
||||
for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))):
|
||||
metadata_path = eval_dir / "eval_metadata.json"
|
||||
if metadata_path.exists():
|
||||
try:
|
||||
with open(metadata_path) as mf:
|
||||
eval_id = json.load(mf).get("eval_id", eval_idx)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
eval_id = eval_idx
|
||||
else:
|
||||
try:
|
||||
eval_id = int(eval_dir.name.split("-")[1])
|
||||
except ValueError:
|
||||
eval_id = eval_idx
|
||||
|
||||
# Discover config directories dynamically rather than hardcoding names
|
||||
for config_dir in sorted(eval_dir.iterdir()):
|
||||
if not config_dir.is_dir():
|
||||
continue
|
||||
# Skip non-config directories (inputs, outputs, etc.)
|
||||
if not list(config_dir.glob("run-*")):
|
||||
continue
|
||||
config = config_dir.name
|
||||
if config not in results:
|
||||
results[config] = []
|
||||
|
||||
for run_dir in sorted(config_dir.glob("run-*")):
|
||||
run_number = int(run_dir.name.split("-")[1])
|
||||
grading_file = run_dir / "grading.json"
|
||||
|
||||
if not grading_file.exists():
|
||||
print(f"Warning: grading.json not found in {run_dir}")
|
||||
continue
|
||||
|
||||
try:
|
||||
with open(grading_file) as f:
|
||||
grading = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Warning: Invalid JSON in {grading_file}: {e}")
|
||||
continue
|
||||
|
||||
# Extract metrics
|
||||
result = {
|
||||
"eval_id": eval_id,
|
||||
"run_number": run_number,
|
||||
"pass_rate": grading.get("summary", {}).get("pass_rate", 0.0),
|
||||
"passed": grading.get("summary", {}).get("passed", 0),
|
||||
"failed": grading.get("summary", {}).get("failed", 0),
|
||||
"total": grading.get("summary", {}).get("total", 0),
|
||||
}
|
||||
|
||||
# Extract timing — check grading.json first, then sibling timing.json
|
||||
timing = grading.get("timing", {})
|
||||
result["time_seconds"] = timing.get("total_duration_seconds", 0.0)
|
||||
timing_file = run_dir / "timing.json"
|
||||
if result["time_seconds"] == 0.0 and timing_file.exists():
|
||||
try:
|
||||
with open(timing_file) as tf:
|
||||
timing_data = json.load(tf)
|
||||
result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0)
|
||||
result["tokens"] = timing_data.get("total_tokens", 0)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Extract metrics if available
|
||||
metrics = grading.get("execution_metrics", {})
|
||||
result["tool_calls"] = metrics.get("total_tool_calls", 0)
|
||||
if not result.get("tokens"):
|
||||
result["tokens"] = metrics.get("output_chars", 0)
|
||||
result["errors"] = metrics.get("errors_encountered", 0)
|
||||
|
||||
# Extract expectations — viewer requires fields: text, passed, evidence
|
||||
raw_expectations = grading.get("expectations", [])
|
||||
for exp in raw_expectations:
|
||||
if "text" not in exp or "passed" not in exp:
|
||||
print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}")
|
||||
result["expectations"] = raw_expectations
|
||||
|
||||
# Extract notes from user_notes_summary
|
||||
notes_summary = grading.get("user_notes_summary", {})
|
||||
notes = []
|
||||
notes.extend(notes_summary.get("uncertainties", []))
|
||||
notes.extend(notes_summary.get("needs_review", []))
|
||||
notes.extend(notes_summary.get("workarounds", []))
|
||||
result["notes"] = notes
|
||||
|
||||
results[config].append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def aggregate_results(results: dict) -> dict:
|
||||
"""
|
||||
Aggregate run results into summary statistics.
|
||||
|
||||
Returns run_summary with stats for each configuration and delta.
|
||||
"""
|
||||
run_summary = {}
|
||||
configs = list(results.keys())
|
||||
|
||||
for config in configs:
|
||||
runs = results.get(config, [])
|
||||
|
||||
if not runs:
|
||||
run_summary[config] = {
|
||||
"pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
|
||||
"time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0},
|
||||
"tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0}
|
||||
}
|
||||
continue
|
||||
|
||||
pass_rates = [r["pass_rate"] for r in runs]
|
||||
times = [r["time_seconds"] for r in runs]
|
||||
tokens = [r.get("tokens", 0) for r in runs]
|
||||
|
||||
run_summary[config] = {
|
||||
"pass_rate": calculate_stats(pass_rates),
|
||||
"time_seconds": calculate_stats(times),
|
||||
"tokens": calculate_stats(tokens)
|
||||
}
|
||||
|
||||
# Calculate delta between the first two configs (if two exist)
|
||||
if len(configs) >= 2:
|
||||
primary = run_summary.get(configs[0], {})
|
||||
baseline = run_summary.get(configs[1], {})
|
||||
else:
|
||||
primary = run_summary.get(configs[0], {}) if configs else {}
|
||||
baseline = {}
|
||||
|
||||
delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0)
|
||||
delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0)
|
||||
delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0)
|
||||
|
||||
run_summary["delta"] = {
|
||||
"pass_rate": f"{delta_pass_rate:+.2f}",
|
||||
"time_seconds": f"{delta_time:+.1f}",
|
||||
"tokens": f"{delta_tokens:+.0f}"
|
||||
}
|
||||
|
||||
return run_summary
|
||||
|
||||
|
||||
def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict:
|
||||
"""
|
||||
Generate complete benchmark.json from run results.
|
||||
"""
|
||||
results = load_run_results(benchmark_dir)
|
||||
run_summary = aggregate_results(results)
|
||||
|
||||
# Build runs array for benchmark.json
|
||||
runs = []
|
||||
for config in results:
|
||||
for result in results[config]:
|
||||
runs.append({
|
||||
"eval_id": result["eval_id"],
|
||||
"configuration": config,
|
||||
"run_number": result["run_number"],
|
||||
"result": {
|
||||
"pass_rate": result["pass_rate"],
|
||||
"passed": result["passed"],
|
||||
"failed": result["failed"],
|
||||
"total": result["total"],
|
||||
"time_seconds": result["time_seconds"],
|
||||
"tokens": result.get("tokens", 0),
|
||||
"tool_calls": result.get("tool_calls", 0),
|
||||
"errors": result.get("errors", 0)
|
||||
},
|
||||
"expectations": result["expectations"],
|
||||
"notes": result["notes"]
|
||||
})
|
||||
|
||||
# Determine eval IDs from results
|
||||
eval_ids = sorted(set(
|
||||
r["eval_id"]
|
||||
for config in results.values()
|
||||
for r in config
|
||||
))
|
||||
|
||||
benchmark = {
|
||||
"metadata": {
|
||||
"skill_name": skill_name or "<skill-name>",
|
||||
"skill_path": skill_path or "<path/to/skill>",
|
||||
"executor_model": "<model-name>",
|
||||
"analyzer_model": "<model-name>",
|
||||
"timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"evals_run": eval_ids,
|
||||
"runs_per_configuration": 3
|
||||
},
|
||||
"runs": runs,
|
||||
"run_summary": run_summary,
|
||||
"notes": [] # To be filled by analyzer
|
||||
}
|
||||
|
||||
return benchmark
|
||||
|
||||
|
||||
def generate_markdown(benchmark: dict) -> str:
|
||||
"""Generate human-readable benchmark.md from benchmark data."""
|
||||
metadata = benchmark["metadata"]
|
||||
run_summary = benchmark["run_summary"]
|
||||
|
||||
# Determine config names (excluding "delta")
|
||||
configs = [k for k in run_summary if k != "delta"]
|
||||
config_a = configs[0] if len(configs) >= 1 else "config_a"
|
||||
config_b = configs[1] if len(configs) >= 2 else "config_b"
|
||||
label_a = config_a.replace("_", " ").title()
|
||||
label_b = config_b.replace("_", " ").title()
|
||||
|
||||
lines = [
|
||||
f"# Skill Benchmark: {metadata['skill_name']}",
|
||||
"",
|
||||
f"**Model**: {metadata['executor_model']}",
|
||||
f"**Date**: {metadata['timestamp']}",
|
||||
f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"| Metric | {label_a} | {label_b} | Delta |",
|
||||
"|--------|------------|---------------|-------|",
|
||||
]
|
||||
|
||||
a_summary = run_summary.get(config_a, {})
|
||||
b_summary = run_summary.get(config_b, {})
|
||||
delta = run_summary.get("delta", {})
|
||||
|
||||
# Format pass rate
|
||||
a_pr = a_summary.get("pass_rate", {})
|
||||
b_pr = b_summary.get("pass_rate", {})
|
||||
lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |")
|
||||
|
||||
# Format time
|
||||
a_time = a_summary.get("time_seconds", {})
|
||||
b_time = b_summary.get("time_seconds", {})
|
||||
lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |")
|
||||
|
||||
# Format tokens
|
||||
a_tokens = a_summary.get("tokens", {})
|
||||
b_tokens = b_summary.get("tokens", {})
|
||||
lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |")
|
||||
|
||||
# Notes section
|
||||
if benchmark.get("notes"):
|
||||
lines.extend([
|
||||
"",
|
||||
"## Notes",
|
||||
""
|
||||
])
|
||||
for note in benchmark["notes"]:
|
||||
lines.append(f"- {note}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Aggregate benchmark run results into summary statistics"
|
||||
)
|
||||
parser.add_argument(
|
||||
"benchmark_dir",
|
||||
type=Path,
|
||||
help="Path to the benchmark directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skill-name",
|
||||
default="",
|
||||
help="Name of the skill being benchmarked"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skill-path",
|
||||
default="",
|
||||
help="Path to the skill being benchmarked"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", "-o",
|
||||
type=Path,
|
||||
help="Output path for benchmark.json (default: <benchmark_dir>/benchmark.json)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.benchmark_dir.exists():
|
||||
print(f"Directory not found: {args.benchmark_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
# Generate benchmark
|
||||
benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path)
|
||||
|
||||
# Determine output paths
|
||||
output_json = args.output or (args.benchmark_dir / "benchmark.json")
|
||||
output_md = output_json.with_suffix(".md")
|
||||
|
||||
# Write benchmark.json
|
||||
with open(output_json, "w") as f:
|
||||
json.dump(benchmark, f, indent=2)
|
||||
print(f"Generated: {output_json}")
|
||||
|
||||
# Write benchmark.md
|
||||
markdown = generate_markdown(benchmark)
|
||||
with open(output_md, "w") as f:
|
||||
f.write(markdown)
|
||||
print(f"Generated: {output_md}")
|
||||
|
||||
# Print summary
|
||||
run_summary = benchmark["run_summary"]
|
||||
configs = [k for k in run_summary if k != "delta"]
|
||||
delta = run_summary.get("delta", {})
|
||||
|
||||
print(f"\nSummary:")
|
||||
for config in configs:
|
||||
pr = run_summary[config]["pass_rate"]["mean"]
|
||||
label = config.replace("_", " ").title()
|
||||
print(f" {label}: {pr*100:.1f}% pass rate")
|
||||
print(f" Delta: {delta.get('pass_rate', '—')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
326
skills/skill-creator/scripts/generate_report.py
Normal file
326
skills/skill-creator/scripts/generate_report.py
Normal file
@@ -0,0 +1,326 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate an HTML report from run_loop.py output.
|
||||
|
||||
Takes the JSON output from run_loop.py and generates a visual HTML report
|
||||
showing each description attempt with check/x for each test case.
|
||||
Distinguishes between train and test queries.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
|
||||
"""Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
|
||||
history = data.get("history", [])
|
||||
holdout = data.get("holdout", 0)
|
||||
title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
|
||||
|
||||
# Get all unique queries from train and test sets, with should_trigger info
|
||||
train_queries: list[dict] = []
|
||||
test_queries: list[dict] = []
|
||||
if history:
|
||||
for r in history[0].get("train_results", history[0].get("results", [])):
|
||||
train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
|
||||
if history[0].get("test_results"):
|
||||
for r in history[0].get("test_results", []):
|
||||
test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
|
||||
|
||||
refresh_tag = ' <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
|
||||
|
||||
html_parts = ["""<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
""" + refresh_tag + """ <title>""" + title_prefix + """Skill Description Optimization</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
body {
|
||||
font-family: 'Lora', Georgia, serif;
|
||||
max-width: 100%;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
background: #faf9f5;
|
||||
color: #141413;
|
||||
}
|
||||
h1 { font-family: 'Poppins', sans-serif; color: #141413; }
|
||||
.explainer {
|
||||
background: white;
|
||||
padding: 15px;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 20px;
|
||||
border: 1px solid #e8e6dc;
|
||||
color: #b0aea5;
|
||||
font-size: 0.875rem;
|
||||
line-height: 1.6;
|
||||
}
|
||||
.summary {
|
||||
background: white;
|
||||
padding: 15px;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 20px;
|
||||
border: 1px solid #e8e6dc;
|
||||
}
|
||||
.summary p { margin: 5px 0; }
|
||||
.best { color: #788c5d; font-weight: bold; }
|
||||
.table-container {
|
||||
overflow-x: auto;
|
||||
width: 100%;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
background: white;
|
||||
border: 1px solid #e8e6dc;
|
||||
border-radius: 6px;
|
||||
font-size: 12px;
|
||||
min-width: 100%;
|
||||
}
|
||||
th, td {
|
||||
padding: 8px;
|
||||
text-align: left;
|
||||
border: 1px solid #e8e6dc;
|
||||
white-space: normal;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
th {
|
||||
font-family: 'Poppins', sans-serif;
|
||||
background: #141413;
|
||||
color: #faf9f5;
|
||||
font-weight: 500;
|
||||
}
|
||||
th.test-col {
|
||||
background: #6a9bcc;
|
||||
}
|
||||
th.query-col { min-width: 200px; }
|
||||
td.description {
|
||||
font-family: monospace;
|
||||
font-size: 11px;
|
||||
word-wrap: break-word;
|
||||
max-width: 400px;
|
||||
}
|
||||
td.result {
|
||||
text-align: center;
|
||||
font-size: 16px;
|
||||
min-width: 40px;
|
||||
}
|
||||
td.test-result {
|
||||
background: #f0f6fc;
|
||||
}
|
||||
.pass { color: #788c5d; }
|
||||
.fail { color: #c44; }
|
||||
.rate {
|
||||
font-size: 9px;
|
||||
color: #b0aea5;
|
||||
display: block;
|
||||
}
|
||||
tr:hover { background: #faf9f5; }
|
||||
.score {
|
||||
display: inline-block;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-weight: bold;
|
||||
font-size: 11px;
|
||||
}
|
||||
.score-good { background: #eef2e8; color: #788c5d; }
|
||||
.score-ok { background: #fef3c7; color: #d97706; }
|
||||
.score-bad { background: #fceaea; color: #c44; }
|
||||
.train-label { color: #b0aea5; font-size: 10px; }
|
||||
.test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
|
||||
.best-row { background: #f5f8f2; }
|
||||
th.positive-col { border-bottom: 3px solid #788c5d; }
|
||||
th.negative-col { border-bottom: 3px solid #c44; }
|
||||
th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
|
||||
th.test-col.negative-col { border-bottom: 3px solid #c44; }
|
||||
.legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
|
||||
.legend-item { display: flex; align-items: center; gap: 6px; }
|
||||
.legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
|
||||
.swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
|
||||
.swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
|
||||
.swatch-test { background: #6a9bcc; }
|
||||
.swatch-train { background: #141413; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>""" + title_prefix + """Skill Description Optimization</h1>
|
||||
<div class="explainer">
|
||||
<strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
|
||||
</div>
|
||||
"""]
|
||||
|
||||
# Summary section
|
||||
best_test_score = data.get('best_test_score')
|
||||
best_train_score = data.get('best_train_score')
|
||||
html_parts.append(f"""
|
||||
<div class="summary">
|
||||
<p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
|
||||
<p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
|
||||
<p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
|
||||
<p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
|
||||
</div>
|
||||
""")
|
||||
|
||||
# Legend
|
||||
html_parts.append("""
|
||||
<div class="legend">
|
||||
<span style="font-weight:600">Query columns:</span>
|
||||
<span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
|
||||
<span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
|
||||
<span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
|
||||
<span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
|
||||
</div>
|
||||
""")
|
||||
|
||||
# Table header
|
||||
html_parts.append("""
|
||||
<div class="table-container">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Iter</th>
|
||||
<th>Train</th>
|
||||
<th>Test</th>
|
||||
<th class="query-col">Description</th>
|
||||
""")
|
||||
|
||||
# Add column headers for train queries
|
||||
for qinfo in train_queries:
|
||||
polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
|
||||
html_parts.append(f' <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
|
||||
|
||||
# Add column headers for test queries (different color)
|
||||
for qinfo in test_queries:
|
||||
polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
|
||||
html_parts.append(f' <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
|
||||
|
||||
html_parts.append(""" </tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
""")
|
||||
|
||||
# Find best iteration for highlighting
|
||||
if test_queries:
|
||||
best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
|
||||
else:
|
||||
best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
|
||||
|
||||
# Add rows for each iteration
|
||||
for h in history:
|
||||
iteration = h.get("iteration", "?")
|
||||
train_passed = h.get("train_passed", h.get("passed", 0))
|
||||
train_total = h.get("train_total", h.get("total", 0))
|
||||
test_passed = h.get("test_passed")
|
||||
test_total = h.get("test_total")
|
||||
description = h.get("description", "")
|
||||
train_results = h.get("train_results", h.get("results", []))
|
||||
test_results = h.get("test_results", [])
|
||||
|
||||
# Create lookups for results by query
|
||||
train_by_query = {r["query"]: r for r in train_results}
|
||||
test_by_query = {r["query"]: r for r in test_results} if test_results else {}
|
||||
|
||||
# Compute aggregate correct/total runs across all retries
|
||||
def aggregate_runs(results: list[dict]) -> tuple[int, int]:
|
||||
correct = 0
|
||||
total = 0
|
||||
for r in results:
|
||||
runs = r.get("runs", 0)
|
||||
triggers = r.get("triggers", 0)
|
||||
total += runs
|
||||
if r.get("should_trigger", True):
|
||||
correct += triggers
|
||||
else:
|
||||
correct += runs - triggers
|
||||
return correct, total
|
||||
|
||||
train_correct, train_runs = aggregate_runs(train_results)
|
||||
test_correct, test_runs = aggregate_runs(test_results)
|
||||
|
||||
# Determine score classes
|
||||
def score_class(correct: int, total: int) -> str:
|
||||
if total > 0:
|
||||
ratio = correct / total
|
||||
if ratio >= 0.8:
|
||||
return "score-good"
|
||||
elif ratio >= 0.5:
|
||||
return "score-ok"
|
||||
return "score-bad"
|
||||
|
||||
train_class = score_class(train_correct, train_runs)
|
||||
test_class = score_class(test_correct, test_runs)
|
||||
|
||||
row_class = "best-row" if iteration == best_iter else ""
|
||||
|
||||
html_parts.append(f""" <tr class="{row_class}">
|
||||
<td>{iteration}</td>
|
||||
<td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
|
||||
<td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
|
||||
<td class="description">{html.escape(description)}</td>
|
||||
""")
|
||||
|
||||
# Add result for each train query
|
||||
for qinfo in train_queries:
|
||||
r = train_by_query.get(qinfo["query"], {})
|
||||
did_pass = r.get("pass", False)
|
||||
triggers = r.get("triggers", 0)
|
||||
runs = r.get("runs", 0)
|
||||
|
||||
icon = "✓" if did_pass else "✗"
|
||||
css_class = "pass" if did_pass else "fail"
|
||||
|
||||
html_parts.append(f' <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
|
||||
|
||||
# Add result for each test query (with different background)
|
||||
for qinfo in test_queries:
|
||||
r = test_by_query.get(qinfo["query"], {})
|
||||
did_pass = r.get("pass", False)
|
||||
triggers = r.get("triggers", 0)
|
||||
runs = r.get("runs", 0)
|
||||
|
||||
icon = "✓" if did_pass else "✗"
|
||||
css_class = "pass" if did_pass else "fail"
|
||||
|
||||
html_parts.append(f' <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
|
||||
|
||||
html_parts.append(" </tr>\n")
|
||||
|
||||
html_parts.append(""" </tbody>
|
||||
</table>
|
||||
</div>
|
||||
""")
|
||||
|
||||
html_parts.append("""
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
return "".join(html_parts)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
|
||||
parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
|
||||
parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
|
||||
parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.input == "-":
|
||||
data = json.load(sys.stdin)
|
||||
else:
|
||||
data = json.loads(Path(args.input).read_text())
|
||||
|
||||
html_output = generate_html(data, skill_name=args.skill_name)
|
||||
|
||||
if args.output:
|
||||
Path(args.output).write_text(html_output)
|
||||
print(f"Report written to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(html_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
247
skills/skill-creator/scripts/improve_description.py
Normal file
247
skills/skill-creator/scripts/improve_description.py
Normal file
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Improve a skill description based on eval results.
|
||||
|
||||
Takes eval results (from run_eval.py) and generates an improved description
|
||||
by calling `claude -p` as a subprocess (same auth pattern as run_eval.py —
|
||||
uses the session's Claude Code auth, no separate ANTHROPIC_API_KEY needed).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.utils import parse_skill_md
|
||||
|
||||
|
||||
def _call_claude(prompt: str, model: str | None, timeout: int = 300) -> str:
|
||||
"""Run `claude -p` with the prompt on stdin and return the text response.
|
||||
|
||||
Prompt goes over stdin (not argv) because it embeds the full SKILL.md
|
||||
body and can easily exceed comfortable argv length.
|
||||
"""
|
||||
cmd = ["claude", "-p", "--output-format", "text"]
|
||||
if model:
|
||||
cmd.extend(["--model", model])
|
||||
|
||||
# Remove CLAUDECODE env var to allow nesting claude -p inside a
|
||||
# Claude Code session. The guard is for interactive terminal conflicts;
|
||||
# programmatic subprocess usage is safe. Same pattern as run_eval.py.
|
||||
env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=prompt,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"claude -p exited {result.returncode}\nstderr: {result.stderr}"
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def improve_description(
|
||||
skill_name: str,
|
||||
skill_content: str,
|
||||
current_description: str,
|
||||
eval_results: dict,
|
||||
history: list[dict],
|
||||
model: str,
|
||||
test_results: dict | None = None,
|
||||
log_dir: Path | None = None,
|
||||
iteration: int | None = None,
|
||||
) -> str:
|
||||
"""Call Claude to improve the description based on eval results."""
|
||||
failed_triggers = [
|
||||
r for r in eval_results["results"]
|
||||
if r["should_trigger"] and not r["pass"]
|
||||
]
|
||||
false_triggers = [
|
||||
r for r in eval_results["results"]
|
||||
if not r["should_trigger"] and not r["pass"]
|
||||
]
|
||||
|
||||
# Build scores summary
|
||||
train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
|
||||
if test_results:
|
||||
test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
|
||||
scores_summary = f"Train: {train_score}, Test: {test_score}"
|
||||
else:
|
||||
scores_summary = f"Train: {train_score}"
|
||||
|
||||
prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
|
||||
|
||||
The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
|
||||
|
||||
Here's the current description:
|
||||
<current_description>
|
||||
"{current_description}"
|
||||
</current_description>
|
||||
|
||||
Current scores ({scores_summary}):
|
||||
<scores_summary>
|
||||
"""
|
||||
if failed_triggers:
|
||||
prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n"
|
||||
for r in failed_triggers:
|
||||
prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
|
||||
prompt += "\n"
|
||||
|
||||
if false_triggers:
|
||||
prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n"
|
||||
for r in false_triggers:
|
||||
prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
|
||||
prompt += "\n"
|
||||
|
||||
if history:
|
||||
prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n"
|
||||
for h in history:
|
||||
train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}"
|
||||
test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None
|
||||
score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "")
|
||||
prompt += f'<attempt {score_str}>\n'
|
||||
prompt += f'Description: "{h["description"]}"\n'
|
||||
if "results" in h:
|
||||
prompt += "Train results:\n"
|
||||
for r in h["results"]:
|
||||
status = "PASS" if r["pass"] else "FAIL"
|
||||
prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n'
|
||||
if h.get("note"):
|
||||
prompt += f'Note: {h["note"]}\n'
|
||||
prompt += "</attempt>\n\n"
|
||||
|
||||
prompt += f"""</scores_summary>
|
||||
|
||||
Skill content (for context on what the skill does):
|
||||
<skill_content>
|
||||
{skill_content}
|
||||
</skill_content>
|
||||
|
||||
Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold:
|
||||
|
||||
1. Avoid overfitting
|
||||
2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description.
|
||||
|
||||
Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy. There is a hard limit of 1024 characters — descriptions over that will be truncated, so stay comfortably under it.
|
||||
|
||||
Here are some tips that we've found to work well in writing these descriptions:
|
||||
- The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does"
|
||||
- The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works.
|
||||
- The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable.
|
||||
- If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings.
|
||||
|
||||
I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end.
|
||||
|
||||
Please respond with only the new description text in <new_description> tags, nothing else."""
|
||||
|
||||
text = _call_claude(prompt, model)
|
||||
|
||||
match = re.search(r"<new_description>(.*?)</new_description>", text, re.DOTALL)
|
||||
description = match.group(1).strip().strip('"') if match else text.strip().strip('"')
|
||||
|
||||
transcript: dict = {
|
||||
"iteration": iteration,
|
||||
"prompt": prompt,
|
||||
"response": text,
|
||||
"parsed_description": description,
|
||||
"char_count": len(description),
|
||||
"over_limit": len(description) > 1024,
|
||||
}
|
||||
|
||||
# Safety net: the prompt already states the 1024-char hard limit, but if
|
||||
# the model blew past it anyway, make one fresh single-turn call that
|
||||
# quotes the too-long version and asks for a shorter rewrite. (The old
|
||||
# SDK path did this as a true multi-turn; `claude -p` is one-shot, so we
|
||||
# inline the prior output into the new prompt instead.)
|
||||
if len(description) > 1024:
|
||||
shorten_prompt = (
|
||||
f"{prompt}\n\n"
|
||||
f"---\n\n"
|
||||
f"A previous attempt produced this description, which at "
|
||||
f"{len(description)} characters is over the 1024-character hard limit:\n\n"
|
||||
f'"{description}"\n\n'
|
||||
f"Rewrite it to be under 1024 characters while keeping the most "
|
||||
f"important trigger words and intent coverage. Respond with only "
|
||||
f"the new description in <new_description> tags."
|
||||
)
|
||||
shorten_text = _call_claude(shorten_prompt, model)
|
||||
match = re.search(r"<new_description>(.*?)</new_description>", shorten_text, re.DOTALL)
|
||||
shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"')
|
||||
|
||||
transcript["rewrite_prompt"] = shorten_prompt
|
||||
transcript["rewrite_response"] = shorten_text
|
||||
transcript["rewrite_description"] = shortened
|
||||
transcript["rewrite_char_count"] = len(shortened)
|
||||
description = shortened
|
||||
|
||||
transcript["final_description"] = description
|
||||
|
||||
if log_dir:
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json"
|
||||
log_file.write_text(json.dumps(transcript, indent=2))
|
||||
|
||||
return description
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Improve a skill description based on eval results")
|
||||
parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)")
|
||||
parser.add_argument("--skill-path", required=True, help="Path to skill directory")
|
||||
parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)")
|
||||
parser.add_argument("--model", required=True, help="Model for improvement")
|
||||
parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr")
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_path = Path(args.skill_path)
|
||||
if not (skill_path / "SKILL.md").exists():
|
||||
print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
eval_results = json.loads(Path(args.eval_results).read_text())
|
||||
history = []
|
||||
if args.history:
|
||||
history = json.loads(Path(args.history).read_text())
|
||||
|
||||
name, _, content = parse_skill_md(skill_path)
|
||||
current_description = eval_results["description"]
|
||||
|
||||
if args.verbose:
|
||||
print(f"Current: {current_description}", file=sys.stderr)
|
||||
print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr)
|
||||
|
||||
new_description = improve_description(
|
||||
skill_name=name,
|
||||
skill_content=content,
|
||||
current_description=current_description,
|
||||
eval_results=eval_results,
|
||||
history=history,
|
||||
model=args.model,
|
||||
)
|
||||
|
||||
if args.verbose:
|
||||
print(f"Improved: {new_description}", file=sys.stderr)
|
||||
|
||||
# Output as JSON with both the new description and updated history
|
||||
output = {
|
||||
"description": new_description,
|
||||
"history": history + [{
|
||||
"description": current_description,
|
||||
"passed": eval_results["summary"]["passed"],
|
||||
"failed": eval_results["summary"]["failed"],
|
||||
"total": eval_results["summary"]["total"],
|
||||
"results": eval_results["results"],
|
||||
}],
|
||||
}
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
136
skills/skill-creator/scripts/package_skill.py
Normal file
136
skills/skill-creator/scripts/package_skill.py
Normal file
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Skill Packager - Creates a distributable .skill file of a skill folder
|
||||
|
||||
Usage:
|
||||
python utils/package_skill.py <path/to/skill-folder> [output-directory]
|
||||
|
||||
Example:
|
||||
python utils/package_skill.py skills/public/my-skill
|
||||
python utils/package_skill.py skills/public/my-skill ./dist
|
||||
"""
|
||||
|
||||
import fnmatch
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from scripts.quick_validate import validate_skill
|
||||
|
||||
# Patterns to exclude when packaging skills.
|
||||
EXCLUDE_DIRS = {"__pycache__", "node_modules"}
|
||||
EXCLUDE_GLOBS = {"*.pyc"}
|
||||
EXCLUDE_FILES = {".DS_Store"}
|
||||
# Directories excluded only at the skill root (not when nested deeper).
|
||||
ROOT_EXCLUDE_DIRS = {"evals"}
|
||||
|
||||
|
||||
def should_exclude(rel_path: Path) -> bool:
|
||||
"""Check if a path should be excluded from packaging."""
|
||||
parts = rel_path.parts
|
||||
if any(part in EXCLUDE_DIRS for part in parts):
|
||||
return True
|
||||
# rel_path is relative to skill_path.parent, so parts[0] is the skill
|
||||
# folder name and parts[1] (if present) is the first subdir.
|
||||
if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS:
|
||||
return True
|
||||
name = rel_path.name
|
||||
if name in EXCLUDE_FILES:
|
||||
return True
|
||||
return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS)
|
||||
|
||||
|
||||
def package_skill(skill_path, output_dir=None):
|
||||
"""
|
||||
Package a skill folder into a .skill file.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill folder
|
||||
output_dir: Optional output directory for the .skill file (defaults to current directory)
|
||||
|
||||
Returns:
|
||||
Path to the created .skill file, or None if error
|
||||
"""
|
||||
skill_path = Path(skill_path).resolve()
|
||||
|
||||
# Validate skill folder exists
|
||||
if not skill_path.exists():
|
||||
print(f"❌ Error: Skill folder not found: {skill_path}")
|
||||
return None
|
||||
|
||||
if not skill_path.is_dir():
|
||||
print(f"❌ Error: Path is not a directory: {skill_path}")
|
||||
return None
|
||||
|
||||
# Validate SKILL.md exists
|
||||
skill_md = skill_path / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
print(f"❌ Error: SKILL.md not found in {skill_path}")
|
||||
return None
|
||||
|
||||
# Run validation before packaging
|
||||
print("🔍 Validating skill...")
|
||||
valid, message = validate_skill(skill_path)
|
||||
if not valid:
|
||||
print(f"❌ Validation failed: {message}")
|
||||
print(" Please fix the validation errors before packaging.")
|
||||
return None
|
||||
print(f"✅ {message}\n")
|
||||
|
||||
# Determine output location
|
||||
skill_name = skill_path.name
|
||||
if output_dir:
|
||||
output_path = Path(output_dir).resolve()
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
output_path = Path.cwd()
|
||||
|
||||
skill_filename = output_path / f"{skill_name}.skill"
|
||||
|
||||
# Create the .skill file (zip format)
|
||||
try:
|
||||
with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
|
||||
# Walk through the skill directory, excluding build artifacts
|
||||
for file_path in skill_path.rglob('*'):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
arcname = file_path.relative_to(skill_path.parent)
|
||||
if should_exclude(arcname):
|
||||
print(f" Skipped: {arcname}")
|
||||
continue
|
||||
zipf.write(file_path, arcname)
|
||||
print(f" Added: {arcname}")
|
||||
|
||||
print(f"\n✅ Successfully packaged skill to: {skill_filename}")
|
||||
return skill_filename
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error creating .skill file: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python utils/package_skill.py <path/to/skill-folder> [output-directory]")
|
||||
print("\nExample:")
|
||||
print(" python utils/package_skill.py skills/public/my-skill")
|
||||
print(" python utils/package_skill.py skills/public/my-skill ./dist")
|
||||
sys.exit(1)
|
||||
|
||||
skill_path = sys.argv[1]
|
||||
output_dir = sys.argv[2] if len(sys.argv) > 2 else None
|
||||
|
||||
print(f"📦 Packaging skill: {skill_path}")
|
||||
if output_dir:
|
||||
print(f" Output directory: {output_dir}")
|
||||
print()
|
||||
|
||||
result = package_skill(skill_path, output_dir)
|
||||
|
||||
if result:
|
||||
sys.exit(0)
|
||||
else:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
103
skills/skill-creator/scripts/quick_validate.py
Normal file
103
skills/skill-creator/scripts/quick_validate.py
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick validation script for skills - minimal version
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
def validate_skill(skill_path):
|
||||
"""Basic validation of a skill"""
|
||||
skill_path = Path(skill_path)
|
||||
|
||||
# Check SKILL.md exists
|
||||
skill_md = skill_path / 'SKILL.md'
|
||||
if not skill_md.exists():
|
||||
return False, "SKILL.md not found"
|
||||
|
||||
# Read and validate frontmatter
|
||||
content = skill_md.read_text()
|
||||
if not content.startswith('---'):
|
||||
return False, "No YAML frontmatter found"
|
||||
|
||||
# Extract frontmatter
|
||||
match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
|
||||
if not match:
|
||||
return False, "Invalid frontmatter format"
|
||||
|
||||
frontmatter_text = match.group(1)
|
||||
|
||||
# Parse YAML frontmatter
|
||||
try:
|
||||
frontmatter = yaml.safe_load(frontmatter_text)
|
||||
if not isinstance(frontmatter, dict):
|
||||
return False, "Frontmatter must be a YAML dictionary"
|
||||
except yaml.YAMLError as e:
|
||||
return False, f"Invalid YAML in frontmatter: {e}"
|
||||
|
||||
# Define allowed properties
|
||||
ALLOWED_PROPERTIES = {'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility'}
|
||||
|
||||
# Check for unexpected properties (excluding nested keys under metadata)
|
||||
unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES
|
||||
if unexpected_keys:
|
||||
return False, (
|
||||
f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. "
|
||||
f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}"
|
||||
)
|
||||
|
||||
# Check required fields
|
||||
if 'name' not in frontmatter:
|
||||
return False, "Missing 'name' in frontmatter"
|
||||
if 'description' not in frontmatter:
|
||||
return False, "Missing 'description' in frontmatter"
|
||||
|
||||
# Extract name for validation
|
||||
name = frontmatter.get('name', '')
|
||||
if not isinstance(name, str):
|
||||
return False, f"Name must be a string, got {type(name).__name__}"
|
||||
name = name.strip()
|
||||
if name:
|
||||
# Check naming convention (kebab-case: lowercase with hyphens)
|
||||
if not re.match(r'^[a-z0-9-]+$', name):
|
||||
return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)"
|
||||
if name.startswith('-') or name.endswith('-') or '--' in name:
|
||||
return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens"
|
||||
# Check name length (max 64 characters per spec)
|
||||
if len(name) > 64:
|
||||
return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters."
|
||||
|
||||
# Extract and validate description
|
||||
description = frontmatter.get('description', '')
|
||||
if not isinstance(description, str):
|
||||
return False, f"Description must be a string, got {type(description).__name__}"
|
||||
description = description.strip()
|
||||
if description:
|
||||
# Check for angle brackets
|
||||
if '<' in description or '>' in description:
|
||||
return False, "Description cannot contain angle brackets (< or >)"
|
||||
# Check description length (max 1024 characters per spec)
|
||||
if len(description) > 1024:
|
||||
return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters."
|
||||
|
||||
# Validate compatibility field if present (optional)
|
||||
compatibility = frontmatter.get('compatibility', '')
|
||||
if compatibility:
|
||||
if not isinstance(compatibility, str):
|
||||
return False, f"Compatibility must be a string, got {type(compatibility).__name__}"
|
||||
if len(compatibility) > 500:
|
||||
return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters."
|
||||
|
||||
return True, "Skill is valid!"
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python quick_validate.py <skill_directory>")
|
||||
sys.exit(1)
|
||||
|
||||
valid, message = validate_skill(sys.argv[1])
|
||||
print(message)
|
||||
sys.exit(0 if valid else 1)
|
||||
310
skills/skill-creator/scripts/run_eval.py
Normal file
310
skills/skill-creator/scripts/run_eval.py
Normal file
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run trigger evaluation for a skill description.
|
||||
|
||||
Tests whether a skill's description causes Claude to trigger (read the skill)
|
||||
for a set of queries. Outputs results as JSON.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.utils import parse_skill_md
|
||||
|
||||
|
||||
def find_project_root() -> Path:
|
||||
"""Find the project root by walking up from cwd looking for .claude/.
|
||||
|
||||
Mimics how Claude Code discovers its project root, so the command file
|
||||
we create ends up where claude -p will look for it.
|
||||
"""
|
||||
current = Path.cwd()
|
||||
for parent in [current, *current.parents]:
|
||||
if (parent / ".claude").is_dir():
|
||||
return parent
|
||||
return current
|
||||
|
||||
|
||||
def run_single_query(
|
||||
query: str,
|
||||
skill_name: str,
|
||||
skill_description: str,
|
||||
timeout: int,
|
||||
project_root: str,
|
||||
model: str | None = None,
|
||||
) -> bool:
|
||||
"""Run a single query and return whether the skill was triggered.
|
||||
|
||||
Creates a command file in .claude/commands/ so it appears in Claude's
|
||||
available_skills list, then runs `claude -p` with the raw query.
|
||||
Uses --include-partial-messages to detect triggering early from
|
||||
stream events (content_block_start) rather than waiting for the
|
||||
full assistant message, which only arrives after tool execution.
|
||||
"""
|
||||
unique_id = uuid.uuid4().hex[:8]
|
||||
clean_name = f"{skill_name}-skill-{unique_id}"
|
||||
project_commands_dir = Path(project_root) / ".claude" / "commands"
|
||||
command_file = project_commands_dir / f"{clean_name}.md"
|
||||
|
||||
try:
|
||||
project_commands_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Use YAML block scalar to avoid breaking on quotes in description
|
||||
indented_desc = "\n ".join(skill_description.split("\n"))
|
||||
command_content = (
|
||||
f"---\n"
|
||||
f"description: |\n"
|
||||
f" {indented_desc}\n"
|
||||
f"---\n\n"
|
||||
f"# {skill_name}\n\n"
|
||||
f"This skill handles: {skill_description}\n"
|
||||
)
|
||||
command_file.write_text(command_content)
|
||||
|
||||
cmd = [
|
||||
"claude",
|
||||
"-p", query,
|
||||
"--output-format", "stream-json",
|
||||
"--verbose",
|
||||
"--include-partial-messages",
|
||||
]
|
||||
if model:
|
||||
cmd.extend(["--model", model])
|
||||
|
||||
# Remove CLAUDECODE env var to allow nesting claude -p inside a
|
||||
# Claude Code session. The guard is for interactive terminal conflicts;
|
||||
# programmatic subprocess usage is safe.
|
||||
env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
cwd=project_root,
|
||||
env=env,
|
||||
)
|
||||
|
||||
triggered = False
|
||||
start_time = time.time()
|
||||
buffer = ""
|
||||
# Track state for stream event detection
|
||||
pending_tool_name = None
|
||||
accumulated_json = ""
|
||||
|
||||
try:
|
||||
while time.time() - start_time < timeout:
|
||||
if process.poll() is not None:
|
||||
remaining = process.stdout.read()
|
||||
if remaining:
|
||||
buffer += remaining.decode("utf-8", errors="replace")
|
||||
break
|
||||
|
||||
ready, _, _ = select.select([process.stdout], [], [], 1.0)
|
||||
if not ready:
|
||||
continue
|
||||
|
||||
chunk = os.read(process.stdout.fileno(), 8192)
|
||||
if not chunk:
|
||||
break
|
||||
buffer += chunk.decode("utf-8", errors="replace")
|
||||
|
||||
while "\n" in buffer:
|
||||
line, buffer = buffer.split("\n", 1)
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
# Early detection via stream events
|
||||
if event.get("type") == "stream_event":
|
||||
se = event.get("event", {})
|
||||
se_type = se.get("type", "")
|
||||
|
||||
if se_type == "content_block_start":
|
||||
cb = se.get("content_block", {})
|
||||
if cb.get("type") == "tool_use":
|
||||
tool_name = cb.get("name", "")
|
||||
if tool_name in ("Skill", "Read"):
|
||||
pending_tool_name = tool_name
|
||||
accumulated_json = ""
|
||||
else:
|
||||
return False
|
||||
|
||||
elif se_type == "content_block_delta" and pending_tool_name:
|
||||
delta = se.get("delta", {})
|
||||
if delta.get("type") == "input_json_delta":
|
||||
accumulated_json += delta.get("partial_json", "")
|
||||
if clean_name in accumulated_json:
|
||||
return True
|
||||
|
||||
elif se_type in ("content_block_stop", "message_stop"):
|
||||
if pending_tool_name:
|
||||
return clean_name in accumulated_json
|
||||
if se_type == "message_stop":
|
||||
return False
|
||||
|
||||
# Fallback: full assistant message
|
||||
elif event.get("type") == "assistant":
|
||||
message = event.get("message", {})
|
||||
for content_item in message.get("content", []):
|
||||
if content_item.get("type") != "tool_use":
|
||||
continue
|
||||
tool_name = content_item.get("name", "")
|
||||
tool_input = content_item.get("input", {})
|
||||
if tool_name == "Skill" and clean_name in tool_input.get("skill", ""):
|
||||
triggered = True
|
||||
elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""):
|
||||
triggered = True
|
||||
return triggered
|
||||
|
||||
elif event.get("type") == "result":
|
||||
return triggered
|
||||
finally:
|
||||
# Clean up process on any exit path (return, exception, timeout)
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
|
||||
return triggered
|
||||
finally:
|
||||
if command_file.exists():
|
||||
command_file.unlink()
|
||||
|
||||
|
||||
def run_eval(
|
||||
eval_set: list[dict],
|
||||
skill_name: str,
|
||||
description: str,
|
||||
num_workers: int,
|
||||
timeout: int,
|
||||
project_root: Path,
|
||||
runs_per_query: int = 1,
|
||||
trigger_threshold: float = 0.5,
|
||||
model: str | None = None,
|
||||
) -> dict:
|
||||
"""Run the full eval set and return results."""
|
||||
results = []
|
||||
|
||||
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
||||
future_to_info = {}
|
||||
for item in eval_set:
|
||||
for run_idx in range(runs_per_query):
|
||||
future = executor.submit(
|
||||
run_single_query,
|
||||
item["query"],
|
||||
skill_name,
|
||||
description,
|
||||
timeout,
|
||||
str(project_root),
|
||||
model,
|
||||
)
|
||||
future_to_info[future] = (item, run_idx)
|
||||
|
||||
query_triggers: dict[str, list[bool]] = {}
|
||||
query_items: dict[str, dict] = {}
|
||||
for future in as_completed(future_to_info):
|
||||
item, _ = future_to_info[future]
|
||||
query = item["query"]
|
||||
query_items[query] = item
|
||||
if query not in query_triggers:
|
||||
query_triggers[query] = []
|
||||
try:
|
||||
query_triggers[query].append(future.result())
|
||||
except Exception as e:
|
||||
print(f"Warning: query failed: {e}", file=sys.stderr)
|
||||
query_triggers[query].append(False)
|
||||
|
||||
for query, triggers in query_triggers.items():
|
||||
item = query_items[query]
|
||||
trigger_rate = sum(triggers) / len(triggers)
|
||||
should_trigger = item["should_trigger"]
|
||||
if should_trigger:
|
||||
did_pass = trigger_rate >= trigger_threshold
|
||||
else:
|
||||
did_pass = trigger_rate < trigger_threshold
|
||||
results.append({
|
||||
"query": query,
|
||||
"should_trigger": should_trigger,
|
||||
"trigger_rate": trigger_rate,
|
||||
"triggers": sum(triggers),
|
||||
"runs": len(triggers),
|
||||
"pass": did_pass,
|
||||
})
|
||||
|
||||
passed = sum(1 for r in results if r["pass"])
|
||||
total = len(results)
|
||||
|
||||
return {
|
||||
"skill_name": skill_name,
|
||||
"description": description,
|
||||
"results": results,
|
||||
"summary": {
|
||||
"total": total,
|
||||
"passed": passed,
|
||||
"failed": total - passed,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description")
|
||||
parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
|
||||
parser.add_argument("--skill-path", required=True, help="Path to skill directory")
|
||||
parser.add_argument("--description", default=None, help="Override description to test")
|
||||
parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
|
||||
parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
|
||||
parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
|
||||
parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
|
||||
parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)")
|
||||
parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_set = json.loads(Path(args.eval_set).read_text())
|
||||
skill_path = Path(args.skill_path)
|
||||
|
||||
if not (skill_path / "SKILL.md").exists():
|
||||
print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
name, original_description, content = parse_skill_md(skill_path)
|
||||
description = args.description or original_description
|
||||
project_root = find_project_root()
|
||||
|
||||
if args.verbose:
|
||||
print(f"Evaluating: {description}", file=sys.stderr)
|
||||
|
||||
output = run_eval(
|
||||
eval_set=eval_set,
|
||||
skill_name=name,
|
||||
description=description,
|
||||
num_workers=args.num_workers,
|
||||
timeout=args.timeout,
|
||||
project_root=project_root,
|
||||
runs_per_query=args.runs_per_query,
|
||||
trigger_threshold=args.trigger_threshold,
|
||||
model=args.model,
|
||||
)
|
||||
|
||||
if args.verbose:
|
||||
summary = output["summary"]
|
||||
print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr)
|
||||
for r in output["results"]:
|
||||
status = "PASS" if r["pass"] else "FAIL"
|
||||
rate_str = f"{r['triggers']}/{r['runs']}"
|
||||
print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr)
|
||||
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
328
skills/skill-creator/scripts/run_loop.py
Normal file
328
skills/skill-creator/scripts/run_loop.py
Normal file
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the eval + improve loop until all pass or max iterations reached.
|
||||
|
||||
Combines run_eval.py and improve_description.py in a loop, tracking history
|
||||
and returning the best description found. Supports train/test split to prevent
|
||||
overfitting.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.generate_report import generate_html
|
||||
from scripts.improve_description import improve_description
|
||||
from scripts.run_eval import find_project_root, run_eval
|
||||
from scripts.utils import parse_skill_md
|
||||
|
||||
|
||||
def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]:
|
||||
"""Split eval set into train and test sets, stratified by should_trigger."""
|
||||
random.seed(seed)
|
||||
|
||||
# Separate by should_trigger
|
||||
trigger = [e for e in eval_set if e["should_trigger"]]
|
||||
no_trigger = [e for e in eval_set if not e["should_trigger"]]
|
||||
|
||||
# Shuffle each group
|
||||
random.shuffle(trigger)
|
||||
random.shuffle(no_trigger)
|
||||
|
||||
# Calculate split points
|
||||
n_trigger_test = max(1, int(len(trigger) * holdout))
|
||||
n_no_trigger_test = max(1, int(len(no_trigger) * holdout))
|
||||
|
||||
# Split
|
||||
test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test]
|
||||
train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:]
|
||||
|
||||
return train_set, test_set
|
||||
|
||||
|
||||
def run_loop(
|
||||
eval_set: list[dict],
|
||||
skill_path: Path,
|
||||
description_override: str | None,
|
||||
num_workers: int,
|
||||
timeout: int,
|
||||
max_iterations: int,
|
||||
runs_per_query: int,
|
||||
trigger_threshold: float,
|
||||
holdout: float,
|
||||
model: str,
|
||||
verbose: bool,
|
||||
live_report_path: Path | None = None,
|
||||
log_dir: Path | None = None,
|
||||
) -> dict:
|
||||
"""Run the eval + improvement loop."""
|
||||
project_root = find_project_root()
|
||||
name, original_description, content = parse_skill_md(skill_path)
|
||||
current_description = description_override or original_description
|
||||
|
||||
# Split into train/test if holdout > 0
|
||||
if holdout > 0:
|
||||
train_set, test_set = split_eval_set(eval_set, holdout)
|
||||
if verbose:
|
||||
print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr)
|
||||
else:
|
||||
train_set = eval_set
|
||||
test_set = []
|
||||
|
||||
history = []
|
||||
exit_reason = "unknown"
|
||||
|
||||
for iteration in range(1, max_iterations + 1):
|
||||
if verbose:
|
||||
print(f"\n{'='*60}", file=sys.stderr)
|
||||
print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr)
|
||||
print(f"Description: {current_description}", file=sys.stderr)
|
||||
print(f"{'='*60}", file=sys.stderr)
|
||||
|
||||
# Evaluate train + test together in one batch for parallelism
|
||||
all_queries = train_set + test_set
|
||||
t0 = time.time()
|
||||
all_results = run_eval(
|
||||
eval_set=all_queries,
|
||||
skill_name=name,
|
||||
description=current_description,
|
||||
num_workers=num_workers,
|
||||
timeout=timeout,
|
||||
project_root=project_root,
|
||||
runs_per_query=runs_per_query,
|
||||
trigger_threshold=trigger_threshold,
|
||||
model=model,
|
||||
)
|
||||
eval_elapsed = time.time() - t0
|
||||
|
||||
# Split results back into train/test by matching queries
|
||||
train_queries_set = {q["query"] for q in train_set}
|
||||
train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set]
|
||||
test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set]
|
||||
|
||||
train_passed = sum(1 for r in train_result_list if r["pass"])
|
||||
train_total = len(train_result_list)
|
||||
train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total}
|
||||
train_results = {"results": train_result_list, "summary": train_summary}
|
||||
|
||||
if test_set:
|
||||
test_passed = sum(1 for r in test_result_list if r["pass"])
|
||||
test_total = len(test_result_list)
|
||||
test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total}
|
||||
test_results = {"results": test_result_list, "summary": test_summary}
|
||||
else:
|
||||
test_results = None
|
||||
test_summary = None
|
||||
|
||||
history.append({
|
||||
"iteration": iteration,
|
||||
"description": current_description,
|
||||
"train_passed": train_summary["passed"],
|
||||
"train_failed": train_summary["failed"],
|
||||
"train_total": train_summary["total"],
|
||||
"train_results": train_results["results"],
|
||||
"test_passed": test_summary["passed"] if test_summary else None,
|
||||
"test_failed": test_summary["failed"] if test_summary else None,
|
||||
"test_total": test_summary["total"] if test_summary else None,
|
||||
"test_results": test_results["results"] if test_results else None,
|
||||
# For backward compat with report generator
|
||||
"passed": train_summary["passed"],
|
||||
"failed": train_summary["failed"],
|
||||
"total": train_summary["total"],
|
||||
"results": train_results["results"],
|
||||
})
|
||||
|
||||
# Write live report if path provided
|
||||
if live_report_path:
|
||||
partial_output = {
|
||||
"original_description": original_description,
|
||||
"best_description": current_description,
|
||||
"best_score": "in progress",
|
||||
"iterations_run": len(history),
|
||||
"holdout": holdout,
|
||||
"train_size": len(train_set),
|
||||
"test_size": len(test_set),
|
||||
"history": history,
|
||||
}
|
||||
live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name))
|
||||
|
||||
if verbose:
|
||||
def print_eval_stats(label, results, elapsed):
|
||||
pos = [r for r in results if r["should_trigger"]]
|
||||
neg = [r for r in results if not r["should_trigger"]]
|
||||
tp = sum(r["triggers"] for r in pos)
|
||||
pos_runs = sum(r["runs"] for r in pos)
|
||||
fn = pos_runs - tp
|
||||
fp = sum(r["triggers"] for r in neg)
|
||||
neg_runs = sum(r["runs"] for r in neg)
|
||||
tn = neg_runs - fp
|
||||
total = tp + tn + fp + fn
|
||||
precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0
|
||||
recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0
|
||||
accuracy = (tp + tn) / total if total > 0 else 0.0
|
||||
print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr)
|
||||
for r in results:
|
||||
status = "PASS" if r["pass"] else "FAIL"
|
||||
rate_str = f"{r['triggers']}/{r['runs']}"
|
||||
print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr)
|
||||
|
||||
print_eval_stats("Train", train_results["results"], eval_elapsed)
|
||||
if test_summary:
|
||||
print_eval_stats("Test ", test_results["results"], 0)
|
||||
|
||||
if train_summary["failed"] == 0:
|
||||
exit_reason = f"all_passed (iteration {iteration})"
|
||||
if verbose:
|
||||
print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr)
|
||||
break
|
||||
|
||||
if iteration == max_iterations:
|
||||
exit_reason = f"max_iterations ({max_iterations})"
|
||||
if verbose:
|
||||
print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr)
|
||||
break
|
||||
|
||||
# Improve the description based on train results
|
||||
if verbose:
|
||||
print(f"\nImproving description...", file=sys.stderr)
|
||||
|
||||
t0 = time.time()
|
||||
# Strip test scores from history so improvement model can't see them
|
||||
blinded_history = [
|
||||
{k: v for k, v in h.items() if not k.startswith("test_")}
|
||||
for h in history
|
||||
]
|
||||
new_description = improve_description(
|
||||
skill_name=name,
|
||||
skill_content=content,
|
||||
current_description=current_description,
|
||||
eval_results=train_results,
|
||||
history=blinded_history,
|
||||
model=model,
|
||||
log_dir=log_dir,
|
||||
iteration=iteration,
|
||||
)
|
||||
improve_elapsed = time.time() - t0
|
||||
|
||||
if verbose:
|
||||
print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr)
|
||||
|
||||
current_description = new_description
|
||||
|
||||
# Find the best iteration by TEST score (or train if no test set)
|
||||
if test_set:
|
||||
best = max(history, key=lambda h: h["test_passed"] or 0)
|
||||
best_score = f"{best['test_passed']}/{best['test_total']}"
|
||||
else:
|
||||
best = max(history, key=lambda h: h["train_passed"])
|
||||
best_score = f"{best['train_passed']}/{best['train_total']}"
|
||||
|
||||
if verbose:
|
||||
print(f"\nExit reason: {exit_reason}", file=sys.stderr)
|
||||
print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr)
|
||||
|
||||
return {
|
||||
"exit_reason": exit_reason,
|
||||
"original_description": original_description,
|
||||
"best_description": best["description"],
|
||||
"best_score": best_score,
|
||||
"best_train_score": f"{best['train_passed']}/{best['train_total']}",
|
||||
"best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None,
|
||||
"final_description": current_description,
|
||||
"iterations_run": len(history),
|
||||
"holdout": holdout,
|
||||
"train_size": len(train_set),
|
||||
"test_size": len(test_set),
|
||||
"history": history,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run eval + improve loop")
|
||||
parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
|
||||
parser.add_argument("--skill-path", required=True, help="Path to skill directory")
|
||||
parser.add_argument("--description", default=None, help="Override starting description")
|
||||
parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
|
||||
parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
|
||||
parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations")
|
||||
parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
|
||||
parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
|
||||
parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)")
|
||||
parser.add_argument("--model", required=True, help="Model for improvement")
|
||||
parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
|
||||
parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)")
|
||||
parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here")
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_set = json.loads(Path(args.eval_set).read_text())
|
||||
skill_path = Path(args.skill_path)
|
||||
|
||||
if not (skill_path / "SKILL.md").exists():
|
||||
print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
name, _, _ = parse_skill_md(skill_path)
|
||||
|
||||
# Set up live report path
|
||||
if args.report != "none":
|
||||
if args.report == "auto":
|
||||
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
||||
live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html"
|
||||
else:
|
||||
live_report_path = Path(args.report)
|
||||
# Open the report immediately so the user can watch
|
||||
live_report_path.write_text("<html><body><h1>Starting optimization loop...</h1><meta http-equiv='refresh' content='5'></body></html>")
|
||||
webbrowser.open(str(live_report_path))
|
||||
else:
|
||||
live_report_path = None
|
||||
|
||||
# Determine output directory (create before run_loop so logs can be written)
|
||||
if args.results_dir:
|
||||
timestamp = time.strftime("%Y-%m-%d_%H%M%S")
|
||||
results_dir = Path(args.results_dir) / timestamp
|
||||
results_dir.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
results_dir = None
|
||||
|
||||
log_dir = results_dir / "logs" if results_dir else None
|
||||
|
||||
output = run_loop(
|
||||
eval_set=eval_set,
|
||||
skill_path=skill_path,
|
||||
description_override=args.description,
|
||||
num_workers=args.num_workers,
|
||||
timeout=args.timeout,
|
||||
max_iterations=args.max_iterations,
|
||||
runs_per_query=args.runs_per_query,
|
||||
trigger_threshold=args.trigger_threshold,
|
||||
holdout=args.holdout,
|
||||
model=args.model,
|
||||
verbose=args.verbose,
|
||||
live_report_path=live_report_path,
|
||||
log_dir=log_dir,
|
||||
)
|
||||
|
||||
# Save JSON output
|
||||
json_output = json.dumps(output, indent=2)
|
||||
print(json_output)
|
||||
if results_dir:
|
||||
(results_dir / "results.json").write_text(json_output)
|
||||
|
||||
# Write final HTML report (without auto-refresh)
|
||||
if live_report_path:
|
||||
live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name))
|
||||
print(f"\nReport: {live_report_path}", file=sys.stderr)
|
||||
|
||||
if results_dir and live_report_path:
|
||||
(results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name))
|
||||
|
||||
if results_dir:
|
||||
print(f"Results saved to: {results_dir}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
47
skills/skill-creator/scripts/utils.py
Normal file
47
skills/skill-creator/scripts/utils.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""Shared utilities for skill-creator scripts."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
|
||||
def parse_skill_md(skill_path: Path) -> tuple[str, str, str]:
|
||||
"""Parse a SKILL.md file, returning (name, description, full_content)."""
|
||||
content = (skill_path / "SKILL.md").read_text()
|
||||
lines = content.split("\n")
|
||||
|
||||
if lines[0].strip() != "---":
|
||||
raise ValueError("SKILL.md missing frontmatter (no opening ---)")
|
||||
|
||||
end_idx = None
|
||||
for i, line in enumerate(lines[1:], start=1):
|
||||
if line.strip() == "---":
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if end_idx is None:
|
||||
raise ValueError("SKILL.md missing frontmatter (no closing ---)")
|
||||
|
||||
name = ""
|
||||
description = ""
|
||||
frontmatter_lines = lines[1:end_idx]
|
||||
i = 0
|
||||
while i < len(frontmatter_lines):
|
||||
line = frontmatter_lines[i]
|
||||
if line.startswith("name:"):
|
||||
name = line[len("name:"):].strip().strip('"').strip("'")
|
||||
elif line.startswith("description:"):
|
||||
value = line[len("description:"):].strip()
|
||||
# Handle YAML multiline indicators (>, |, >-, |-)
|
||||
if value in (">", "|", ">-", "|-"):
|
||||
continuation_lines: list[str] = []
|
||||
i += 1
|
||||
while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith(" ") or frontmatter_lines[i].startswith("\t")):
|
||||
continuation_lines.append(frontmatter_lines[i].strip())
|
||||
i += 1
|
||||
description = " ".join(continuation_lines)
|
||||
continue
|
||||
else:
|
||||
description = value.strip('"').strip("'")
|
||||
i += 1
|
||||
|
||||
return name, description, content
|
||||
171
skills/vba-best-practices/SKILL.md
Normal file
171
skills/vba-best-practices/SKILL.md
Normal file
@@ -0,0 +1,171 @@
|
||||
---
|
||||
name: vba-best-practices
|
||||
description: VBA (Visual Basic for Applications) coding standards for Excel, Word, and Office automation. Trigger this skill when the user asks for help writing or reviewing VBA macros, wants to improve code quality, or shares VBA code asking "is this good?" or "how can I improve this?"
|
||||
---
|
||||
|
||||
# VBA Best Practices
|
||||
|
||||
## Code Structure
|
||||
|
||||
**Always start every module with `Option Explicit`** — forces explicit variable declaration, catches typos at compile time. Enable globally via Tools > Options > Editor > "Require Variable Declaration".
|
||||
|
||||
**Declare variables at the top of each procedure**, one per line, with descriptive names. Avoid single letters (`i`, `j`) and reserved words.
|
||||
|
||||
**Naming conventions:**
|
||||
- Subs/Functions → `PascalCase`
|
||||
- Local variables → Hungarian prefix (`iCount`, `sName`, `bFlag`)
|
||||
- Module-level → `PascalCase`; globals prefix with `g_`
|
||||
- Constants → `ALL_CAPITALS`; helper functions prefix with `fn`/`fx`
|
||||
|
||||
**Indent with tabs.** Everything inside loops, conditionals, and `With` blocks gets one level deeper.
|
||||
|
||||
---
|
||||
|
||||
## Variables & Data Types
|
||||
|
||||
- **Avoid `Variant`** — it's 16 bytes and slow. Use it only when the type is genuinely unknown or when passing arrays of mixed types.
|
||||
- **Never use `Option Base 1`** — keep the default array base of 0.
|
||||
- **Never use `Option Compare Text`** — keep string comparisons case-sensitive by default.
|
||||
- **Use `vbNullString` instead of `""`** for empty string assignment; use `Len(s) = 0` to check emptiness — both are faster than comparing against `""`.
|
||||
- **Use `vb` string constants** instead of `Chr()` calls: `vbTab`, `vbLf`, `vbCr`, `vbNewLine` (fastest), `vbNullChar`.
|
||||
|
||||
### No Duplicate `Dim` Declarations Within a Procedure
|
||||
|
||||
VBA uses **procedure-level scope** — a variable declared anywhere inside a Sub/Function is scoped to the entire procedure. Declaring the same variable name more than once (even in separate `If/Else` branches or nested loops) causes a **compile error**.
|
||||
|
||||
**Rule: declare every variable exactly once, at the top of the procedure.**
|
||||
|
||||
```vb
|
||||
' ❌ Compile error — same name declared twice in different branches
|
||||
Sub BadExample()
|
||||
If condition Then
|
||||
Dim x As Integer ' first declaration
|
||||
x = 1
|
||||
Else
|
||||
Dim x As Integer ' DUPLICATE — compiler rejects this
|
||||
x = 2
|
||||
End If
|
||||
End Sub
|
||||
|
||||
' ✅ Correct — declared once at the top, assigned freely in any branch
|
||||
Sub GoodExample()
|
||||
Dim x As Integer ' single declaration
|
||||
|
||||
If condition Then
|
||||
x = 1
|
||||
Else
|
||||
x = 2
|
||||
End If
|
||||
End Sub
|
||||
```
|
||||
|
||||
The same rule applies to loop variables reused across multiple `For Each` loops:
|
||||
|
||||
```vb
|
||||
' ❌ Compile error — rawMat declared inside two separate loops
|
||||
For Each rawMat In rawMaterials ' implicit Dim inside loop
|
||||
allRawMaterials.Add rawMat
|
||||
Next rawMat
|
||||
|
||||
For Each rawMat In allRawMaterials ' duplicate Dim — compile error
|
||||
' ...
|
||||
Next rawMat
|
||||
|
||||
' ✅ Correct — declare once at the top, reuse freely
|
||||
Dim rawMat As Variant
|
||||
|
||||
For Each rawMat In rawMaterials
|
||||
allRawMaterials.Add rawMat
|
||||
Next rawMat
|
||||
|
||||
For Each rawMat In allRawMaterials
|
||||
' ...
|
||||
Next rawMat
|
||||
```
|
||||
|
||||
**Tip:** A quick way to catch duplicates is to search for `Dim <varname>` within the same procedure. With `Option Explicit` enabled, the compiler will also surface this error on the first compile attempt.
|
||||
|
||||
---
|
||||
|
||||
## Parameters & Scope
|
||||
|
||||
- **Always declare `ByVal` or `ByRef` explicitly** — VBA defaults to `ByRef`, which can cause silent side effects. Use `ByVal` unless you intentionally need to modify and return the argument.
|
||||
- **Minimize global variables.** Use `Public`/`Private` at module level (not `Global`/bare `Dim`). If a global is unavoidable, prefix with `g_`.
|
||||
- **Validate arguments** at the top of every Sub/Function before using them.
|
||||
|
||||
---
|
||||
|
||||
## Error Handling
|
||||
|
||||
**Never use `On Error Resume Next`** — it silently swallows bugs. Instead, add a custom error handler to every procedure:
|
||||
|
||||
```vb
|
||||
Sub DoSomething()
|
||||
On Error GoTo ErrorHandler
|
||||
' ... code ...
|
||||
Exit Sub
|
||||
ErrorHandler:
|
||||
Debug.Print "Error in DoSomething: " & Err.Description
|
||||
' Log, re-raise, or recover as appropriate
|
||||
End Sub
|
||||
```
|
||||
|
||||
Handle the error in the same procedure where it occurs.
|
||||
|
||||
---
|
||||
|
||||
## Control Flow & Objects
|
||||
|
||||
- **No `GoTo`** for program logic — use `If/ElseIf`, `Select Case`, and loops. (`GoTo ErrorHandler` is the one accepted exception.)
|
||||
- **Use `With … End With`** whenever you reference the same object more than once — fewer dot-lookups means faster execution:
|
||||
|
||||
```vb
|
||||
With ws.Range("A1").Font
|
||||
.Bold = True
|
||||
.Size = 12
|
||||
End With
|
||||
```
|
||||
|
||||
- **Never reference `ActiveCell`, `ActiveSheet`, or `ActiveWorkbook`** — always bind to explicit object variables:
|
||||
|
||||
```vb
|
||||
Dim ws As Worksheet
|
||||
Set ws = ThisWorkbook.Sheets("Data")
|
||||
ws.Range("A1").Value = "Hello"
|
||||
```
|
||||
|
||||
- **Use built-in constants and enumerations** with full prefix (`VBA.vbMsgBoxResult.vbYes`, `msoFileType.msoFileTypeExcelWorkbooks`) for readability and IntelliSense support. Group related custom constants into `Enum` blocks.
|
||||
|
||||
---
|
||||
|
||||
## Performance
|
||||
|
||||
- **Bulk data via arrays** — read a range into an array, process in memory, write back in one shot. Never loop over individual cells for large datasets:
|
||||
|
||||
```vb
|
||||
Dim arData As Variant
|
||||
arData = Sheet1.Range("A1:D500").Value ' one read
|
||||
' ... process arData() in memory ...
|
||||
Sheet1.Range("A1:D500").Value = arData ' one write
|
||||
```
|
||||
|
||||
- **`ReDim Preserve` inside a loop is forbidden.** Oversize the array upfront; `ReDim` (without `Preserve`) once at the end if needed.
|
||||
- **Minimize `Application.ScreenUpdating = False`** — use only when necessary, and always restore it (including inside error handlers).
|
||||
|
||||
---
|
||||
|
||||
## Quick Checklist
|
||||
|
||||
| Rule | Why it matters |
|
||||
|---|---|
|
||||
| `Option Explicit` in every module | Prevents undeclared-variable bugs |
|
||||
| Descriptive names + Hungarian prefixes | Readability & self-documentation |
|
||||
| Each variable declared **once**, at procedure top | Duplicate `Dim` = compile error |
|
||||
| `ByVal`/`ByRef` always explicit | No silent side effects |
|
||||
| Custom `GoTo ErrorHandler`, not `Resume Next` | Errors surface instead of hiding |
|
||||
| `With…End With` for repeated object refs | Speed + clarity |
|
||||
| No `ActiveCell`/`ActiveSheet`/`ActiveWorkbook` | Robust against user selections |
|
||||
| `vbNullString` / `Len()` for strings | Faster than `""` comparison |
|
||||
| Arrays for bulk range I/O | Orders-of-magnitude speed gain |
|
||||
| No `Option Base 1` or `Option Compare Text` | Predictable defaults |
|
||||
| No `GoTo` except for error handler label | Structured, readable flow |
|
||||
Reference in New Issue
Block a user