From b70a7ba54f6c9e0e537fe3aefaa32876b5b4a183 Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Wed, 15 Apr 2026 23:25:37 +0200 Subject: [PATCH 1/9] feat(skill-analysis): add --analyze-skill mode for token efficiency scanning Adds static token analysis for AI agent skill packages, enabling Tank's security pipeline to also report token usage findings. New: - src/skill-analyzer.ts: engine that discovers skill files and runs rules - src/skill-rules/prompt-size.ts: flags oversized SKILL.md/atom files - src/skill-rules/claude-md-size.ts: flags large CLAUDE.md injections - src/skill-rules/tool-overhead.ts: counts tool definitions in manifest - src/skill-rules/large-files.ts: flags files >500 lines - src/skill-rules/redundant-instructions.ts: detects duplicated instructions - tests/skill-analyzer.test.ts: 26 tests covering all rules - 9 files changed, 26 tests added (208 total passing) - Bump version to 2.3.0 Co-Authored-By: Claude --- package.json | 2 +- src/analyze.ts | 16 +- src/skill-analyzer.ts | 161 ++++++++++ src/skill-rules/claude-md-size.ts | 91 ++++++ src/skill-rules/large-files.ts | 32 ++ src/skill-rules/prompt-size.ts | 57 ++++ src/skill-rules/redundant-instructions.ts | 96 ++++++ src/skill-rules/tool-overhead.ts | 101 ++++++ src/types.ts | 39 +++ tests/skill-analyzer.test.ts | 362 ++++++++++++++++++++++ 10 files changed, 955 insertions(+), 2 deletions(-) create mode 100644 src/skill-analyzer.ts create mode 100644 src/skill-rules/claude-md-size.ts create mode 100644 src/skill-rules/large-files.ts create mode 100644 src/skill-rules/prompt-size.ts create mode 100644 src/skill-rules/redundant-instructions.ts create mode 100644 src/skill-rules/tool-overhead.ts create mode 100644 tests/skill-analyzer.test.ts diff --git a/package.json b/package.json index 7b66cd2..96ad5ee 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tokenomics", - "version": "2.2.2", + "version": "2.3.0", "description": "Token intelligence for Claude Code — analyze sessions, find waste, get actionable fixes", "type": "module", "main": "dist/analyze.js", diff --git a/src/analyze.ts b/src/analyze.ts index f9980d2..bbf2954 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -20,6 +20,7 @@ import { writeFileSync, mkdirSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; import type { AnalysisOutput, CliOptions, DetectorResult, SessionData } from './types.js'; +import { analyzeSkill } from './skill-analyzer.js'; import { discoverFiles, logDiscoverySummary } from './discovery.js'; import { extractHumanReadableBlock, renderTerminalBlock } from './recommendation.js'; import { parseSessionFiles } from './parser.js'; @@ -34,7 +35,7 @@ import { auditPrompt } from './auditor.js'; import { renderPromptOutput } from './prompt-output.js'; import { ensureBudgetConfig, readBudgetConfig } from './budget-config.js'; -const VERSION = '2.2.2'; +const VERSION = '2.3.0'; // ─── CLI ────────────────────────────────────────────────────────────────────── @@ -60,6 +61,7 @@ function parseCliArgs(): CliOptions { budget: { type: 'boolean', default: false }, 'budget-check': { type: 'boolean', default: false }, 'no-alerts': { type: 'boolean', default: false }, + 'analyze-skill': { type: 'string' }, }, strict: true, }); @@ -88,6 +90,7 @@ function parseCliArgs(): CliOptions { budget: values.budget, budgetCheck: values['budget-check'], noAlerts: values['no-alerts'], + analyzeSkill: values['analyze-skill'], }; } @@ -131,6 +134,10 @@ PROMPT ANALYSIS --budget-check Lightweight budget check (for hooks) --no-alerts Suppress budget alerts (no CLAUDE.md injection) +SKILL ANALYSIS + --analyze-skill Analyze skill package for token efficiency + Outputs JSON with findings and efficiency score + OTHER --verbose Show discovery progress and debug info --help Show this message @@ -516,6 +523,13 @@ async function main(): Promise { process.exit(result.ceilingExceeded ? 1 : 0); } + // ── Skill analysis mode ── + if (options.analyzeSkill) { + const result = analyzeSkill(options.analyzeSkill); + console.log(JSON.stringify(result, null, options.verbose ? 2 : 0)); + return; + } + // Discover JSONL files (auto-detects all ~/.claude* installations) const discoveryOpts: import('./types.js').DiscoveryOptions = { days: options.days, diff --git a/src/skill-analyzer.ts b/src/skill-analyzer.ts new file mode 100644 index 0000000..ac44445 --- /dev/null +++ b/src/skill-analyzer.ts @@ -0,0 +1,161 @@ +/** + * Skill Analyzer Engine + * + * Analyzes an AI agent skill package directory for token efficiency. + * Discovers skill files, runs all skill rules, and returns findings. + */ + +import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs' +import { join, extname, relative } from 'node:path' +import type { SkillAnalysisContext, SkillAnalysisResult, SkillAnalysisSummary, SkillFinding, SkillRule } from './types.js' + +// Import all rules +import * as promptSizeRule from './skill-rules/prompt-size.js' +import * as claudeMdSizeRule from './skill-rules/claude-md-size.js' +import * as toolOverheadRule from './skill-rules/tool-overhead.js' +import * as largeFilesRule from './skill-rules/large-files.js' +import * as redundantInstructionsRule from './skill-rules/redundant-instructions.js' + +const CHARS_PER_TOKEN = 4 + +const SKILL_FILES = [ + 'SKILL.md', + 'CLAUDE.md', + 'tank.json', + 'skills.json', +] + +const SKILL_EXTENSIONS = ['.md', '.json'] + +const rules: SkillRule[] = [ + { name: 'prompt-size', analyze: promptSizeRule.analyze }, + { name: 'claude-md-size', analyze: claudeMdSizeRule.analyze }, + { name: 'tool-overhead', analyze: toolOverheadRule.analyze }, + { name: 'large-files', analyze: largeFilesRule.analyze }, + { name: 'redundant-instructions', analyze: redundantInstructionsRule.analyze }, +] + +function discoverSkillFiles(dir: string): Map { + const files = new Map() + + function walk(currentDir: string): void { + const entries = readdirSync(currentDir, { withFileTypes: true }) + + for (const entry of entries) { + // Skip symlinks to prevent path traversal + if (entry.isSymbolicLink()) continue + + if (entry.isDirectory()) { + if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === 'dist') continue + walk(join(currentDir, entry.name)) + continue + } + + const fullPath = join(currentDir, entry.name) + const relativePath = relative(dir, fullPath) + + // Include known skill files or files with skill extensions + const isKnownFile = SKILL_FILES.some(f => entry.name.toLowerCase() === f.toLowerCase()) + const isAtomFile = entry.name.toLowerCase().endsWith('.atom.md') + const hasSkillExt = SKILL_EXTENSIONS.includes(extname(entry.name).toLowerCase()) + + if (isKnownFile || isAtomFile || hasSkillExt) { + try { + const content = readFileSync(fullPath, 'utf-8') + files.set(relativePath, content) + } catch { + // Skip unreadable files + } + } + } + } + + walk(dir) + return files +} + +function loadManifest(files: Map): Record | null { + // Try tank.json first, then skills.json + for (const manifestName of ['tank.json', 'skills.json']) { + for (const [filename, content] of files) { + if (filename.toLowerCase().endsWith(manifestName.toLowerCase())) { + try { + return JSON.parse(content) as Record + } catch { + // Invalid JSON, skip + } + } + } + } + return null +} + +function estimateTokens(files: Map): number { + let totalChars = 0 + for (const content of files.values()) { + totalChars += content.length + } + return Math.ceil(totalChars / CHARS_PER_TOKEN) +} + +function calculateEfficiencyScore(findings: SkillFinding[], totalTokens: number): number { + // Start at 100, deduct points based on findings + let score = 100 + + for (const finding of findings) { + const deduction = finding.severity === 'high' ? 15 + : finding.severity === 'medium' ? 8 + : finding.severity === 'low' ? 3 + : 1 // info + score -= deduction + } + + // Bonus for low total token count + if (totalTokens < 1000) score = Math.min(score + 10, 100) + + return Math.max(0, Math.min(100, score)) +} + +export function analyzeSkill(dir: string): SkillAnalysisResult { + // Validate directory + if (!existsSync(dir)) { + throw new Error(`Directory not found: ${dir}`) + } + + if (!statSync(dir).isDirectory()) { + throw new Error(`Not a directory: ${dir}`) + } + + // Discover and read files + const files = discoverSkillFiles(dir) + const manifest = loadManifest(files) + + // Build analysis context + const context: SkillAnalysisContext = { + skillDir: dir, + files, + skillManifest: manifest, + } + + // Run all rules + const allFindings: SkillFinding[] = [] + for (const rule of rules) { + const ruleFindings = rule.analyze(context) + allFindings.push(...ruleFindings) + } + + // Calculate summary + const estimatedTokens = estimateTokens(files) + const efficiencyScore = calculateEfficiencyScore(allFindings, estimatedTokens) + + const summary: SkillAnalysisSummary = { + total_findings: allFindings.length, + estimated_tokens_per_invocation: estimatedTokens, + efficiency_score: efficiencyScore, + } + + return { + findings: allFindings, + summary, + } +} diff --git a/src/skill-rules/claude-md-size.ts b/src/skill-rules/claude-md-size.ts new file mode 100644 index 0000000..7646e6d --- /dev/null +++ b/src/skill-rules/claude-md-size.ts @@ -0,0 +1,91 @@ +/** + * CLAUDE.md Size Rule + * + * Detects CLAUDE.md injection blocks and flags oversized ones. + * Looks for markers or large markdown blocks + * in CLAUDE.md files. + * + * Thresholds: + * >1500 tokens (6000 chars) → medium + * >3000 tokens (12000 chars) → high + */ + +import type { SkillAnalysisContext, SkillFinding } from '../types.js' + +const CHARS_PER_TOKEN = 4 +const MEDIUM_TOKEN_THRESHOLD = 1500 +const HIGH_TOKEN_THRESHOLD = 3000 + +export function analyze(context: SkillAnalysisContext): SkillFinding[] { + const findings: SkillFinding[] = [] + + for (const [filename, content] of context.files) { + if (!filename.toLowerCase().includes('claude.md')) continue + + // Check for TOKENOMICS injection blocks + const tokenomicsBlocks = extractTokenomicsBlocks(content) + if (tokenomicsBlocks.length > 0) { + for (const block of tokenomicsBlocks) { + const tokenEstimate = Math.ceil(block.length / CHARS_PER_TOKEN) + flagBlock(findings, filename, tokenEstimate, 'tokenomics injection block') + } + continue + } + + // No injection markers — evaluate the whole file if it looks like an injection target + const tokenEstimate = Math.ceil(content.length / CHARS_PER_TOKEN) + flagBlock(findings, filename, tokenEstimate, 'CLAUDE.md content') + } + + return findings +} + +function extractTokenomicsBlocks(content: string): string[] { + const blocks: string[] = [] + const startMarker = '\n' + 'A'.repeat(12001) + '\n' + const ctx = makeContext({ 'CLAUDE.md': block }) + const findings = claudeMdSizeRule.analyze(ctx) + + expect(findings.length).toBe(1) + expect(findings[0]!.description).toContain('tokenomics injection block') + }) +}) + +// ─── Tool Overhead Rule ───────────────────────────────────────────────────── + +describe('tool-overhead rule', () => { + it('flags >15 tools as high severity', () => { + const tools: Record = {} + for (let i = 0; i < 16; i++) { + tools[`tool-${i}`] = { command: 'echo' } + } + const manifest = { mcpServers: tools } + const ctx = makeContext({}, manifest) + + const findings = toolOverheadRule.analyze(ctx) + expect(findings.length).toBe(1) + expect(findings[0]!.severity).toBe('high') + expect(findings[0]!.rule).toBe('tool-overhead') + }) + + it('flags >8 tools as medium severity', () => { + const tools: Record = {} + for (let i = 0; i < 10; i++) { + tools[`tool-${i}`] = { command: 'echo' } + } + const manifest = { mcpServers: tools } + const ctx = makeContext({}, manifest) + + const findings = toolOverheadRule.analyze(ctx) + expect(findings.length).toBe(1) + expect(findings[0]!.severity).toBe('medium') + }) + + it('does not flag <=8 tools', () => { + const tools: Record = {} + for (let i = 0; i < 5; i++) { + tools[`tool-${i}`] = { command: 'echo' } + } + const manifest = { mcpServers: tools } + const ctx = makeContext({}, manifest) + + const findings = toolOverheadRule.analyze(ctx) + expect(findings.length).toBe(0) + }) + + it('scans config files for tool definitions', () => { + const tankJson = JSON.stringify({ + mcpServers: Object.fromEntries( + Array.from({ length: 12 }, (_, i) => [`tool-${i}`, { command: 'echo' }]) + ), + }) + const ctx = makeContext({ 'tank.json': tankJson }, null) + + const findings = toolOverheadRule.analyze(ctx) + expect(findings.length).toBe(1) + expect(findings[0]!.severity).toBe('medium') + }) +}) + +// ─── Large Files Rule ──────────────────────────────────────────────────────── + +describe('large-files rule', () => { + it('flags files >500 lines', () => { + const lines = Array(501).fill('line of code').join('\n') + const ctx = makeContext({ 'src/large.ts': lines }) + const findings = largeFilesRule.analyze(ctx) + + expect(findings.length).toBe(1) + expect(findings[0]!.rule).toBe('large-files') + expect(findings[0]!.location).toBe('src/large.ts') + }) + + it('flags files >1000 lines as medium severity', () => { + const lines = Array(1001).fill('line of code').join('\n') + const ctx = makeContext({ 'src/very-large.ts': lines }) + const findings = largeFilesRule.analyze(ctx) + + expect(findings.length).toBe(1) + expect(findings[0]!.severity).toBe('medium') + }) + + it('does not flag files <=500 lines', () => { + const lines = Array(200).fill('line of code').join('\n') + const ctx = makeContext({ 'src/small.ts': lines }) + const findings = largeFilesRule.analyze(ctx) + + expect(findings.length).toBe(0) + }) +}) + +// ─── Redundant Instructions Rule ──────────────────────────────────────────── + +describe('redundant-instructions rule', () => { + it('flags when >30% duplication across skill files', () => { + const sharedInstructions = Array(10).fill('Always use TypeScript strict mode for all new files in this project').join('\n') + const uniqueA = 'Specific to SKILL.md that is unique content' + const uniqueB = 'Specific to CLAUDE.md that is different content' + + const ctx = makeContext({ + 'SKILL.md': sharedInstructions + '\n' + uniqueA, + 'CLAUDE.md': sharedInstructions + '\n' + uniqueB, + }) + const findings = redundantInstructionsRule.analyze(ctx) + + expect(findings.length).toBeGreaterThanOrEqual(1) + expect(findings[0]!.rule).toBe('redundant-instructions') + }) + + it('does not flag when duplication is low', () => { + const ctx = makeContext({ + 'SKILL.md': 'Unique content about the first skill that is quite different from others', + 'CLAUDE.md': 'Different content about project guidelines that does not overlap at all', + }) + const findings = redundantInstructionsRule.analyze(ctx) + + expect(findings.length).toBe(0) + }) + + it('returns nothing with fewer than 2 skill files', () => { + const ctx = makeContext({ + 'SKILL.md': 'A'.repeat(500), + }) + const findings = redundantInstructionsRule.analyze(ctx) + + expect(findings.length).toBe(0) + }) +}) + +// ─── Integration: Full Skill Analyzer ──────────────────────────────────────── + +describe('skill analyzer integration', () => { + let tempDir: string + + beforeEach(() => { + tempDir = createTempDir() + }) + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }) + }) + + it('returns valid SkillAnalysisResult for a clean skill', () => { + createFixture(tempDir, { + 'SKILL.md': 'A simple skill that does one thing well.', + 'tank.json': JSON.stringify({ name: 'test-skill', version: '1.0.0' }), + }) + + const result = analyzeSkill(tempDir) + + expect(result).toHaveProperty('findings') + expect(result).toHaveProperty('summary') + expect(Array.isArray(result.findings)).toBe(true) + expect(result.summary.total_findings).toBe(result.findings.length) + expect(result.summary.efficiency_score).toBeGreaterThanOrEqual(0) + expect(result.summary.efficiency_score).toBeLessThanOrEqual(100) + expect(result.summary.estimated_tokens_per_invocation).toBeGreaterThan(0) + }) + + it('detects all issue types in a bloated skill', () => { + const largeSkill = 'Large prompt content. '.repeat(800) // ~19k chars + const largeClaude = 'Inject all the things. '.repeat(800) // ~19k chars + const duplicated = Array(15).fill('Always use TypeScript strict mode for all files in this project').join('\n') + + const tools: Record = {} + for (let i = 0; i < 16; i++) { + tools[`tool-${i}`] = { command: `echo ${i}` } + } + + createFixture(tempDir, { + 'SKILL.md': largeSkill + '\n' + duplicated, + 'CLAUDE.md': largeClaude + '\n' + duplicated, + 'search.atom.md': largeSkill, + 'tank.json': JSON.stringify({ name: 'bloated-skill', mcpServers: tools }), + }) + + const result = analyzeSkill(tempDir) + + expect(result.findings.length).toBeGreaterThanOrEqual(3) + + const rules = result.findings.map(f => f.rule) + expect(rules).toContain('prompt-size') + expect(rules).toContain('claude-md-size') + expect(rules).toContain('tool-overhead') + expect(rules).toContain('redundant-instructions') + }) + + it('throws for non-existent directory', () => { + expect(() => analyzeSkill('/nonexistent/path')).toThrow('Directory not found') + }) + + it('throws for a file path instead of directory', () => { + const filePath = join(tempDir, 'file.txt') + writeFileSync(filePath, 'test', 'utf-8') + expect(() => analyzeSkill(filePath)).toThrow('Not a directory') + }) + + it('produces valid JSON output', () => { + createFixture(tempDir, { + 'SKILL.md': 'Simple skill content.', + }) + + const result = analyzeSkill(tempDir) + const json = JSON.stringify(result) + + expect(() => JSON.parse(json)).not.toThrow() + + const parsed = JSON.parse(json) + expect(parsed).toHaveProperty('findings') + expect(parsed).toHaveProperty('summary') + expect(parsed.summary).toHaveProperty('total_findings') + expect(parsed.summary).toHaveProperty('estimated_tokens_per_invocation') + expect(parsed.summary).toHaveProperty('efficiency_score') + }) + + it('calculates efficiency score correctly for clean skill', () => { + createFixture(tempDir, { + 'SKILL.md': 'A concise skill with minimal overhead.', + }) + + const result = analyzeSkill(tempDir) + + // Clean small skill should score high + expect(result.summary.efficiency_score).toBeGreaterThanOrEqual(90) + }) + + it('calculates efficiency score correctly for bloated skill', () => { + const largeContent = 'X'.repeat(20000) + createFixture(tempDir, { + 'SKILL.md': largeContent, + 'CLAUDE.md': largeContent, + }) + + const result = analyzeSkill(tempDir) + + // Bloated skill should score lower + expect(result.summary.efficiency_score).toBeLessThan(90) + }) +}) From ffd577d2fb284a9925c4edd50e1d7267394df06e Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Wed, 15 Apr 2026 23:42:02 +0200 Subject: [PATCH 2/9] docs: add --analyze-skill section to README, bump version description to v2.3 - Add Skill Analysis section with output format, rules table, efficiency score - Add integration example for calling from Python (Tank pipeline) - Add --analyze-skill to Options table - Update version description from v2.2 to v2.3 Co-Authored-By: Claude --- README.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a0684c..19152fc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A CLI tool that analyzes your Claude Code session history to identify token waste patterns and provide actionable fixes. Runs locally, no LLM needed. -**v2.2** adds human-readable findings with plain-English explanations, collapsible HTML report sections, and evidence-based action items. +**v2.3** adds `--analyze-skill` mode for static token analysis of AI agent skill packages — designed for integration with skill registries like [Tank](https://github.com/galando/tank). **The mental model: tokenomics is a coach, not a remote control.** It writes suggestions into your CLAUDE.md where Claude can see them. Claude is smart enough to follow most of them — suggesting `/compact` when context grows, recommending Sonnet for simple tasks, warning when you're overspending. But it cannot switch models, run commands, or force behavior. The user is always in control. @@ -240,6 +240,7 @@ Budget config lives at `~/.claude/tokenomics.json`: | `--inject` | Re-analyze sessions + update CLAUDE.md findings | false | | `--quiet` | Suppress terminal output (used by hooks) | false | | `--verbose` | Show discovery progress and debug info | false | +| `--analyze-skill ` | Analyze skill package for token efficiency (outputs JSON) | - | | `--help` | Show help message | - | | `--version` | Show version | - | @@ -264,6 +265,82 @@ Budget config lives at `~/.claude/tokenomics.json`: | Subagent Opportunity | Delegation opportunities | Parallel execution | | Smart Router | Historical routing patterns | ~80% on simple tasks | +## Skill Analysis (`--analyze-skill`) + +Analyze an AI agent skill package directory for token efficiency. Useful for skill authors before publishing, and for skill registries (like [Tank](https://github.com/galando/tank)) to include token findings alongside security scan results. + +```bash +# Analyze a skill package +tokenomics --analyze-skill ./my-skill + +# With pretty-printed JSON +tokenomics --analyze-skill ./my-skill --verbose +``` + +### Output format + +```json +{ + "findings": [ + { + "rule": "prompt-size", + "severity": "medium", + "confidence": 0.9, + "description": "Prompt file \"SKILL.md\" is ~4,200 tokens (16,800 chars)...", + "location": "SKILL.md", + "remediation": "Trim to under 2000 tokens. Move detailed examples to separate files..." + }, + { + "rule": "tool-overhead", + "severity": "medium", + "confidence": 0.85, + "description": "12 tool definitions found. Each adds ~200-500 tokens of context...", + "location": "manifest/config", + "remediation": "Consider reducing to 8 or fewer tools..." + } + ], + "summary": { + "total_findings": 3, + "estimated_tokens_per_invocation": 8500, + "efficiency_score": 72 + } +} +``` + +### Analysis rules + +| Rule | What it checks | Thresholds | +|------|---------------|------------| +| `prompt-size` | SKILL.md and `.atom.md` files exceeding token limits | >2000 tokens = medium, >4000 = high | +| `claude-md-size` | CLAUDE.md injection blocks that bloat system prompts | >1500 tokens = medium, >3000 = high | +| `tool-overhead` | Tool definitions in manifest (tank.json, skills.json) | >8 tools = medium, >15 = high | +| `large-files` | Files with >500 lines that are expensive to read | >500 lines = low, >1000 = medium | +| `redundant-instructions` | Duplicated instruction lines across skill files | >30% duplication = low/medium | + +### Efficiency score + +The `efficiency_score` (0-100) reflects overall token efficiency. It starts at 100 and deducts points per finding: -15 for high, -8 for medium, -3 for low, -1 for info. A bonus +10 is given for skill packages under 1000 total tokens. + +### Integration with skill registries + +The `--analyze-skill` mode is designed to be called as a subprocess from security scanning pipelines. Example from Tank's Python scanner: + +```python +import subprocess, json + +result = subprocess.run( + ["tokenomics", "--analyze-skill", skill_dir, "--json"], + capture_output=True, text=True, timeout=10 +) +if result.returncode == 0: + analysis = json.loads(result.stdout) + for finding in analysis["findings"]: + # Convert to your scan Finding model + print(f"[{finding['severity']}] {finding['rule']}: {finding['description']}") +``` + +Graceful degradation: if tokenomics is not installed, the calling pipeline should skip this stage and continue. + ## Output Examples ### Terminal From a4970868de4bba3fadc677a63be4c0c7951a0e79 Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Wed, 15 Apr 2026 23:43:14 +0200 Subject: [PATCH 3/9] docs: fix Tank repo URL to github.com/tankpkg/tank Co-Authored-By: Claude --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 19152fc..424174f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A CLI tool that analyzes your Claude Code session history to identify token waste patterns and provide actionable fixes. Runs locally, no LLM needed. -**v2.3** adds `--analyze-skill` mode for static token analysis of AI agent skill packages — designed for integration with skill registries like [Tank](https://github.com/galando/tank). +**v2.3** adds `--analyze-skill` mode for static token analysis of AI agent skill packages — designed for integration with skill registries like [Tank](https://github.com/tankpkg/tank). **The mental model: tokenomics is a coach, not a remote control.** It writes suggestions into your CLAUDE.md where Claude can see them. Claude is smart enough to follow most of them — suggesting `/compact` when context grows, recommending Sonnet for simple tasks, warning when you're overspending. But it cannot switch models, run commands, or force behavior. The user is always in control. @@ -267,7 +267,7 @@ Budget config lives at `~/.claude/tokenomics.json`: ## Skill Analysis (`--analyze-skill`) -Analyze an AI agent skill package directory for token efficiency. Useful for skill authors before publishing, and for skill registries (like [Tank](https://github.com/galando/tank)) to include token findings alongside security scan results. +Analyze an AI agent skill package directory for token efficiency. Useful for skill authors before publishing, and for skill registries (like [Tank](https://github.com/tankpkg/tank)) to include token findings alongside security scan results. ```bash # Analyze a skill package From c97770abdf15f618985159428eb258a9688ef83f Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Thu, 16 Apr 2026 11:03:56 +0200 Subject: [PATCH 4/9] feat(skill-analysis): add section-level token analysis with redundancy detection New section-analysis rule that parses markdown headings and provides: - Per-section token count breakdown - Cross-section redundancy detection (Jaccard similarity >40%) - Shortening tips: repeated lines, excessive code blocks, long lists, long paragraphs - Sections array in findings for structured consumption - src/skill-rules/section-analysis.ts: new rule with markdown section parser - src/types.ts: added SkillSection interface with tokens, redundantWith, shorteningTip - src/skill-analyzer.ts: wired section-analysis rule - tests/skill-analyzer.test.ts: 10 new tests (36 total for skill analyzer) Co-Authored-By: Claude --- src/skill-analyzer.ts | 2 + src/skill-rules/section-analysis.ts | 259 ++++++++++++++++++++++++++++ src/types.ts | 17 ++ tests/skill-analyzer.test.ts | 219 +++++++++++++++++++++++ 4 files changed, 497 insertions(+) create mode 100644 src/skill-rules/section-analysis.ts diff --git a/src/skill-analyzer.ts b/src/skill-analyzer.ts index ac44445..548e7aa 100644 --- a/src/skill-analyzer.ts +++ b/src/skill-analyzer.ts @@ -15,6 +15,7 @@ import * as claudeMdSizeRule from './skill-rules/claude-md-size.js' import * as toolOverheadRule from './skill-rules/tool-overhead.js' import * as largeFilesRule from './skill-rules/large-files.js' import * as redundantInstructionsRule from './skill-rules/redundant-instructions.js' +import * as sectionAnalysisRule from './skill-rules/section-analysis.js' const CHARS_PER_TOKEN = 4 @@ -33,6 +34,7 @@ const rules: SkillRule[] = [ { name: 'tool-overhead', analyze: toolOverheadRule.analyze }, { name: 'large-files', analyze: largeFilesRule.analyze }, { name: 'redundant-instructions', analyze: redundantInstructionsRule.analyze }, + { name: 'section-analysis', analyze: sectionAnalysisRule.analyze }, ] function discoverSkillFiles(dir: string): Map { diff --git a/src/skill-rules/section-analysis.ts b/src/skill-rules/section-analysis.ts new file mode 100644 index 0000000..9aeac7d --- /dev/null +++ b/src/skill-rules/section-analysis.ts @@ -0,0 +1,259 @@ +/** + * Section Analysis Rule + * + * Parses markdown files into sections (by headings) and: + * 1. Reports per-section token counts + * 2. Detects sections with redundant/overlapping content + * 3. Suggests which sections can be shortened and why + * + * Only analyzes SKILL.md and .atom.md files. + */ + +import type { SkillAnalysisContext, SkillFinding, SkillSection } from '../types.js' + +const CHARS_PER_TOKEN = 4 + +const SECTION_TOKEN_THRESHOLD = 500 // Flag sections >500 tokens +const LARGE_SECTION_THRESHOLD = 1000 // Flag sections >1000 tokens + +const PROMPT_FILE_PATTERNS = [ + /^SKILL\.md$/i, + /\.atom\.md$/i, +] + +function isPromptFile(filename: string): boolean { + return PROMPT_FILE_PATTERNS.some(p => p.test(filename)) +} + +interface ParsedSection { + heading: string + level: number + lineStart: number + content: string + tokens: number +} + +function parseSections(content: string): ParsedSection[] { + const lines = content.split('\n') + const sections: ParsedSection[] = [] + let currentHeading = '(preamble)' + let currentLevel = 0 + let currentLineStart = 0 + let currentLines: string[] = [] + + for (let i = 0; i < lines.length; i++) { + const headingMatch = lines[i]!.match(/^(#{1,6})\s+(.+)$/) + if (headingMatch) { + // Flush previous section + if (currentLines.length > 0 || currentHeading !== '(preamble)') { + const sectionContent = currentLines.join('\n') + sections.push({ + heading: currentHeading, + level: currentLevel, + lineStart: currentLineStart, + content: sectionContent, + tokens: Math.ceil(sectionContent.length / CHARS_PER_TOKEN), + }) + } + currentHeading = headingMatch[2]!.trim() + currentLevel = headingMatch[1]!.length + currentLineStart = i + currentLines = [] + } else { + currentLines.push(lines[i]!) + } + } + + // Flush last section + const lastContent = currentLines.join('\n') + if (lastContent.trim().length > 0 || currentHeading !== '(preamble)') { + sections.push({ + heading: currentHeading, + level: currentLevel, + lineStart: currentLineStart, + content: lastContent, + tokens: Math.ceil(lastContent.length / CHARS_PER_TOKEN), + }) + } + + return sections +} + +function normalizeContent(content: string): string { + return content + .toLowerCase() + .replace(/```[\s\S]*?```/g, '') // Remove code blocks + .replace(/`[^`]+`/g, '') // Remove inline code + .replace(/[^\w\s]/g, '') // Remove punctuation + .replace(/\s+/g, ' ') + .trim() +} + +function detectRedundantSections(sections: ParsedSection[]): Map { + const redundancyMap = new Map() + + for (let i = 0; i < sections.length; i++) { + const normA = normalizeContent(sections[i]!.content) + if (normA.length < 40) continue // Skip tiny sections + + for (let j = i + 1; j < sections.length; j++) { + const normB = normalizeContent(sections[j]!.content) + if (normB.length < 40) continue + + // Jaccard similarity on word sets + const wordsA = new Set(normA.split(' ').filter(w => w.length > 3)) + const wordsB = new Set(normB.split(' ').filter(w => w.length > 3)) + + if (wordsA.size === 0 || wordsB.size === 0) continue + + const intersection = [...wordsA].filter(w => wordsB.has(w)) + const union = new Set([...wordsA, ...wordsB]) + const similarity = intersection.length / union.size + + if (similarity > 0.4) { + const existing = redundancyMap.get(i) ?? [] + existing.push(sections[j]!.heading) + redundancyMap.set(i, existing) + + const existingJ = redundancyMap.get(j) ?? [] + existingJ.push(sections[i]!.heading) + redundancyMap.set(j, existingJ) + } + } + } + + return redundancyMap +} + +function getShorteningTip(section: ParsedSection): string | undefined { + const lines = section.content.split('\n').filter(l => l.trim().length > 0) + const tips: string[] = [] + + // Check for repeated lines + const lineCounts = new Map() + for (const line of lines) { + const norm = line.toLowerCase().trim() + if (norm.length < 20) continue + lineCounts.set(norm, (lineCounts.get(norm) ?? 0) + 1) + } + const repeatedLines = [...lineCounts.entries()].filter(([, c]) => c > 1) + if (repeatedLines.length > 0) { + const example = repeatedLines[0]![0]!.slice(0, 60) + tips.push(`Contains ${repeatedLines.length} repeated line(s), e.g. "${example}..."`) + } + + // Check for code blocks + const codeBlockCount = (section.content.match(/```/g) ?? []).length / 2 + if (codeBlockCount >= 2) { + tips.push(`Has ${codeBlockCount} code blocks — consider replacing examples with file references`) + } + + // Check for list items that could be collapsed + const listItems = lines.filter(l => /^\s*[-*]\s/.test(l) || /^\s*\d+\.\s/.test(l)) + if (listItems.length > 8) { + tips.push(`${listItems.length} list items — consider grouping into categories or moving details to separate files`) + } + + // Check for very long paragraphs + const paragraphs = section.content.split(/\n\s*\n/).filter(p => p.trim().length > 0) + const longParagraphs = paragraphs.filter(p => p.length > 1500) + if (longParagraphs.length > 0) { + tips.push(`${longParagraphs.length} long paragraph(s) over 375 tokens — consider breaking into bullet points`) + } + + return tips.length > 0 ? tips.join('. ') + '.' : undefined +} + +export function analyze(context: SkillAnalysisContext): SkillFinding[] { + const findings: SkillFinding[] = [] + + for (const [filename, content] of context.files) { + if (!isPromptFile(filename)) continue + if (content.trim().length === 0) continue + + const sections = parseSections(content) + if (sections.length === 0) continue + + const redundancyMap = detectRedundantSections(sections) + + // Build per-section data with tips + const sectionData: SkillSection[] = sections.map((s, idx) => ({ + heading: s.heading, + level: s.level, + lineStart: s.lineStart, + tokens: s.tokens, + redundantWith: redundancyMap.get(idx), + shorteningTip: getShorteningTip(s), + })) + + // Find oversized sections + const largeSections = sections.filter(s => s.tokens > SECTION_TOKEN_THRESHOLD) + + // Find sections with redundancy + const redundantSections = [...redundancyMap.entries()] + .filter(([, peers]) => peers.length > 0) + .map(([idx]) => sections[idx]!) + + // Find sections with shortening opportunities + const shortenableSections = sections.filter((_, idx) => { + const tip = sectionData[idx]!.shorteningTip + return tip !== undefined + }) + + // Generate finding if anything actionable + const hasIssues = largeSections.length > 0 || redundantSections.length > 0 || shortenableSections.length > 0 + + if (!hasIssues) continue + + // Build description + const parts: string[] = [] + if (largeSections.length > 0) { + const top = largeSections.sort((a, b) => b.tokens - a.tokens)[0]! + parts.push(`Largest section "${top.heading}" is ~${top.tokens.toLocaleString()} tokens`) + } + if (redundantSections.length > 0) { + const names = redundantSections.map(s => `"${s.heading}"`) + parts.push(`${redundantSections.length} section(s) overlap: ${names.join(', ')}`) + } + if (shortenableSections.length > 0) { + parts.push(`${shortenableSections.length} section(s) have shortening opportunities`) + } + + // Determine severity + const maxTokens = Math.max(...sections.map(s => s.tokens)) + const severity = maxTokens > LARGE_SECTION_THRESHOLD + ? 'high' + : largeSections.length > 0 + ? 'medium' + : redundantSections.length > 0 + ? 'low' + : 'info' + + // Build remediation + const remediationParts: string[] = [] + for (const s of largeSections.sort((a, b) => b.tokens - a.tokens).slice(0, 3)) { + remediationParts.push(`"${s.heading}" (${s.tokens} tokens): split into smaller focused subsections or move details to separate files`) + } + for (const s of redundantSections.slice(0, 3)) { + const peers = redundancyMap.get(sections.indexOf(s)) ?? [] + remediationParts.push(`"${s.heading}" overlaps with ${peers.join(', ')}: consolidate shared content into one section`) + } + for (let i = 0; i < shortenableSections.length; i++) { + const idx = sections.indexOf(shortenableSections[i]!) + const tip = sectionData[idx]!.shorteningTip + remediationParts.push(`"${shortenableSections[i]!.heading}": ${tip}`) + } + + findings.push({ + rule: 'section-analysis', + severity, + confidence: 0.8, + description: `${filename}: ${parts.join('. ')}. Total file: ~${Math.ceil(content.length / CHARS_PER_TOKEN).toLocaleString()} tokens across ${sections.length} sections.`, + location: filename, + remediation: remediationParts.join('\n'), + sections: sectionData, + }) + } + + return findings +} diff --git a/src/types.ts b/src/types.ts index 0d06f3b..68ca2b0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -690,6 +690,23 @@ export interface SkillFinding { description: string; location: string; remediation: string; + /** Optional per-section token breakdown (section-analysis rule) */ + sections?: SkillSection[]; +} + +export interface SkillSection { + /** Section heading text (e.g. "## Instructions") */ + heading: string; + /** Heading level (1-6) */ + level: number; + /** 0-based line offset where this section starts */ + lineStart: number; + /** Estimated token count for this section's content (excluding heading) */ + tokens: number; + /** Whether this section overlaps with another section */ + redundantWith?: string[]; + /** Why this section could be shortened */ + shorteningTip?: string; } export interface SkillAnalysisSummary { diff --git a/tests/skill-analyzer.test.ts b/tests/skill-analyzer.test.ts index 8387f97..a391f15 100644 --- a/tests/skill-analyzer.test.ts +++ b/tests/skill-analyzer.test.ts @@ -9,6 +9,7 @@ import * as claudeMdSizeRule from '../src/skill-rules/claude-md-size.js' import * as toolOverheadRule from '../src/skill-rules/tool-overhead.js' import * as largeFilesRule from '../src/skill-rules/large-files.js' import * as redundantInstructionsRule from '../src/skill-rules/redundant-instructions.js' +import * as sectionAnalysisRule from '../src/skill-rules/section-analysis.js' import type { SkillAnalysisContext } from '../src/types.js' function createTempDir(): string { @@ -250,6 +251,224 @@ describe('redundant-instructions rule', () => { }) }) +// ─── Section Analysis Rule ──────────────────────────────────────────────────── + +describe('section-analysis rule', () => { + it('reports per-section token counts', () => { + const skill = [ + '# My Skill', + '', + 'A brief description of the skill.', + '', + '## Instructions', + '', + ...Array(50).fill('When the user asks for help, follow these steps carefully and provide detailed feedback.'), + '', + '## Examples', + '', + ...Array(30).fill('Here is an example of how to handle a request from the user.'), + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + expect(findings.length).toBe(1) + expect(findings[0]!.rule).toBe('section-analysis') + expect(findings[0]!.sections).toBeDefined() + expect(findings[0]!.sections!.length).toBe(3) // preamble, Instructions, Examples + + // Check that sections have token counts + const instructionsSection = findings[0]!.sections!.find(s => s.heading === 'Instructions') + expect(instructionsSection).toBeDefined() + expect(instructionsSection!.tokens).toBeGreaterThan(0) + }) + + it('flags sections with redundant/overlapping content', () => { + const sharedContent = Array(20).fill( + 'Always use TypeScript strict mode for all new files in this project and follow the established patterns' + ).join(' ') + + const skill = [ + '# My Skill', + '', + '## Code Review', + '', + sharedContent, + 'Also check for security issues in the authentication module.', + '', + '## Security Review', + '', + sharedContent, + 'Also verify that authentication tokens are properly validated.', + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + expect(findings.length).toBe(1) + const finding = findings[0]! + expect(finding.description).toContain('overlap') + + // Check redundantWith is populated + const codeReview = finding.sections!.find(s => s.heading === 'Code Review') + expect(codeReview!.redundantWith).toBeDefined() + expect(codeReview!.redundantWith).toContain('Security Review') + }) + + it('suggests shortening tips for sections with repeated lines', () => { + const skill = [ + '# My Skill', + '', + '## Guidelines', + '', + ...Array(5).fill('Always check the types before submitting code changes.'), + '', + 'Some unique content here that is different.', + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + // Should have findings since Guidelines has repeated lines + if (findings.length > 0) { + const guidelines = findings[0]!.sections!.find(s => s.heading === 'Guidelines') + if (guidelines?.shorteningTip) { + expect(guidelines.shorteningTip).toContain('repeated') + } + } + }) + + it('suggests shortening for sections with many list items', () => { + const listItems = Array(12).fill(0).map((_, i) => `- Item number ${i + 1} that describes a specific check to perform`).join('\n') + + const skill = [ + '# My Skill', + '', + '## Checklist', + '', + listItems, + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + if (findings.length > 0) { + const checklist = findings[0]!.sections!.find(s => s.heading === 'Checklist') + if (checklist?.shorteningTip) { + expect(checklist.shorteningTip).toContain('list items') + } + } + }) + + it('suggests shortening for sections with many code blocks', () => { + const codeBlocks = Array(3).fill(0).map((_, i) => `Example ${i}:\n\`\`\`ts\nconst x = ${i};\n\`\`\``).join('\n\n') + + const skill = [ + '# My Skill', + '', + '## Examples', + '', + codeBlocks, + '', + 'Some additional content to make this section large enough to be over the threshold.', + 'More filler content that pushes the token count above the detection minimum.', + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + if (findings.length > 0) { + const examples = findings[0]!.sections!.find(s => s.heading === 'Examples') + if (examples?.shorteningTip) { + expect(examples.shorteningTip).toContain('code blocks') + } + } + }) + + it('does not flag small, clean sections', () => { + const skill = [ + '# My Skill', + '', + 'A brief skill that does one thing.', + '', + '## Usage', + '', + 'Run the command with the file path.', + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + expect(findings.length).toBe(0) + }) + + it('ignores non-prompt files', () => { + const ctx = makeContext({ 'package.json': '{"name": "test"}' }) + const findings = sectionAnalysisRule.analyze(ctx) + + expect(findings.length).toBe(0) + }) + + it('analyzes .atom.md files', () => { + const skill = [ + '## Search', + '', + ...Array(60).fill('Detailed search instructions that explain how to search for files in the project directory.'), + ].join('\n') + + const ctx = makeContext({ 'tools/search.atom.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + expect(findings.length).toBe(1) + expect(findings[0]!.location).toContain('.atom.md') + }) + + it('includes sections array in JSON output', () => { + const skill = [ + '# Skill', + '', + 'Preamble content here.', + '', + '## Section A', + '', + 'Content for section A that is reasonably sized.', + '', + '## Section B', + '', + 'Content for section B that is also reasonably sized.', + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + // Even if no findings (sections are small), verify JSON serialization + if (findings.length > 0) { + const json = JSON.stringify(findings[0]) + const parsed = JSON.parse(json) + expect(parsed.sections).toBeDefined() + expect(Array.isArray(parsed.sections)).toBe(true) + } + }) + + it('high severity for sections >1000 tokens', () => { + const hugeSection = 'Very detailed instruction content. '.repeat(600) // ~16800 chars = ~4200 tokens + + const skill = [ + '# Skill', + '', + '## Deep Dive', + '', + hugeSection, + ].join('\n') + + const ctx = makeContext({ 'SKILL.md': skill }) + const findings = sectionAnalysisRule.analyze(ctx) + + expect(findings.length).toBe(1) + expect(findings[0]!.severity).toBe('high') + }) +}) + // ─── Integration: Full Skill Analyzer ──────────────────────────────────────── describe('skill analyzer integration', () => { From b15610ee8ece2d7f38d76da0f0c6f0eba480532b Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Thu, 16 Apr 2026 11:58:56 +0200 Subject: [PATCH 5/9] feat(skill-analysis): cross-file redundancy detection, always-report breakdown Major improvements to section-analysis rule based on testing against real Tank skills (@tank/bdd-e2e-testing, @tank/idd-bdd-pack): - Cross-file redundancy: detects when SKILL.md sections duplicate content from references/*.md files, with specific file+section attribution - Always-report: files over 1000 tokens get section breakdown even if no issues found (info severity) - Lowered Jaccard threshold from 40% to 25% for better recall on real skills - Table-heavy section detection (>15 table rows) - Top-5 section token breakdown in description for quick scan - Fixed within-file vs cross-file distinction in remediation messages Co-Authored-By: Claude --- src/skill-rules/section-analysis.ts | 243 ++++++++++++++++++++-------- tests/skill-analyzer.test.ts | 2 +- 2 files changed, 177 insertions(+), 68 deletions(-) diff --git a/src/skill-rules/section-analysis.ts b/src/skill-rules/section-analysis.ts index 9aeac7d..6a92ab3 100644 --- a/src/skill-rules/section-analysis.ts +++ b/src/skill-rules/section-analysis.ts @@ -2,29 +2,35 @@ * Section Analysis Rule * * Parses markdown files into sections (by headings) and: - * 1. Reports per-section token counts - * 2. Detects sections with redundant/overlapping content + * 1. Reports per-section token counts (always for files >1000 tokens) + * 2. Detects sections with redundant/overlapping content (within file AND cross-file) * 3. Suggests which sections can be shortened and why * - * Only analyzes SKILL.md and .atom.md files. + * Analyzes SKILL.md, .atom.md, and reference files under references/. */ import type { SkillAnalysisContext, SkillFinding, SkillSection } from '../types.js' const CHARS_PER_TOKEN = 4 - -const SECTION_TOKEN_THRESHOLD = 500 // Flag sections >500 tokens -const LARGE_SECTION_THRESHOLD = 1000 // Flag sections >1000 tokens +const SECTION_TOKEN_THRESHOLD = 500 +const LARGE_SECTION_THRESHOLD = 1000 +const FILE_TOKEN_BREAKDOWN_THRESHOLD = 1000 // Always report breakdown for files above this const PROMPT_FILE_PATTERNS = [ /^SKILL\.md$/i, /\.atom\.md$/i, ] +const ALL_MD_PATTERN = /\.md$/i + function isPromptFile(filename: string): boolean { return PROMPT_FILE_PATTERNS.some(p => p.test(filename)) } +function isMarkdownFile(filename: string): boolean { + return ALL_MD_PATTERN.test(filename) +} + interface ParsedSection { heading: string level: number @@ -33,6 +39,12 @@ interface ParsedSection { tokens: number } +interface FileSections { + filename: string + sections: ParsedSection[] + totalTokens: number +} + function parseSections(content: string): ParsedSection[] { const lines = content.split('\n') const sections: ParsedSection[] = [] @@ -42,6 +54,7 @@ function parseSections(content: string): ParsedSection[] { let currentLines: string[] = [] for (let i = 0; i < lines.length; i++) { + // Match markdown headings: # through ######, also handle frontmatter (---) const headingMatch = lines[i]!.match(/^(#{1,6})\s+(.+)$/) if (headingMatch) { // Flush previous section @@ -82,47 +95,72 @@ function parseSections(content: string): ParsedSection[] { function normalizeContent(content: string): string { return content .toLowerCase() - .replace(/```[\s\S]*?```/g, '') // Remove code blocks - .replace(/`[^`]+`/g, '') // Remove inline code - .replace(/[^\w\s]/g, '') // Remove punctuation + .replace(/```[\s\S]*?```/g, '') + .replace(/`[^`]+`/g, '') + .replace(/[^\w\s]/g, '') .replace(/\s+/g, ' ') .trim() } -function detectRedundantSections(sections: ParsedSection[]): Map { - const redundancyMap = new Map() +function jaccardSimilarity(textA: string, textB: string): number { + const normA = normalizeContent(textA) + const normB = normalizeContent(textB) + if (normA.length < 20 || normB.length < 20) return 0 - for (let i = 0; i < sections.length; i++) { - const normA = normalizeContent(sections[i]!.content) - if (normA.length < 40) continue // Skip tiny sections + const wordsA = new Set(normA.split(' ').filter(w => w.length > 3)) + const wordsB = new Set(normB.split(' ').filter(w => w.length > 3)) + if (wordsA.size === 0 || wordsB.size === 0) return 0 - for (let j = i + 1; j < sections.length; j++) { - const normB = normalizeContent(sections[j]!.content) - if (normB.length < 40) continue + const intersection = [...wordsA].filter(w => wordsB.has(w)) + const union = new Set([...wordsA, ...wordsB]) + return intersection.length / union.size +} - // Jaccard similarity on word sets - const wordsA = new Set(normA.split(' ').filter(w => w.length > 3)) - const wordsB = new Set(normB.split(' ').filter(w => w.length > 3)) +interface RedundancyLink { + sourceFile: string + sourceSection: string + targetFile: string + targetSection: string + similarity: number +} - if (wordsA.size === 0 || wordsB.size === 0) continue +interface RedundancyPeer { + file: string + section: string +} - const intersection = [...wordsA].filter(w => wordsB.has(w)) - const union = new Set([...wordsA, ...wordsB]) - const similarity = intersection.length / union.size +function detectRedundancy(allFileSections: FileSections[]): RedundancyLink[] { + const links: RedundancyLink[] = [] + const allSections: Array<{ file: string; section: ParsedSection }> = [] - if (similarity > 0.4) { - const existing = redundancyMap.get(i) ?? [] - existing.push(sections[j]!.heading) - redundancyMap.set(i, existing) + for (const fs of allFileSections) { + for (const s of fs.sections) { + allSections.push({ file: fs.filename, section: s }) + } + } - const existingJ = redundancyMap.get(j) ?? [] - existingJ.push(sections[i]!.heading) - redundancyMap.set(j, existingJ) + for (let i = 0; i < allSections.length; i++) { + const a = allSections[i]! + if (a.section.tokens < 20) continue // Skip tiny sections + + for (let j = i + 1; j < allSections.length; j++) { + const b = allSections[j]! + if (b.section.tokens < 20) continue + + const similarity = jaccardSimilarity(a.section.content, b.section.content) + if (similarity > 0.25) { + links.push({ + sourceFile: a.file, + sourceSection: a.section.heading, + targetFile: b.file, + targetSection: b.section.heading, + similarity, + }) } } } - return redundancyMap + return links.sort((a, b) => b.similarity - a.similarity) } function getShorteningTip(section: ParsedSection): string | undefined { @@ -145,7 +183,7 @@ function getShorteningTip(section: ParsedSection): string | undefined { // Check for code blocks const codeBlockCount = (section.content.match(/```/g) ?? []).length / 2 if (codeBlockCount >= 2) { - tips.push(`Has ${codeBlockCount} code blocks — consider replacing examples with file references`) + tips.push(`Has ${Math.floor(codeBlockCount)} code blocks — consider replacing examples with file references`) } // Check for list items that could be collapsed @@ -161,49 +199,91 @@ function getShorteningTip(section: ParsedSection): string | undefined { tips.push(`${longParagraphs.length} long paragraph(s) over 375 tokens — consider breaking into bullet points`) } + // Check for table-heavy content + const tableRows = lines.filter(l => /^\|/.test(l.trim())) + if (tableRows.length > 15) { + tips.push(`${tableRows.length} table rows — consider moving detailed tables to reference files and keeping only a summary`) + } + return tips.length > 0 ? tips.join('. ') + '.' : undefined } export function analyze(context: SkillAnalysisContext): SkillFinding[] { const findings: SkillFinding[] = [] + // Parse all markdown files into sections + const promptFileSections: FileSections[] = [] + const referenceFileSections: FileSections[] = [] + for (const [filename, content] of context.files) { - if (!isPromptFile(filename)) continue + if (!isMarkdownFile(filename)) continue if (content.trim().length === 0) continue const sections = parseSections(content) if (sections.length === 0) continue - const redundancyMap = detectRedundantSections(sections) + const totalTokens = Math.ceil(content.length / CHARS_PER_TOKEN) + const entry: FileSections = { filename, sections, totalTokens } - // Build per-section data with tips - const sectionData: SkillSection[] = sections.map((s, idx) => ({ - heading: s.heading, - level: s.level, - lineStart: s.lineStart, - tokens: s.tokens, - redundantWith: redundancyMap.get(idx), - shorteningTip: getShorteningTip(s), - })) - - // Find oversized sections - const largeSections = sections.filter(s => s.tokens > SECTION_TOKEN_THRESHOLD) + if (isPromptFile(filename)) { + promptFileSections.push(entry) + } else if (filename.includes('references/')) { + referenceFileSections.push(entry) + } + } - // Find sections with redundancy - const redundantSections = [...redundancyMap.entries()] - .filter(([, peers]) => peers.length > 0) - .map(([idx]) => sections[idx]!) + // Cross-file + within-file redundancy detection across all files + const allFileSections = [...promptFileSections, ...referenceFileSections] + const redundancyLinks = detectRedundancy(allFileSections) + + // Build redundancy lookup: filename → section heading → redundant peers + const redundancyLookup = new Map>() + for (const link of redundancyLinks) { + // Forward direction + let fileMap = redundancyLookup.get(link.sourceFile) + if (!fileMap) { fileMap = new Map(); redundancyLookup.set(link.sourceFile, fileMap) } + let peers = fileMap.get(link.sourceSection) + if (!peers) { peers = []; fileMap.set(link.sourceSection, peers) } + peers.push({ file: link.targetFile, section: link.targetSection }) + + // Reverse direction + fileMap = redundancyLookup.get(link.targetFile) + if (!fileMap) { fileMap = new Map(); redundancyLookup.set(link.targetFile, fileMap) } + peers = fileMap.get(link.targetSection) + if (!peers) { peers = []; fileMap.set(link.targetSection, peers) } + peers.push({ file: link.sourceFile, section: link.sourceSection }) + } - // Find sections with shortening opportunities - const shortenableSections = sections.filter((_, idx) => { - const tip = sectionData[idx]!.shorteningTip - return tip !== undefined + // Generate findings per prompt file + for (const { filename, sections, totalTokens } of promptFileSections) { + const fileRedundancy = redundancyLookup.get(filename) ?? new Map() + + // Build per-section data + const sectionData: SkillSection[] = sections.map((s) => { + const peers: RedundancyPeer[] | undefined = fileRedundancy.get(s.heading) + // Deduplicate peer labels + const peerLabels: string[] | undefined = peers + ? [...new Set(peers.map((p: RedundancyPeer) => p.file === filename ? `"${p.section}"` : `${p.file} → "${p.section}"`))] + : undefined + + return { + heading: s.heading, + level: s.level, + lineStart: s.lineStart, + tokens: s.tokens, + redundantWith: peerLabels && peerLabels.length > 0 ? peerLabels : undefined, + shorteningTip: getShorteningTip(s), + } }) - // Generate finding if anything actionable - const hasIssues = largeSections.length > 0 || redundantSections.length > 0 || shortenableSections.length > 0 + // Determine what's actionable + const largeSections = sections.filter(s => s.tokens > SECTION_TOKEN_THRESHOLD) + const redundantSections = sections.filter(s => (fileRedundancy.get(s.heading) ?? []).length > 0) + const shortenableSections = sections.filter((_, idx) => sectionData[idx]!.shorteningTip !== undefined) + const hasLargeFile = totalTokens > FILE_TOKEN_BREAKDOWN_THRESHOLD - if (!hasIssues) continue + // Generate finding if anything actionable OR file is large enough to warrant breakdown + if (!hasLargeFile && largeSections.length === 0 && redundantSections.length === 0 && shortenableSections.length === 0) continue // Build description const parts: string[] = [] @@ -212,8 +292,21 @@ export function analyze(context: SkillAnalysisContext): SkillFinding[] { parts.push(`Largest section "${top.heading}" is ~${top.tokens.toLocaleString()} tokens`) } if (redundantSections.length > 0) { - const names = redundantSections.map(s => `"${s.heading}"`) - parts.push(`${redundantSections.length} section(s) overlap: ${names.join(', ')}`) + const crossFile = redundantSections.filter(s => { + const peers: RedundancyPeer[] = fileRedundancy.get(s.heading) ?? [] + return peers.some((p: RedundancyPeer) => p.file !== filename) + }) + if (crossFile.length > 0) { + parts.push(`${crossFile.length} section(s) duplicate content from reference files`) + } + const withinFile = redundantSections.filter(s => { + const peers: RedundancyPeer[] = fileRedundancy.get(s.heading) ?? [] + return peers.some((p: RedundancyPeer) => p.file === filename) + }) + if (withinFile.length > 0) { + const names = withinFile.map(s => `"${s.heading}"`) + parts.push(`${withinFile.length} section(s) overlap each other: ${names.join(', ')}`) + } } if (shortenableSections.length > 0) { parts.push(`${shortenableSections.length} section(s) have shortening opportunities`) @@ -235,22 +328,38 @@ export function analyze(context: SkillAnalysisContext): SkillFinding[] { remediationParts.push(`"${s.heading}" (${s.tokens} tokens): split into smaller focused subsections or move details to separate files`) } for (const s of redundantSections.slice(0, 3)) { - const peers = redundancyMap.get(sections.indexOf(s)) ?? [] - remediationParts.push(`"${s.heading}" overlaps with ${peers.join(', ')}: consolidate shared content into one section`) + const peers: RedundancyPeer[] = fileRedundancy.get(s.heading) ?? [] + const crossFilePeers = peers.filter((p: RedundancyPeer) => p.file !== filename) + const sameFilePeers = peers.filter((p: RedundancyPeer) => p.file === filename) + + if (crossFilePeers.length > 0) { + const targets = [...new Set(crossFilePeers.map((p: RedundancyPeer) => `${p.file} "${p.section}"`))].join(', ') + remediationParts.push(`"${s.heading}" duplicates content from ${targets}: keep it in one place, reference from the other`) + } + if (sameFilePeers.length > 0) { + const targets = [...new Set(sameFilePeers.map((p: RedundancyPeer) => `"${p.section}"`))].join(', ') + remediationParts.push(`"${s.heading}" overlaps with ${targets}: consolidate shared content into one section`) + } } - for (let i = 0; i < shortenableSections.length; i++) { - const idx = sections.indexOf(shortenableSections[i]!) + for (const s of shortenableSections.slice(0, 3)) { + const idx = sections.indexOf(s) const tip = sectionData[idx]!.shorteningTip - remediationParts.push(`"${shortenableSections[i]!.heading}": ${tip}`) + remediationParts.push(`"${s.heading}": ${tip}`) } + // Section token breakdown for context + const topSections = [...sections].sort((a, b) => b.tokens - a.tokens).slice(0, 5) + const breakdown = topSections.map(s => `"${s.heading}": ${s.tokens}`).join(', ') + findings.push({ rule: 'section-analysis', severity, confidence: 0.8, - description: `${filename}: ${parts.join('. ')}. Total file: ~${Math.ceil(content.length / CHARS_PER_TOKEN).toLocaleString()} tokens across ${sections.length} sections.`, + description: parts.length > 0 + ? `${filename}: ${parts.join('. ')}. Total: ~${totalTokens.toLocaleString()} tokens across ${sections.length} sections (top: ${breakdown})` + : `${filename}: ~${totalTokens.toLocaleString()} tokens across ${sections.length} sections. Token breakdown: ${breakdown}`, location: filename, - remediation: remediationParts.join('\n'), + remediation: remediationParts.length > 0 ? remediationParts.join('\n') : 'No shortening opportunities detected — structure looks efficient.', sections: sectionData, }) } diff --git a/tests/skill-analyzer.test.ts b/tests/skill-analyzer.test.ts index a391f15..ae8280e 100644 --- a/tests/skill-analyzer.test.ts +++ b/tests/skill-analyzer.test.ts @@ -312,7 +312,7 @@ describe('section-analysis rule', () => { // Check redundantWith is populated const codeReview = finding.sections!.find(s => s.heading === 'Code Review') expect(codeReview!.redundantWith).toBeDefined() - expect(codeReview!.redundantWith).toContain('Security Review') + expect(codeReview!.redundantWith!.some(r => r.includes('Security Review'))).toBe(true) }) it('suggests shortening tips for sections with repeated lines', () => { From 44bf20719037571677fe24cbfdd52e1a5e2e9c88 Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Thu, 16 Apr 2026 15:23:11 +0200 Subject: [PATCH 6/9] feat(skill-analysis): add content-level shortening suggestions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New content-aware tips beyond structural suggestions: - Concept repetition: detects key phrases restated 3+ times across a section — suggests stating the rule once and referencing it - Restatement detection: flags "remember that", "note that", "in other words", "this is important because" — AI follows instructions without persuasion, so justifications are token waste - Bad/good example pairs: when a section shows both wrong and right patterns extensively, suggests stating the rule as a negative constraint instead of doubling content with examples - Specific file path overenumeration: flags sections referencing 6+ file paths — suggests glob patterns or naming conventions - Verbose justification: flags "this is important because" / "the reason for this is" — remove persuasion, keep the rule - Large directory trees: flags ASCII trees over 10 lines — suggest keeping only top-level and linking to reference Tested against @tank/bdd-e2e-testing and @tank/idd-bdd-pack skills. Co-Authored-By: Claude --- src/skill-rules/section-analysis.ts | 72 ++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/src/skill-rules/section-analysis.ts b/src/skill-rules/section-analysis.ts index 6a92ab3..017b535 100644 --- a/src/skill-rules/section-analysis.ts +++ b/src/skill-rules/section-analysis.ts @@ -192,19 +192,77 @@ function getShorteningTip(section: ParsedSection): string | undefined { tips.push(`${listItems.length} list items — consider grouping into categories or moving details to separate files`) } - // Check for very long paragraphs - const paragraphs = section.content.split(/\n\s*\n/).filter(p => p.trim().length > 0) - const longParagraphs = paragraphs.filter(p => p.length > 1500) - if (longParagraphs.length > 0) { - tips.push(`${longParagraphs.length} long paragraph(s) over 375 tokens — consider breaking into bullet points`) - } - // Check for table-heavy content const tableRows = lines.filter(l => /^\|/.test(l.trim())) if (tableRows.length > 15) { tips.push(`${tableRows.length} table rows — consider moving detailed tables to reference files and keeping only a summary`) } + // ── Content-level suggestions ────────────────────────────────────────── + + const normalizedContent = normalizeContent(section.content) + + // Detect concept repetition: same key phrase appearing many times + const PHRASE_MIN_LENGTH = 4 + const words = normalizedContent.split(' ').filter(w => w.length > 2) + const phraseCounts = new Map() + for (let i = 0; i < words.length - PHRASE_MIN_LENGTH; i++) { + const phrase = words.slice(i, i + PHRASE_MIN_LENGTH).join(' ') + phraseCounts.set(phrase, (phraseCounts.get(phrase) ?? 0) + 1) + } + const repeatedPhrases = [...phraseCounts.entries()].filter(([, c]) => c >= 3) + if (repeatedPhrases.length > 0) { + const topPhrase = repeatedPhrases.sort((a, b) => b[1]! - a[1]!)[0]! + tips.push(`Concept "${topPhrase[0]!.slice(0, 50)}..." is restated ${topPhrase[1]!} times — state the rule once, reference it elsewhere`) + } + + // Detect restatement patterns: "X is Y" followed later by "X means Y" or "remember: X is Y" + const restatementPatterns = [ + /remember.{0,10}(that|:)\s/gi, + /note.{0,5}(that|:)\s/gi, + /this (is|means)\s/gi, + /in other words/gi, + /that (is to say|means)\s/gi, + /this is (non-)?negotiable/gi, + ] + let restatementCount = 0 + for (const pattern of restatementPatterns) { + const matches = section.content.match(pattern) + if (matches) restatementCount += matches.length + } + if (restatementCount >= 2) { + tips.push(`${restatementCount} restatement(s) detected ("remember that", "note that", "in other words") — the AI already understood the first time; remove re-explanations`) + } + + // Detect anti-pattern examples that double the content + // Pattern: shows bad example then immediately shows the fix + const badGoodPairs = section.content.match(/\bad\b|\bwrong\b|\bdon't\b|\bavoid\b|\bnever\b/gi) ?? [] + const goodExamples = section.content.match(/\bgood\b|\bcorrect\b|\bfixed\b|\bfixed\b|\binstead\b|\brather\b/gi) ?? [] + if (badGoodPairs.length >= 3 && goodExamples.length >= 3) { + tips.push(`Shows ${badGoodPairs.length} bad examples alongside good ones — consider showing only the correct pattern and stating the rule as a negative constraint (e.g. "never do X")`) + } + + // Detect overly specific enumeration that could be a rule + // e.g. listing 8 specific file paths when a glob pattern or rule would suffice + const specificPaths = section.content.match(/[\w/-]+\.\w{2,4}/g) ?? [] + const uniquePaths = new Set(specificPaths.map(p => p.toLowerCase())) + if (uniquePaths.size > 6) { + tips.push(`References ${uniquePaths.size} specific file paths — consider replacing some with a glob pattern or naming convention rule`) + } + + // Detect verbose justification that the AI doesn't need + // "This is important because..." / "The reason for this is..." / "Why? Because..." + const justificationPatterns = section.content.match(/(this is important|the reason for|why\? because|this matters because|this is critical because)/gi) ?? [] + if (justificationPatterns.length >= 2) { + tips.push(`${justificationPatterns.length} justification(s) ("this is important because", "the reason for") — the AI follows instructions without needing persuasion; state the rule directly`) + } + + // Detect duplicate directory trees / ASCII diagrams + const treeLineCount = lines.filter(l => /^[├└│─┤┬┴┼┌┐└┘││ ]+[a-z]/.test(l) || /^\s+[a-z_]+\/\s*$/i.test(l)).length + if (treeLineCount > 10) { + tips.push(`Directory tree is ${treeLineCount} lines — consider keeping only the top-level structure and linking to a reference file for the full tree`) + } + return tips.length > 0 ? tips.join('. ') + '.' : undefined } From 259df5c5431aed2fb306d0d3557d33b44604d773 Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Thu, 16 Apr 2026 20:34:58 +0200 Subject: [PATCH 7/9] feat(skill-analysis): add cost, grade, one-liner, comparison to output Enriched the analysis report with human-facing metrics: - Cost per use: real money estimates (~$0.18 Sonnet / ~$0.91 Opus for avg skill) based on 80/20 input/output split pricing - Grade: single letter A/B/C/D instead of raw 0-100 score - One-liner: plain English summary at the top ("Slightly above average size. Works fine, could be leaner.") - Comparison: token count vs average (~20k tokens) - Size bar: visual indicator with avg marker - "What this means": 2-sentence blurb about cost/speed tradeoff - Human-readable terminal output by default; JSON only with --json Co-Authored-By: Claude --- src/analyze.ts | 8 ++- src/skill-analyzer.ts | 163 ++++++++++++++++++++++++++++++++++++++++-- src/types.ts | 23 +++++- 3 files changed, 187 insertions(+), 7 deletions(-) diff --git a/src/analyze.ts b/src/analyze.ts index bbf2954..3cf8674 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -20,7 +20,7 @@ import { writeFileSync, mkdirSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; import type { AnalysisOutput, CliOptions, DetectorResult, SessionData } from './types.js'; -import { analyzeSkill } from './skill-analyzer.js'; +import { analyzeSkill, renderSkillReport } from './skill-analyzer.js'; import { discoverFiles, logDiscoverySummary } from './discovery.js'; import { extractHumanReadableBlock, renderTerminalBlock } from './recommendation.js'; import { parseSessionFiles } from './parser.js'; @@ -526,7 +526,11 @@ async function main(): Promise { // ── Skill analysis mode ── if (options.analyzeSkill) { const result = analyzeSkill(options.analyzeSkill); - console.log(JSON.stringify(result, null, options.verbose ? 2 : 0)); + if (options.json) { + console.log(JSON.stringify(result, null, options.verbose ? 2 : 0)); + } else { + console.log(renderSkillReport(result)); + } return; } diff --git a/src/skill-analyzer.ts b/src/skill-analyzer.ts index 548e7aa..6b6a8cd 100644 --- a/src/skill-analyzer.ts +++ b/src/skill-analyzer.ts @@ -7,7 +7,7 @@ import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs' import { join, extname, relative } from 'node:path' -import type { SkillAnalysisContext, SkillAnalysisResult, SkillAnalysisSummary, SkillFinding, SkillRule } from './types.js' +import type { SkillAnalysisContext, SkillAnalysisReport, SkillAnalysisSummary, SkillCostEstimate, SkillFinding, SkillGrade, SkillRule } from './types.js' // Import all rules import * as promptSizeRule from './skill-rules/prompt-size.js' @@ -100,8 +100,49 @@ function estimateTokens(files: Map): number { return Math.ceil(totalChars / CHARS_PER_TOKEN) } +// ── Cost estimation ────────────────────────────────────────────────────────── + +// Approximate USD per 1M tokens (as of 2026) +const SONNET_INPUT_PER_M = 3.00 +const SONNET_OUTPUT_PER_M = 15.00 +const OPUS_INPUT_PER_M = 15.00 +const OPUS_OUTPUT_PER_M = 75.00 + +// Assume ~80% input / ~20% output split for skill context loading +const INPUT_RATIO = 0.8 +const OUTPUT_RATIO = 0.2 + +const AVG_SKILL_TOKENS = 20000 + +function estimateCost(tokens: number): SkillCostEstimate { + const tokenMillions = tokens / 1_000_000 + const sonnetCost = tokenMillions * (SONNET_INPUT_PER_M * INPUT_RATIO + SONNET_OUTPUT_PER_M * OUTPUT_RATIO) + const opusCost = tokenMillions * (OPUS_INPUT_PER_M * INPUT_RATIO + OPUS_OUTPUT_PER_M * OUTPUT_RATIO) + + return { + sonnet: formatUsd(sonnetCost), + opus: formatUsd(opusCost), + } +} + +function formatUsd(amount: number): string { + if (amount < 0.001) return '<$0.001' + if (amount < 0.01) return `~$${amount.toFixed(3)}` + if (amount < 0.1) return `~$${amount.toFixed(2)}` + if (amount < 1) return `~$${amount.toFixed(2)}` + return `~$${amount.toFixed(2)}` +} + +// ── Grading ────────────────────────────────────────────────────────────────── + +function calculateGrade(score: number): SkillGrade { + if (score >= 85) return 'A' + if (score >= 65) return 'B' + if (score >= 40) return 'C' + return 'D' +} + function calculateEfficiencyScore(findings: SkillFinding[], totalTokens: number): number { - // Start at 100, deduct points based on findings let score = 100 for (const finding of findings) { @@ -112,13 +153,55 @@ function calculateEfficiencyScore(findings: SkillFinding[], totalTokens: number) score -= deduction } - // Bonus for low total token count if (totalTokens < 1000) score = Math.min(score + 10, 100) return Math.max(0, Math.min(100, score)) } -export function analyzeSkill(dir: string): SkillAnalysisResult { +// ── Plain-English summaries ────────────────────────────────────────────────── + +function generateOneLiner(grade: SkillGrade, tokens: number, findingsCount: number): string { + if (grade === 'A' && tokens < 5000) return 'Lean and efficient. No meaningful improvements needed.' + if (grade === 'A') return 'Well-structured skill with no significant waste.' + if (grade === 'B') { + if (findingsCount <= 2) return 'Slightly above average size. Works fine, could be leaner.' + return 'Good shape overall, with a few areas that could be tightened up.' + } + if (grade === 'C') return 'Carries noticeable token overhead. Several sections could be trimmed or consolidated.' + return 'Bloated — significant token waste. Multiple sections duplicate content or over-explain.' +} + +function generateComparison(tokens: number): string { + const ratio = tokens / AVG_SKILL_TOKENS + if (ratio < 0.3) return `${tokens.toLocaleString()} tokens — much smaller than avg (~${AVG_SKILL_TOKENS.toLocaleString()} tokens)` + if (ratio < 0.7) return `${tokens.toLocaleString()} tokens — below average (avg is ~${AVG_SKILL_TOKENS.toLocaleString()} tokens)` + if (ratio < 1.3) return `${tokens.toLocaleString()} tokens — about average for a skill (~${AVG_SKILL_TOKENS.toLocaleString()} tokens)` + if (ratio < 2.0) return `${tokens.toLocaleString()} tokens — above average (avg is ~${AVG_SKILL_TOKENS.toLocaleString()} tokens)` + return `${tokens.toLocaleString()} tokens — much larger than avg (~${AVG_SKILL_TOKENS.toLocaleString()} tokens)` +} + +function generateWhatThisMeans(tokens: number): string { + const ratio = tokens / AVG_SKILL_TOKENS + if (ratio < 0.5) return 'Low overhead — the AI loads this context quickly and cheaply. No action needed.' + if (ratio < 1.2) return 'Bigger skills cost more per invocation and leave less room for conversation. Smaller skills respond faster and cost less.' + return 'This skill is expensive to load on every turn — every byte competes with your actual conversation for the context window. Trimming it saves real money and improves response speed.' +} + +// ── Size bar ───────────────────────────────────────────────────────────────── + +function sizeBar(tokens: number): string { + const maxTokens = 80000 + const barWidth = 30 + const fill = Math.min(Math.round((tokens / maxTokens) * barWidth), barWidth) + const avgPos = Math.min(Math.round((AVG_SKILL_TOKENS / maxTokens) * barWidth), barWidth - 1) + const bar = '░'.repeat(fill) + '█' + '░'.repeat(Math.max(barWidth - fill - 1, 0)) + const marker = ' '.repeat(avgPos) + '▲' + return `[${bar}]\n [${marker} avg]` +} + +// ── Main export ────────────────────────────────────────────────────────────── + +export function analyzeSkill(dir: string): SkillAnalysisReport { // Validate directory if (!existsSync(dir)) { throw new Error(`Directory not found: ${dir}`) @@ -149,6 +232,7 @@ export function analyzeSkill(dir: string): SkillAnalysisResult { // Calculate summary const estimatedTokens = estimateTokens(files) const efficiencyScore = calculateEfficiencyScore(allFindings, estimatedTokens) + const grade = calculateGrade(efficiencyScore) const summary: SkillAnalysisSummary = { total_findings: allFindings.length, @@ -157,7 +241,78 @@ export function analyzeSkill(dir: string): SkillAnalysisResult { } return { + one_liner: generateOneLiner(grade, estimatedTokens, allFindings.length), + grade, + estimated_tokens: estimatedTokens, + comparison: generateComparison(estimatedTokens), + cost_per_use: estimateCost(estimatedTokens), + what_this_means: generateWhatThisMeans(estimatedTokens), findings: allFindings, summary, } } + +// ── Terminal renderer ──────────────────────────────────────────────────────── + +function gradeColor(grade: SkillGrade): string { + switch (grade) { + case 'A': return '\x1b[32m' // green + case 'B': return '\x1b[33m' // yellow + case 'C': return '\x1b[33m\x1b[1m' // bold yellow + case 'D': return '\x1b[31m\x1b[1m' // bold red + } +} + +function severityIcon(sev: string): string { + switch (sev) { + case 'high': return '\x1b[31m●\x1b[0m' + case 'medium': return '\x1b[33m●\x1b[0m' + case 'low': return '\x1b[36m●\x1b[0m' + default: return '\x1b[2m●\x1b[0m' + } +} + +export function renderSkillReport(report: SkillAnalysisReport): string { + const bold = '\x1b[1m' + const dim = '\x1b[2m' + const reset = '\x1b[0m' + const cyan = '\x1b[36m' + const gc = gradeColor(report.grade) + + const lines: string[] = [] + lines.push('') + lines.push(`${cyan}${bold} SKILL TOKEN ANALYSIS${reset}`) + lines.push(`${dim} ${'─'.repeat(50)}${reset}`) + lines.push('') + lines.push(` ${bold}"${report.one_liner}"${reset}`) + lines.push('') + lines.push(` Grade: ${gc}${bold}${report.grade}${reset} (${report.summary.efficiency_score}/100)`) + lines.push(` Size: ${report.comparison}`) + lines.push('') + lines.push(` ${sizeBar(report.estimated_tokens)}`) + lines.push('') + lines.push(` Cost per use: ${report.cost_per_use.sonnet} (Sonnet) | ${report.cost_per_use.opus} (Opus)`) + lines.push('') + + if (report.findings.length > 0) { + lines.push(` ${bold}Findings (${report.findings.length}):${reset}`) + for (const f of report.findings) { + const icon = severityIcon(f.severity) + lines.push(` ${icon} ${bold}${f.rule}${reset} [${f.severity}]`) + lines.push(` ${dim}${f.description.slice(0, 120)}${f.description.length > 120 ? '...' : ''}${reset}`) + if (f.remediation) { + const firstRemediation = f.remediation.split('\n')[0]! + lines.push(` ${dim}Fix: ${firstRemediation.slice(0, 100)}${firstRemediation.length > 100 ? '...' : ''}${reset}`) + } + lines.push('') + } + } else { + lines.push(` ${dim}No findings. Structure looks clean.${reset}`) + lines.push('') + } + + lines.push(` ${dim}${report.what_this_means}${reset}`) + lines.push('') + + return lines.join('\n') +} diff --git a/src/types.ts b/src/types.ts index 68ca2b0..7368f05 100644 --- a/src/types.ts +++ b/src/types.ts @@ -715,7 +715,28 @@ export interface SkillAnalysisSummary { efficiency_score: number; // 0-100 } -export interface SkillAnalysisResult { +export interface SkillCostEstimate { + /** Cost per invocation on Claude Sonnet (USD) */ + sonnet: string; + /** Cost per invocation on Claude Opus (USD) */ + opus: string; +} + +export type SkillGrade = 'A' | 'B' | 'C' | 'D'; + +export interface SkillAnalysisReport { + /** Plain English one-liner summarizing the skill */ + one_liner: string; + /** Single letter grade */ + grade: SkillGrade; + /** Estimated token count */ + estimated_tokens: number; + /** Comparison to average skill size */ + comparison: string; + /** Cost per invocation in real money */ + cost_per_use: SkillCostEstimate; + /** What this means for the user, 2 sentences max */ + what_this_means: string; findings: SkillFinding[]; summary: SkillAnalysisSummary; } From 00d0fd5d0cd78d928cd37dc0997c13aed43cc7eb Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Thu, 16 Apr 2026 20:41:50 +0200 Subject: [PATCH 8/9] fix(skill-analysis): show full finding text instead of truncating Replaced 120-char/100-char truncation with word-wrapping at 76 chars. All finding descriptions and remediation lines are now fully visible in terminal output, wrapped to fit the display. Co-Authored-By: Claude --- src/skill-analyzer.ts | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/skill-analyzer.ts b/src/skill-analyzer.ts index 6b6a8cd..78b5bfb 100644 --- a/src/skill-analyzer.ts +++ b/src/skill-analyzer.ts @@ -272,6 +272,22 @@ function severityIcon(sev: string): string { } } +function wrapText(text: string, width: number): string[] { + const words = text.split(' ') + const lines: string[] = [] + let current = '' + for (const word of words) { + if (current.length + word.length + 1 > width && current.length > 0) { + lines.push(current) + current = word + } else { + current = current.length === 0 ? word : `${current} ${word}` + } + } + if (current.length > 0) lines.push(current) + return lines +} + export function renderSkillReport(report: SkillAnalysisReport): string { const bold = '\x1b[1m' const dim = '\x1b[2m' @@ -299,10 +315,16 @@ export function renderSkillReport(report: SkillAnalysisReport): string { for (const f of report.findings) { const icon = severityIcon(f.severity) lines.push(` ${icon} ${bold}${f.rule}${reset} [${f.severity}]`) - lines.push(` ${dim}${f.description.slice(0, 120)}${f.description.length > 120 ? '...' : ''}${reset}`) + // Wrap description to 76 chars (terminal width minus indentation) + for (const ln of wrapText(f.description, 76)) { + lines.push(` ${dim}${ln}${reset}`) + } if (f.remediation) { - const firstRemediation = f.remediation.split('\n')[0]! - lines.push(` ${dim}Fix: ${firstRemediation.slice(0, 100)}${firstRemediation.length > 100 ? '...' : ''}${reset}`) + for (const remediationLine of f.remediation.split('\n')) { + for (const ln of wrapText(`Fix: ${remediationLine}`, 76)) { + lines.push(` ${dim}${ln}${reset}`) + } + } } lines.push('') } From efb71627b9c994ea69b8665d60cb2dbec525bccf Mon Sep 17 00:00:00 2001 From: Gal Naor Date: Thu, 16 Apr 2026 21:10:17 +0200 Subject: [PATCH 9/9] docs: rewrite Skill Analysis README section with new output format - Terminal output example with grade, size bar, cost per use - JSON output example with all new fields - Report fields table (one_liner, grade, comparison, cost_per_use, etc.) - Section analysis details: cross-file redundancy, content-level tips table - Cost estimation table with real money examples - Bump terminal example version to v2.3.0 Co-Authored-By: Claude --- README.md | 135 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 424174f..8005fa7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A CLI tool that analyzes your Claude Code session history to identify token waste patterns and provide actionable fixes. Runs locally, no LLM needed. -**v2.3** adds `--analyze-skill` mode for static token analysis of AI agent skill packages — designed for integration with skill registries like [Tank](https://github.com/tankpkg/tank). +**v2.3** adds `--analyze-skill` — analyze any AI agent skill package for token efficiency with cost estimates, letter grades, section-level breakdowns, and actionable shortening tips. Designed for integration with skill registries like [Tank](https://github.com/tankpkg/tank). **The mental model: tokenomics is a coach, not a remote control.** It writes suggestions into your CLAUDE.md where Claude can see them. Claude is smart enough to follow most of them — suggesting `/compact` when context grows, recommending Sonnet for simple tasks, warning when you're overspending. But it cannot switch models, run commands, or force behavior. The user is always in control. @@ -240,7 +240,7 @@ Budget config lives at `~/.claude/tokenomics.json`: | `--inject` | Re-analyze sessions + update CLAUDE.md findings | false | | `--quiet` | Suppress terminal output (used by hooks) | false | | `--verbose` | Show discovery progress and debug info | false | -| `--analyze-skill ` | Analyze skill package for token efficiency (outputs JSON) | - | +| `--analyze-skill ` | Analyze skill package for token efficiency | - | | `--help` | Show help message | - | | `--version` | Show version | - | @@ -267,36 +267,72 @@ Budget config lives at `~/.claude/tokenomics.json`: ## Skill Analysis (`--analyze-skill`) -Analyze an AI agent skill package directory for token efficiency. Useful for skill authors before publishing, and for skill registries (like [Tank](https://github.com/tankpkg/tank)) to include token findings alongside security scan results. +Analyze an AI agent skill package directory for token efficiency. Produces a grade (A/B/C/D), cost per use in real money, per-section token breakdowns, and actionable suggestions for what to cut and why. ```bash -# Analyze a skill package +# Terminal report (default) tokenomics --analyze-skill ./my-skill -# With pretty-printed JSON -tokenomics --analyze-skill ./my-skill --verbose +# JSON output (for scripts, CI, Tank integration) +tokenomics --analyze-skill ./my-skill --json ``` -### Output format +### Terminal output + +``` + SKILL TOKEN ANALYSIS + ────────────────────────────────────────────────── + + "Well-structured skill with no significant waste." + + Grade: A (86/100) + Size: 33,598 tokens — above average (avg is ~20,000 tokens) + + [░░░░░░░░░░░░░█░░░░░░░░░░░░░░░░] + ▲ avg + + Cost per use: ~$0.18 (Sonnet) | ~$0.91 (Opus) + + Findings (3): + ● prompt-size [medium] + Prompt file "SKILL.md" is ~2,388 tokens (9,549 chars). + Consider trimming for faster invocations. + Fix: Consider trimming "SKILL.md" to under 2000 tokens. + Extract verbose examples into separate reference files. + + ● section-analysis [low] + SKILL.md: 2 section(s) overlap each other. + Fix: "web app setup" overlaps with "library setup": + consolidate shared content into one section. + Tip: References 18 specific file paths — consider + replacing with a glob pattern or naming convention rule. +``` + +### JSON output + +With `--json`, the output includes all fields for programmatic consumption: ```json { + "one_liner": "Slightly above average size. Works fine, could be leaner.", + "grade": "B", + "estimated_tokens": 8500, + "comparison": "8,500 tokens — below average (avg is ~20,000 tokens)", + "cost_per_use": { + "sonnet": "~$0.05", + "opus": "~$0.23" + }, + "what_this_means": "Bigger skills cost more per invocation and leave less room for conversation. Smaller skills respond faster and cost less.", "findings": [ { - "rule": "prompt-size", - "severity": "medium", - "confidence": 0.9, - "description": "Prompt file \"SKILL.md\" is ~4,200 tokens (16,800 chars)...", - "location": "SKILL.md", - "remediation": "Trim to under 2000 tokens. Move detailed examples to separate files..." - }, - { - "rule": "tool-overhead", - "severity": "medium", - "confidence": 0.85, - "description": "12 tool definitions found. Each adds ~200-500 tokens of context...", - "location": "manifest/config", - "remediation": "Consider reducing to 8 or fewer tools..." + "rule": "section-analysis", + "severity": "low", + "description": "SKILL.md: 2 section(s) overlap each other...", + "remediation": "\"Setup\" overlaps with \"Config\": consolidate...", + "sections": [ + { "heading": "Setup", "tokens": 420, "redundantWith": ["Config"] }, + { "heading": "Config", "tokens": 380, "redundantWith": ["Setup"] } + ] } ], "summary": { @@ -307,6 +343,18 @@ tokenomics --analyze-skill ./my-skill --verbose } ``` +### Report fields + +| Field | What it means | +|-------|---------------| +| `one_liner` | Plain English summary of the skill's efficiency | +| `grade` | A (lean) / B (good) / C (bloated) / D (wasteful) | +| `estimated_tokens` | Total token count across all skill files | +| `comparison` | How this skill compares to average (~20k tokens) | +| `cost_per_use` | Real money per invocation on Sonnet and Opus | +| `what_this_means` | Why size matters: cost + speed + context window competition | +| `sections[]` | Per-heading token breakdown with redundancy and shortening tips | + ### Analysis rules | Rule | What it checks | Thresholds | @@ -316,14 +364,47 @@ tokenomics --analyze-skill ./my-skill --verbose | `tool-overhead` | Tool definitions in manifest (tank.json, skills.json) | >8 tools = medium, >15 = high | | `large-files` | Files with >500 lines that are expensive to read | >500 lines = low, >1000 = medium | | `redundant-instructions` | Duplicated instruction lines across skill files | >30% duplication = low/medium | +| `section-analysis` | Per-section token breakdown, cross-file redundancy, content tips | See below | + +### Section analysis details + +The `section-analysis` rule parses markdown headings and provides: + +**Token breakdown**: Every section gets a token count, so you can see exactly where the weight is. + +**Cross-file redundancy**: Detects when a section in SKILL.md says the same thing as a section in `references/*.md`. Example: `"The Bridge"` in SKILL.md duplicates `"The Bridge Model"` in `references/orchestration-workflow.md`. + +**Content-level shortening tips** (not just structural): + +| Tip | What it detects | +|-----|----------------| +| Concept repetition | Same key phrase restated 3+ times — state once, reference elsewhere | +| Restatements | "remember that", "note that", "in other words" — AI understood the first time | +| Bad/good example pairs | Shows both wrong and right patterns — use a negative constraint rule instead | +| File path enumeration | 6+ specific paths — replace with glob patterns or naming conventions | +| Verbose justification | "this is important because" — AI follows rules without persuasion | +| Large directory trees | ASCII trees >10 lines — keep top-level, link to reference | +| Repeated lines | Identical lines within a section — dedup or collapse | +| Code block bloat | 2+ code blocks — replace with file references | +| Long lists | 8+ list items — group into categories | +| Table bloat | 15+ table rows — move details to reference files | + +### Cost estimation + +Based on Claude API pricing with an 80/20 input/output token split: -### Efficiency score +| Tokens | Sonnet cost | Opus cost | +|--------|-------------|-----------| +| 5,000 | ~$0.03 | ~$0.15 | +| 20,000 (avg) | ~$0.12 | ~$0.59 | +| 50,000 | ~$0.29 | ~$1.47 | +| 100,000 | ~$0.59 | ~$2.94 | -The `efficiency_score` (0-100) reflects overall token efficiency. It starts at 100 and deducts points per finding: -15 for high, -8 for medium, -3 for low, -1 for info. A bonus +10 is given for skill packages under 1000 total tokens. +Every skill is loaded into context on every turn. A 50k-token skill costs ~$1.47 per turn on Opus — trimming it to 20k saves ~$0.88 per turn. ### Integration with skill registries -The `--analyze-skill` mode is designed to be called as a subprocess from security scanning pipelines. Example from Tank's Python scanner: +Designed to be called as a subprocess from scanning pipelines: ```python import subprocess, json @@ -334,12 +415,12 @@ result = subprocess.run( ) if result.returncode == 0: analysis = json.loads(result.stdout) + print(f"Grade: {analysis['grade']} | Cost: {analysis['cost_per_use']['sonnet']}/use") for finding in analysis["findings"]: - # Convert to your scan Finding model print(f"[{finding['severity']}] {finding['rule']}: {finding['description']}") ``` -Graceful degradation: if tokenomics is not installed, the calling pipeline should skip this stage and continue. +Graceful degradation: if tokenomics is not installed, skip the stage and continue. ## Output Examples @@ -347,7 +428,7 @@ Graceful degradation: if tokenomics is not installed, the calling pipeline shoul ``` TOKENOMICS — Token Intelligence for Claude Code - 60 sessions // 30 day range // v2.2.2 + 60 sessions // 30 day range // v2.3.0 Sessions: 60 Total: 1.2M tokens