diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ce373d7a..d6cf7d19 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,6 +11,16 @@ permissions: contents: read jobs: + skillsaw: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: stbenjam/skillsaw@c325108b6cf94284762948b6a463e75ab5288520 # v0.18.0 + with: + strict: true + test: runs-on: ubuntu-24.04 steps: diff --git a/.skillsaw-baseline.json b/.skillsaw-baseline.json new file mode 100644 index 00000000..67075f26 --- /dev/null +++ b/.skillsaw-baseline.json @@ -0,0 +1,52 @@ +{ + "version": "1", + "generated_by": "skillsaw 0.18.0", + "generated_at": "2026-08-14T00:51:52.809910+00:00", + "violations": [ + { + "fingerprint": "22073290a7e46a35", + "rule_id": "context-budget", + "file_path": "skills/retro-analysis/SKILL.md", + "message": "Estimated 3,434 tokens exceeds skill warn limit of 3,000", + "severity": "warning", + "value": 3434, + "baseline_mode": "ceiling" + }, + { + "fingerprint": "0285e1a7243ebead", + "rule_id": "context-budget", + "file_path": "skills/pr-review/SKILL.md", + "message": "Estimated 15,058 tokens exceeds skill error limit of 6,000", + "severity": "error", + "value": 15058, + "baseline_mode": "ceiling" + }, + { + "fingerprint": "910d4f5f259c6510", + "rule_id": "context-budget", + "file_path": "skills/fix-review/SKILL.md", + "message": "Estimated 3,859 tokens exceeds skill warn limit of 3,000", + "severity": "warning", + "value": 3859, + "baseline_mode": "ceiling" + }, + { + "fingerprint": "43a3bb24cb22288e", + "rule_id": "context-budget", + "file_path": "skills/code-review/SKILL.md", + "message": "Estimated 3,842 tokens exceeds skill warn limit of 3,000", + "severity": "warning", + "value": 3842, + "baseline_mode": "ceiling" + }, + { + "fingerprint": "8aba1d160bb4c413", + "rule_id": "context-budget", + "file_path": "skills/code-implementation/SKILL.md", + "message": "Estimated 9,116 tokens exceeds skill error limit of 6,000", + "severity": "error", + "value": 9116, + "baseline_mode": "ceiling" + } + ] +} diff --git a/.skillsaw.yaml b/.skillsaw.yaml new file mode 100644 index 00000000..91d309b3 --- /dev/null +++ b/.skillsaw.yaml @@ -0,0 +1,338 @@ +# skillsaw configuration +# https://github.com/stbenjam/skillsaw + +version: "0.18.0" + +rules: + + # Plugin must have .claude-plugin/plugin.json + claude-plugin-json-required: + enabled: auto + severity: error + + # plugin.json must be valid JSON with required fields + claude-plugin-json-valid: + enabled: auto + severity: error + recommended-fields: + - description + - version + - author + + # Plugin names should use kebab-case + claude-plugin-naming: + enabled: auto + severity: warning + + # Command files should use kebab-case naming + claude-command-naming: + enabled: true + severity: warning + + # Command files must have valid frontmatter with description + claude-command-frontmatter: + enabled: true + severity: error + + # Command files should have Name, Synopsis, Description, and Implementation sections + claude-command-sections: + enabled: false + severity: warning + + # Command Name section should be 'plugin-name:command-name' + claude-command-name-format: + enabled: false + severity: warning + + # Marketplace.json must be valid JSON with required fields + # Disabled: this repo has no top-level .claude-plugin/marketplace.json, + # so it isn't a distributable Claude Code marketplace. plugins/gopls-lsp/ + # plugin.json sits directly under plugins/gopls-lsp/, not under a + # .claude-plugin/ directory, so it's never recognized as a plugin either + # — confirmed zero plugins are detected here. Re-enable both this and + # claude-marketplace-registration below if this repo ever adds a real + # top-level .claude-plugin/marketplace.json. + claude-marketplace-json-valid: + enabled: false + severity: error + + # Plugins must be registered in marketplace.json + # Disabled: see claude-marketplace-json-valid above. + claude-marketplace-registration: + enabled: false + severity: error + + # Plugin should have a README.md file + claude-plugin-readme: + enabled: auto + severity: warning + + # Agent files must have valid frontmatter with name and description + claude-agent-frontmatter: + enabled: true + severity: error + + # hooks.json must be valid JSON with proper hook configuration structure + hooks-json-valid: + enabled: true + severity: error + + # Flags hook commands that execute scripts from dotfile directories, download-and-execute chains (curl|sh), obfuscation (eval/base64), or perform network requests + hooks-dangerous: + enabled: auto + severity: error + # allowlist: [] + + # All hook commands are prohibited unless explicitly allowlisted; catches new or unexpected hooks added to a project + hooks-prohibited: + enabled: false + severity: error + # allowlist: [] + + # MCP configuration must be valid JSON with proper mcpServers structure + mcp-valid-json: + enabled: true + severity: error + + # Repository should not enable non-allowlisted MCP servers + mcp-prohibited: + enabled: false + severity: error + # allowlist: [] + + # .claude/rules/ files must be markdown with valid optional paths frontmatter + claude-rules-valid: + enabled: auto + severity: error + + # SKILL.md must have valid frontmatter with name and description + agentskill-valid: + enabled: auto + severity: error + # required-fields: [] + # required-metadata: [] + + # Skill name must be lowercase with hyphens and match directory name + agentskill-name: + enabled: auto + severity: error + + # Update stale skill name references after a rename + agentskill-rename-refs: + enabled: auto + severity: warning + + # Skill description should be meaningful and within length limits + agentskill-description: + enabled: auto + severity: warning + + # Skill directories should only contain recognized subdirectories (stricter than spec) + agentskill-structure: + enabled: false + severity: warning + # allowed_dirs: + # - assets + # - evals + # - references + # - scripts + + # Require evals/evals.json for each skill (opt-in) + agentskill-evals-required: + enabled: false + severity: warning + + # Validate evals/evals.json format when present + agentskill-evals: + enabled: auto + severity: warning + + # Validate metadata.openclaw fields against the openclaw spec + openclaw-metadata: + enabled: auto + severity: warning + + # Instruction files (AGENTS.md, CLAUDE.md, GEMINI.md) must be valid and non-empty + instruction-file-valid: + enabled: auto + severity: warning + + # Import references (@path) in AGENTS.md, CLAUDE.md, and GEMINI.md must point to existing files + instruction-imports-valid: + enabled: auto + severity: warning + + # Warn when instruction or config files exceed recommended token limits + context-budget: + enabled: auto + severity: warning + limits: + # agents-md: + # warn: 6000 + # error: 12000 + # claude-md: + # warn: 6000 + # error: 12000 + # gemini-md: + # warn: 6000 + # error: 12000 + # instruction: + # warn: 4000 + # error: 8000 + # skill: + # warn: 3000 + # error: 6000 + # command: + # warn: 2000 + # error: 4000 + # agent: + # warn: 2000 + # error: 4000 + # rule: + # warn: 2000 + # error: 4000 + # skill-description: + # warn: 200 + # error: 500 + # command-description: + # warn: 200 + # error: 500 + + # Detect hedging, vague, and non-actionable language in instruction files + content-weak-language: + enabled: auto + severity: warning + + # Detect tautological instructions that the model already follows by default + content-tautological: + enabled: auto + severity: warning + + # Detect instructions that duplicate .editorconfig, ESLint, Prettier, or tsconfig settings + content-redundant-with-tooling: + enabled: auto + severity: warning + + # Check if instruction count in a file exceeds LLM instruction budget (~150) + content-instruction-budget: + enabled: auto + severity: warning + + # Detect prohibitions without a positive alternative (agent has no path forward) + content-negative-only: + enabled: auto + severity: warning + + # Warn about markdown sections longer than ~500 tokens + content-section-length: + enabled: auto + severity: info + # max-tokens: 500 + + # Detect likely contradictions within instruction files using keyword-pair heuristics + content-contradiction: + enabled: auto + severity: warning + + # Detect instructions that should be automated as hooks instead of prose instructions + content-hook-candidate: + enabled: auto + severity: info + + # Check that instruction files are organized into cognitive chunks with headings + content-cognitive-chunks: + enabled: auto + severity: info + + # Detect potential API keys, tokens, and passwords in instruction files + content-embedded-secrets: + enabled: auto + severity: error + + # Detect banned or deprecated model names, APIs, and custom patterns + content-banned-references: + enabled: auto + severity: warning + # banned: [] + # skip-builtins: false + + # Detect inconsistent terminology across instruction files (e.g., mixing 'directory' and 'folder') + content-inconsistent-terminology: + enabled: auto + severity: info + + # Detect markdown links where the target file does not exist + content-broken-internal-reference: + enabled: auto + severity: warning + + # Detect bare path-like strings not wrapped in markdown link syntax + content-unlinked-internal-reference: + enabled: auto + severity: info + # patterns: + # - ./**/*.* + # - references/**/*.md + + # Detect TODO markers, bracket placeholders, and unfilled template text + content-placeholder-text: + enabled: auto + severity: warning + + # Flags settings keys that execute arbitrary commands (apiKeyHelper, awsAuthRefresh, awsCredentialExport, gcpAuthRefresh, otelHeadersHelper) and dangerous env vars (LD_PRELOAD, NODE_OPTIONS, proxy settings, GIT_SSH_COMMAND, etc.) + claude-settings-dangerous: + enabled: auto + severity: error + # allow_command_exec_keys: [] + # allow_env_vars: [] + + # Validate promptfoo eval YAML config structure and file references + promptfoo-valid: + enabled: auto + severity: error + + # Require specific assertion types in all promptfoo eval tests + promptfoo-assertions: + enabled: false + severity: warning + # required-types: [] + # threshold-constraints: {} + + # Require specific metadata keys on all promptfoo eval tests + promptfoo-metadata: + enabled: false + severity: warning + # required-keys: [] + + # .coderabbit.yaml must be valid YAML + coderabbit-yaml-valid: + enabled: auto + severity: error + + # apm.yml must exist with valid YAML and required fields (name, version, description) + apm-yaml-valid: + enabled: auto + severity: error + + # .apm/ directory must contain skills/ or instructions/ with valid structure + apm-structure-valid: + enabled: auto + severity: warning + +# Load custom rules from these files +custom-rules: [] + +# Exclude patterns (glob format) +# Use exclude: [] to disable all excludes including defaults +exclude: + # - "**/template/**" + # - "**/templates/**" + # - "**/_template/**" + +# Additional markdown files to run content rules on (glob format) +content-paths: [] + +# Not enforced here: CI (lint.yml) and `make lint` both pass an explicit +# strict flag that overrides this config value, so this setting only +# matters if you invoke skillsaw directly without either wrapper. +strict: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4429aae..0a0eb1b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,7 +35,7 @@ This project uses the [Probot DCO app](https://github.com/apps/dco) to enforce s ### Opening a PR -- Stage your changes and fix any lint failures before pushing. +- Stage your changes and fix any lint failures before pushing (`make lint`, or `make lint-fix` to apply automatic fixes). - Keep PRs focused. One problem area or decision per PR is easier to review than a grab-bag. - **If your PR introduces a breaking change**, the PR title must carry the `!` suffix (e.g., `feat(harness)!: require role field`). See [COMMITS.md](COMMITS.md#breaking-changes) for how to identify breaking changes and what to include in the commit body. diff --git a/Makefile b/Makefile index eafe0d8a..dff77535 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,12 @@ .DEFAULT_GOAL := help -.PHONY: help script-build check-bundle script-test test +.PHONY: help script-build check-bundle script-test test lint lint-fix BUNDLE_SRCS := scripts/pre-code.src.sh scripts/post-code.src.sh scripts/post-fix.src.sh scripts/post-prioritize.src.sh scripts/pre-triage.src.sh scripts/post-triage.src.sh BUNDLE_OUTS := $(BUNDLE_SRCS:.src.sh=.sh) LIB_DEPS := $(wildcard scripts/lib/*.lib.sh) +SKILLSAW_VERSION := 0.18.0 + help: @echo "Available targets:" @echo " help - Show this help message" @@ -12,6 +14,14 @@ help: @echo " check-bundle - Verify committed bundles match script-build output" @echo " script-test - Run agent shell script unit tests" @echo " test - Alias for script-test" + @echo " lint - Lint skills/agents/instructions with skillsaw" + @echo " lint-fix - Apply skillsaw's automatic lint fixes" + +lint: + uvx skillsaw@$(SKILLSAW_VERSION) --strict + +lint-fix: + uvx skillsaw@$(SKILLSAW_VERSION) fix define run-timed @start=$$(date +%s); \ diff --git a/README.md b/README.md index 90d8c71d..28292625 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,13 @@ make test This is an alias for `make script-test`, which runs the `scripts/*-test.sh` suites. CI also runs `make check-bundle` and executes `make script-test` twice (source and bundled modes) via `.github/workflows/script-test.yml`. +Lint skills, agents, and instructions with [skillsaw](https://github.com/stbenjam/skillsaw): + +```bash +make lint # check for issues (strict: warnings fail) +make lint-fix # apply automatic fixes +``` + ## Script bundling Harness fetches each runner script as an isolated blob, so post-scripts cannot `source` files from `scripts/lib/` at runtime. Scripts that use shared libraries are maintained as source files and bundled before commit: @@ -88,4 +95,5 @@ This repository is versioned in lockstep with [fullsend](https://github.com/full |------|-----------|---------| | `fullsend.yaml` | fullsend (centrally managed) | Routes GitHub events to agent dispatch workflows | | `release.yml` | This repo | Creates GitHub Releases and moves the `v0` tag on version tag push | +| `lint.yml` | This repo | Runs pre-commit checks, commit-message linting, and skillsaw on PRs and main branch pushes | | `script-test.yml` | This repo | Runs agent shell script tests on PRs and main branch pushes |