diff --git a/.github/scripts/conformance/README.md b/.github/scripts/conformance/README.md new file mode 100644 index 00000000..4b1074ba --- /dev/null +++ b/.github/scripts/conformance/README.md @@ -0,0 +1,47 @@ +# Skill conformance corpus + +`cases.json` holds the test cases for the skill rules of this repository. The rules are in `.github/scripts/validate_skill_publishing.py` and in the "Publishing Rules" section of `CONTRIBUTING.md`. + +The file is a JSON array of case objects, sorted by `id`, with a 2-space indent and a final newline. Each case is one skill folder, `skills//`: + +```json +{ + "id": "name-not-folder-name", + "name": "demo", + "files": { + "SKILL.md": { "text": "---\nname: other\n..." }, + "images/pixel.png": { "base64": "iVBORw0KGgo..." } + }, + "expect": "fail", + "error": "name must equal the folder name" +} +``` + +- `id`: the case name. It is unique in the file. +- `name`: the skill folder name. Many cases use the same folder name. +- `files`: each path is relative to the skill folder. `text` is UTF-8 text, and `base64` is raw bytes. +- `expect`: `pass` or `fail`. +- `error`: for a `fail` case, the exact error message or a part of it. +- `stricter` (optional): the case fails because of a rule that is stricter than skills need to be published. For example, a two-part `metadata.version` or a YAML anchor. + +## Rule for every validator + +Every tool that validates or publishes the skills of this repository must give the `expect` result for every case. A tool can pass a case that has `"stricter": true`. A tool must not pass any other `fail` case. + +`validate_skill_publishing.py` runs every case through its head-skill rules before it checks any skill. This self-check also requires the golden skill content hash and the digest below. When the self-check fails, the script exits with code 2 and checks no skill. To run only the self-check: + +```bash +python3 .github/scripts/validate_skill_publishing.py --self-check-only +``` + +## Corpus digest + +Corpus digest: `df26439cc6d8f5c7eb74250b6ab5ca512ca6f6902464c8affc30b3b82ac7ed2e` + +The digest is the SHA-256 (hex) of the bytes of `cases.json`. The script has the same value in its `CONFORMANCE_SHA256` constant, and the self-check fails when the file has a different digest. Thus a change to `cases.json` must change `CONFORMANCE_SHA256` and this line in the same pull request. After you add or change a case, print the new digest: + +```bash +sha256sum .github/scripts/conformance/cases.json +``` + +On macOS, use `shasum -a 256` instead of `sha256sum`. diff --git a/.github/scripts/conformance/cases.json b/.github/scripts/conformance/cases.json new file mode 100644 index 00000000..155aa2fe --- /dev/null +++ b/.github/scripts/conformance/cases.json @@ -0,0 +1,1557 @@ +[ + { + "id": "anchor-and-alias", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n author: &who someone\n owner: *who\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses an anchor, which skills in this repository must not use", + "stricter": true + }, + { + "id": "binary-file-allowed", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "images/pixel.png": { + "base64": "iVBORw0KGgoA/w==" + } + }, + "expect": "pass" + }, + { + "id": "block-scalar-indicator-not-indented", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription:\n>\n Use this skill.\nmetadata:\n version: \"1.0.0\"\n---\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter is not valid YAML" + }, + { + "id": "block-scalar-indicator-on-own-line", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription:\n >\n Use this skill.\nmetadata:\n version: \"1.0.0\"\n---\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter is not valid YAML", + "stricter": true + }, + { + "id": "block-scalar-keep-chomping", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: |+\n Use this skill.\n\nmetadata:\n version: \"1.0.0\"\n---\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a keep (+) chomping indicator", + "stricter": true + }, + { + "id": "byte-order-mark", + "name": "demo", + "files": { + "SKILL.md": { + "base64": "77u/LS0tCm5hbWU6IGRlbW8KZGVzY3JpcHRpb246IFVzZSB0aGlzIHNraWxsIHdoZW4gYSB0ZXN0IG5lZWRzIGEgdmFsaWQgc2tpbGwuCm1ldGFkYXRhOgogIHZlcnNpb246ICIxLjAuMCIKLS0tCgojIERlbW8K" + } + }, + "expect": "pass" + }, + { + "id": "comment-without-space", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"#bump\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "Comments must be separated from other tokens by white space" + }, + { + "id": "crlf-line-ends", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\r\nname: demo\r\ndescription: Use this skill when a test needs a valid skill.\r\nmetadata:\r\n version: \"1.0.0\"\r\n---\r\n\r\n# Demo\r\n" + } + }, + "expect": "pass" + }, + { + "id": "deprecated-quoted-true", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n deprecated: \"true\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "metadata.deprecated must be a boolean" + }, + { + "id": "deprecated-true", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n deprecated: true\n---\n\n# Demo\n" + } + }, + "expect": "pass" + }, + { + "id": "description-1024-characters", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "pass" + }, + { + "id": "description-1025-characters", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "description must be a string of 1 to 1024 characters" + }, + { + "id": "description-boolean", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: true\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "description must be a string of 1 to 1024 characters" + }, + { + "id": "description-folded-1023-last-key", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\nmetadata:\n version: \"1.0.0\"\ndescription: >\n ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\n---\n" + } + }, + "expect": "pass" + }, + { + "id": "description-folded-1024-last-key", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\nmetadata:\n version: \"1.0.0\"\ndescription: >\n dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\n---\n" + } + }, + "expect": "fail", + "error": "description must be a string of 1 to 1024 characters" + }, + { + "id": "description-missing", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\nmetadata:\n version: \"1.0.0\"\n---\n" + } + }, + "expect": "fail", + "error": "description must be a string of 1 to 1024 characters" + }, + { + "id": "document-end-marker", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\n...\ndescription: d\n---\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a document marker" + }, + { + "id": "duplicate-key", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n version: \"1.0.1\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "Map keys must be unique" + }, + { + "id": "empty-frontmatter", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\n\n---\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter is not an object" + }, + { + "id": "excluded-files-not-published", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "CHANGELOG.md": { + "text": "# Changelog\n" + }, + ".skilleval.yaml": { + "text": "a: 1\n" + }, + "evals/bad name.json": { + "text": "{}\n" + }, + "evals/r000.json": { + "text": "{}\n" + }, + "evals/r001.json": { + "text": "{}\n" + }, + "evals/r002.json": { + "text": "{}\n" + }, + "evals/r003.json": { + "text": "{}\n" + }, + "evals/r004.json": { + "text": "{}\n" + }, + "evals/r005.json": { + "text": "{}\n" + }, + "evals/r006.json": { + "text": "{}\n" + }, + "evals/r007.json": { + "text": "{}\n" + }, + "evals/r008.json": { + "text": "{}\n" + }, + "evals/r009.json": { + "text": "{}\n" + }, + "evals/r010.json": { + "text": "{}\n" + }, + "evals/r011.json": { + "text": "{}\n" + }, + "evals/r012.json": { + "text": "{}\n" + }, + "evals/r013.json": { + "text": "{}\n" + }, + "evals/r014.json": { + "text": "{}\n" + }, + "evals/r015.json": { + "text": "{}\n" + }, + "evals/r016.json": { + "text": "{}\n" + }, + "evals/r017.json": { + "text": "{}\n" + }, + "evals/r018.json": { + "text": "{}\n" + }, + "evals/r019.json": { + "text": "{}\n" + }, + "evals/r020.json": { + "text": "{}\n" + }, + "evals/r021.json": { + "text": "{}\n" + }, + "evals/r022.json": { + "text": "{}\n" + }, + "evals/r023.json": { + "text": "{}\n" + }, + "evals/r024.json": { + "text": "{}\n" + }, + "evals/r025.json": { + "text": "{}\n" + }, + "evals/r026.json": { + "text": "{}\n" + }, + "evals/r027.json": { + "text": "{}\n" + }, + "evals/r028.json": { + "text": "{}\n" + }, + "evals/r029.json": { + "text": "{}\n" + }, + "evals/r030.json": { + "text": "{}\n" + }, + "evals/r031.json": { + "text": "{}\n" + }, + "evals/r032.json": { + "text": "{}\n" + }, + "evals/r033.json": { + "text": "{}\n" + }, + "evals/r034.json": { + "text": "{}\n" + }, + "evals/r035.json": { + "text": "{}\n" + }, + "evals/r036.json": { + "text": "{}\n" + }, + "evals/r037.json": { + "text": "{}\n" + }, + "evals/r038.json": { + "text": "{}\n" + }, + "evals/r039.json": { + "text": "{}\n" + }, + "evals/r040.json": { + "text": "{}\n" + }, + "evals/r041.json": { + "text": "{}\n" + }, + "evals/r042.json": { + "text": "{}\n" + }, + "evals/r043.json": { + "text": "{}\n" + }, + "evals/r044.json": { + "text": "{}\n" + }, + "evals/r045.json": { + "text": "{}\n" + }, + "evals/r046.json": { + "text": "{}\n" + }, + "evals/r047.json": { + "text": "{}\n" + }, + "evals/r048.json": { + "text": "{}\n" + }, + "evals/r049.json": { + "text": "{}\n" + }, + "evals/r050.json": { + "text": "{}\n" + }, + "evals/r051.json": { + "text": "{}\n" + }, + "evals/r052.json": { + "text": "{}\n" + }, + "evals/r053.json": { + "text": "{}\n" + }, + "evals/r054.json": { + "text": "{}\n" + }, + "evals/r055.json": { + "text": "{}\n" + }, + "evals/r056.json": { + "text": "{}\n" + }, + "evals/r057.json": { + "text": "{}\n" + }, + "evals/r058.json": { + "text": "{}\n" + }, + "evals/r059.json": { + "text": "{}\n" + }, + "evals/r060.json": { + "text": "{}\n" + }, + "evals/r061.json": { + "text": "{}\n" + }, + "evals/r062.json": { + "text": "{}\n" + }, + "evals/r063.json": { + "text": "{}\n" + }, + "evals/r064.json": { + "text": "{}\n" + }, + "evals/r065.json": { + "text": "{}\n" + }, + "evals/r066.json": { + "text": "{}\n" + }, + "evals/r067.json": { + "text": "{}\n" + }, + "evals/r068.json": { + "text": "{}\n" + }, + "evals/r069.json": { + "text": "{}\n" + }, + "evals/r070.json": { + "text": "{}\n" + }, + "evals/r071.json": { + "text": "{}\n" + }, + "evals/r072.json": { + "text": "{}\n" + }, + "evals/r073.json": { + "text": "{}\n" + }, + "evals/r074.json": { + "text": "{}\n" + }, + "evals/r075.json": { + "text": "{}\n" + }, + "evals/r076.json": { + "text": "{}\n" + }, + "evals/r077.json": { + "text": "{}\n" + }, + "evals/r078.json": { + "text": "{}\n" + }, + "evals/r079.json": { + "text": "{}\n" + }, + "evals/r080.json": { + "text": "{}\n" + }, + "evals/r081.json": { + "text": "{}\n" + }, + "evals/r082.json": { + "text": "{}\n" + }, + "evals/r083.json": { + "text": "{}\n" + }, + "evals/r084.json": { + "text": "{}\n" + }, + "evals/r085.json": { + "text": "{}\n" + }, + "evals/r086.json": { + "text": "{}\n" + }, + "evals/r087.json": { + "text": "{}\n" + }, + "evals/r088.json": { + "text": "{}\n" + }, + "evals/r089.json": { + "text": "{}\n" + }, + "evals/r090.json": { + "text": "{}\n" + }, + "evals/r091.json": { + "text": "{}\n" + }, + "evals/r092.json": { + "text": "{}\n" + }, + "evals/r093.json": { + "text": "{}\n" + }, + "evals/r094.json": { + "text": "{}\n" + }, + "evals/r095.json": { + "text": "{}\n" + }, + "evals/r096.json": { + "text": "{}\n" + }, + "evals/r097.json": { + "text": "{}\n" + }, + "evals/r098.json": { + "text": "{}\n" + }, + "evals/r099.json": { + "text": "{}\n" + }, + "evals/r100.json": { + "text": "{}\n" + }, + "evals/r101.json": { + "text": "{}\n" + }, + "evals/r102.json": { + "text": "{}\n" + }, + "evals/r103.json": { + "text": "{}\n" + }, + "evals/r104.json": { + "text": "{}\n" + }, + "evals/r105.json": { + "text": "{}\n" + }, + "evals/r106.json": { + "text": "{}\n" + }, + "evals/r107.json": { + "text": "{}\n" + }, + "evals/r108.json": { + "text": "{}\n" + }, + "evals/r109.json": { + "text": "{}\n" + }, + "evals/r110.json": { + "text": "{}\n" + }, + "evals/r111.json": { + "text": "{}\n" + }, + "evals/r112.json": { + "text": "{}\n" + }, + "evals/r113.json": { + "text": "{}\n" + }, + "evals/r114.json": { + "text": "{}\n" + }, + "evals/r115.json": { + "text": "{}\n" + }, + "evals/r116.json": { + "text": "{}\n" + }, + "evals/r117.json": { + "text": "{}\n" + }, + "evals/r118.json": { + "text": "{}\n" + }, + "evals/r119.json": { + "text": "{}\n" + } + }, + "expect": "pass" + }, + { + "id": "explicit-tag", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n author: !!str someone\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses an explicit tag, which skills in this repository must not use", + "stricter": true + }, + { + "id": "files-100", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "references/r00.md": { + "text": "0\n" + }, + "references/r01.md": { + "text": "1\n" + }, + "references/r02.md": { + "text": "2\n" + }, + "references/r03.md": { + "text": "3\n" + }, + "references/r04.md": { + "text": "4\n" + }, + "references/r05.md": { + "text": "5\n" + }, + "references/r06.md": { + "text": "6\n" + }, + "references/r07.md": { + "text": "7\n" + }, + "references/r08.md": { + "text": "8\n" + }, + "references/r09.md": { + "text": "9\n" + }, + "references/r10.md": { + "text": "10\n" + }, + "references/r11.md": { + "text": "11\n" + }, + "references/r12.md": { + "text": "12\n" + }, + "references/r13.md": { + "text": "13\n" + }, + "references/r14.md": { + "text": "14\n" + }, + "references/r15.md": { + "text": "15\n" + }, + "references/r16.md": { + "text": "16\n" + }, + "references/r17.md": { + "text": "17\n" + }, + "references/r18.md": { + "text": "18\n" + }, + "references/r19.md": { + "text": "19\n" + }, + "references/r20.md": { + "text": "20\n" + }, + "references/r21.md": { + "text": "21\n" + }, + "references/r22.md": { + "text": "22\n" + }, + "references/r23.md": { + "text": "23\n" + }, + "references/r24.md": { + "text": "24\n" + }, + "references/r25.md": { + "text": "25\n" + }, + "references/r26.md": { + "text": "26\n" + }, + "references/r27.md": { + "text": "27\n" + }, + "references/r28.md": { + "text": "28\n" + }, + "references/r29.md": { + "text": "29\n" + }, + "references/r30.md": { + "text": "30\n" + }, + "references/r31.md": { + "text": "31\n" + }, + "references/r32.md": { + "text": "32\n" + }, + "references/r33.md": { + "text": "33\n" + }, + "references/r34.md": { + "text": "34\n" + }, + "references/r35.md": { + "text": "35\n" + }, + "references/r36.md": { + "text": "36\n" + }, + "references/r37.md": { + "text": "37\n" + }, + "references/r38.md": { + "text": "38\n" + }, + "references/r39.md": { + "text": "39\n" + }, + "references/r40.md": { + "text": "40\n" + }, + "references/r41.md": { + "text": "41\n" + }, + "references/r42.md": { + "text": "42\n" + }, + "references/r43.md": { + "text": "43\n" + }, + "references/r44.md": { + "text": "44\n" + }, + "references/r45.md": { + "text": "45\n" + }, + "references/r46.md": { + "text": "46\n" + }, + "references/r47.md": { + "text": "47\n" + }, + "references/r48.md": { + "text": "48\n" + }, + "references/r49.md": { + "text": "49\n" + }, + "references/r50.md": { + "text": "50\n" + }, + "references/r51.md": { + "text": "51\n" + }, + "references/r52.md": { + "text": "52\n" + }, + "references/r53.md": { + "text": "53\n" + }, + "references/r54.md": { + "text": "54\n" + }, + "references/r55.md": { + "text": "55\n" + }, + "references/r56.md": { + "text": "56\n" + }, + "references/r57.md": { + "text": "57\n" + }, + "references/r58.md": { + "text": "58\n" + }, + "references/r59.md": { + "text": "59\n" + }, + "references/r60.md": { + "text": "60\n" + }, + "references/r61.md": { + "text": "61\n" + }, + "references/r62.md": { + "text": "62\n" + }, + "references/r63.md": { + "text": "63\n" + }, + "references/r64.md": { + "text": "64\n" + }, + "references/r65.md": { + "text": "65\n" + }, + "references/r66.md": { + "text": "66\n" + }, + "references/r67.md": { + "text": "67\n" + }, + "references/r68.md": { + "text": "68\n" + }, + "references/r69.md": { + "text": "69\n" + }, + "references/r70.md": { + "text": "70\n" + }, + "references/r71.md": { + "text": "71\n" + }, + "references/r72.md": { + "text": "72\n" + }, + "references/r73.md": { + "text": "73\n" + }, + "references/r74.md": { + "text": "74\n" + }, + "references/r75.md": { + "text": "75\n" + }, + "references/r76.md": { + "text": "76\n" + }, + "references/r77.md": { + "text": "77\n" + }, + "references/r78.md": { + "text": "78\n" + }, + "references/r79.md": { + "text": "79\n" + }, + "references/r80.md": { + "text": "80\n" + }, + "references/r81.md": { + "text": "81\n" + }, + "references/r82.md": { + "text": "82\n" + }, + "references/r83.md": { + "text": "83\n" + }, + "references/r84.md": { + "text": "84\n" + }, + "references/r85.md": { + "text": "85\n" + }, + "references/r86.md": { + "text": "86\n" + }, + "references/r87.md": { + "text": "87\n" + }, + "references/r88.md": { + "text": "88\n" + }, + "references/r89.md": { + "text": "89\n" + }, + "references/r90.md": { + "text": "90\n" + }, + "references/r91.md": { + "text": "91\n" + }, + "references/r92.md": { + "text": "92\n" + }, + "references/r93.md": { + "text": "93\n" + }, + "references/r94.md": { + "text": "94\n" + }, + "references/r95.md": { + "text": "95\n" + }, + "references/r96.md": { + "text": "96\n" + }, + "references/r97.md": { + "text": "97\n" + }, + "references/r98.md": { + "text": "98\n" + } + }, + "expect": "pass" + }, + { + "id": "files-101", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "references/r000.md": { + "text": "0\n" + }, + "references/r001.md": { + "text": "1\n" + }, + "references/r002.md": { + "text": "2\n" + }, + "references/r003.md": { + "text": "3\n" + }, + "references/r004.md": { + "text": "4\n" + }, + "references/r005.md": { + "text": "5\n" + }, + "references/r006.md": { + "text": "6\n" + }, + "references/r007.md": { + "text": "7\n" + }, + "references/r008.md": { + "text": "8\n" + }, + "references/r009.md": { + "text": "9\n" + }, + "references/r010.md": { + "text": "10\n" + }, + "references/r011.md": { + "text": "11\n" + }, + "references/r012.md": { + "text": "12\n" + }, + "references/r013.md": { + "text": "13\n" + }, + "references/r014.md": { + "text": "14\n" + }, + "references/r015.md": { + "text": "15\n" + }, + "references/r016.md": { + "text": "16\n" + }, + "references/r017.md": { + "text": "17\n" + }, + "references/r018.md": { + "text": "18\n" + }, + "references/r019.md": { + "text": "19\n" + }, + "references/r020.md": { + "text": "20\n" + }, + "references/r021.md": { + "text": "21\n" + }, + "references/r022.md": { + "text": "22\n" + }, + "references/r023.md": { + "text": "23\n" + }, + "references/r024.md": { + "text": "24\n" + }, + "references/r025.md": { + "text": "25\n" + }, + "references/r026.md": { + "text": "26\n" + }, + "references/r027.md": { + "text": "27\n" + }, + "references/r028.md": { + "text": "28\n" + }, + "references/r029.md": { + "text": "29\n" + }, + "references/r030.md": { + "text": "30\n" + }, + "references/r031.md": { + "text": "31\n" + }, + "references/r032.md": { + "text": "32\n" + }, + "references/r033.md": { + "text": "33\n" + }, + "references/r034.md": { + "text": "34\n" + }, + "references/r035.md": { + "text": "35\n" + }, + "references/r036.md": { + "text": "36\n" + }, + "references/r037.md": { + "text": "37\n" + }, + "references/r038.md": { + "text": "38\n" + }, + "references/r039.md": { + "text": "39\n" + }, + "references/r040.md": { + "text": "40\n" + }, + "references/r041.md": { + "text": "41\n" + }, + "references/r042.md": { + "text": "42\n" + }, + "references/r043.md": { + "text": "43\n" + }, + "references/r044.md": { + "text": "44\n" + }, + "references/r045.md": { + "text": "45\n" + }, + "references/r046.md": { + "text": "46\n" + }, + "references/r047.md": { + "text": "47\n" + }, + "references/r048.md": { + "text": "48\n" + }, + "references/r049.md": { + "text": "49\n" + }, + "references/r050.md": { + "text": "50\n" + }, + "references/r051.md": { + "text": "51\n" + }, + "references/r052.md": { + "text": "52\n" + }, + "references/r053.md": { + "text": "53\n" + }, + "references/r054.md": { + "text": "54\n" + }, + "references/r055.md": { + "text": "55\n" + }, + "references/r056.md": { + "text": "56\n" + }, + "references/r057.md": { + "text": "57\n" + }, + "references/r058.md": { + "text": "58\n" + }, + "references/r059.md": { + "text": "59\n" + }, + "references/r060.md": { + "text": "60\n" + }, + "references/r061.md": { + "text": "61\n" + }, + "references/r062.md": { + "text": "62\n" + }, + "references/r063.md": { + "text": "63\n" + }, + "references/r064.md": { + "text": "64\n" + }, + "references/r065.md": { + "text": "65\n" + }, + "references/r066.md": { + "text": "66\n" + }, + "references/r067.md": { + "text": "67\n" + }, + "references/r068.md": { + "text": "68\n" + }, + "references/r069.md": { + "text": "69\n" + }, + "references/r070.md": { + "text": "70\n" + }, + "references/r071.md": { + "text": "71\n" + }, + "references/r072.md": { + "text": "72\n" + }, + "references/r073.md": { + "text": "73\n" + }, + "references/r074.md": { + "text": "74\n" + }, + "references/r075.md": { + "text": "75\n" + }, + "references/r076.md": { + "text": "76\n" + }, + "references/r077.md": { + "text": "77\n" + }, + "references/r078.md": { + "text": "78\n" + }, + "references/r079.md": { + "text": "79\n" + }, + "references/r080.md": { + "text": "80\n" + }, + "references/r081.md": { + "text": "81\n" + }, + "references/r082.md": { + "text": "82\n" + }, + "references/r083.md": { + "text": "83\n" + }, + "references/r084.md": { + "text": "84\n" + }, + "references/r085.md": { + "text": "85\n" + }, + "references/r086.md": { + "text": "86\n" + }, + "references/r087.md": { + "text": "87\n" + }, + "references/r088.md": { + "text": "88\n" + }, + "references/r089.md": { + "text": "89\n" + }, + "references/r090.md": { + "text": "90\n" + }, + "references/r091.md": { + "text": "91\n" + }, + "references/r092.md": { + "text": "92\n" + }, + "references/r093.md": { + "text": "93\n" + }, + "references/r094.md": { + "text": "94\n" + }, + "references/r095.md": { + "text": "95\n" + }, + "references/r096.md": { + "text": "96\n" + }, + "references/r097.md": { + "text": "97\n" + }, + "references/r098.md": { + "text": "98\n" + }, + "references/r099.md": { + "text": "99\n" + } + }, + "expect": "fail", + "error": "skill has 101 files; the maximum is 100" + }, + { + "id": "flow-collection-not-indented", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n tags: [a,\nb]\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter is not valid YAML" + }, + { + "id": "frontmatter-is-a-list", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\n- name: demo\n---\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter is not an object" + }, + { + "id": "invalid-utf8", + "name": "demo", + "files": { + "SKILL.md": { + "base64": "LS0tCm5hbWU6IGRlbW8KZGVzY3JpcHRpb246IFVzZSB0aGlzIHNraWxsIHdoZW4gYSB0ZXN0IG5lZWRzIGEgdmFsaWQgc2tpbGwuCm1ldGFkYXRhOgogIHZlcnNpb246ICIxLjAuMCIKLS0tCgojIERlbW8K//4K" + } + }, + "expect": "fail", + "error": "SKILL.md is not valid UTF-8" + }, + { + "id": "line-break-lone-carriage-return", + "name": "demo", + "files": { + "SKILL.md": { + "base64": "LS0tCm5hbWU6IGRlbW8KZGVzY3JpcHRpb246IFVzZSB0aGlzIHNraWxsIHdoZW4gYSB0ZXN0IG5lZWRzIGEgdmFsaWQgc2tpbGwuCm1ldGFkYXRhOgogIHZlcnNpb246ICIxLjAuMCIKICBhdXRob3I6IHgNICBvd25lcjogeQotLS0KCiMgRGVtbwo=" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a line break character" + }, + { + "id": "line-break-next-line-before-key", + "name": "demo", + "files": { + "SKILL.md": { + "base64": "LS0tCm5hbWU6IGRlbW8KZGVzY3JpcHRpb246IGTChW1ldGFkYXRhOiB7dmVyc2lvbjogIjEuMC4wIn0KLS0tCg==" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a line break character" + }, + { + "id": "line-break-unicode-separator", + "name": "demo", + "files": { + "SKILL.md": { + "base64": "LS0tCm5hbWU6IGRlbW8KZGVzY3JpcHRpb246IEZpcnN0IGxpbmXigKhzZWNvbmQgbGluZQptZXRhZGF0YToKICB2ZXJzaW9uOiAiMS4wLjAiCi0tLQoKIyBEZW1vCg==" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a line break character", + "stricter": true + }, + { + "id": "map-key-not-a-scalar", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n ? [a, b]\n : c\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a map key that is not a scalar" + }, + { + "id": "metadata-not-a-map", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: d\nmetadata: 1.0.0\n---\n" + } + }, + "expect": "fail", + "error": "metadata.version must be a string" + }, + { + "id": "name-64-characters", + "name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "files": { + "SKILL.md": { + "text": "---\nname: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "pass" + }, + { + "id": "name-65-characters", + "name": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "files": { + "SKILL.md": { + "text": "---\nname: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "name must be a string of 1 to 64 characters" + }, + { + "id": "name-not-a-slug", + "name": "Bad_Name", + "files": { + "SKILL.md": { + "text": "---\nname: Bad_Name\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "name must match ^[a-z0-9]+(-[a-z0-9]+)*$" + }, + { + "id": "name-not-a-string", + "name": "123", + "files": { + "SKILL.md": { + "text": "---\nname: 123\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "name must be a string of 1 to 64 characters" + }, + { + "id": "name-not-folder-name", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: other\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "name must equal the folder name" + }, + { + "id": "no-frontmatter", + "name": "demo", + "files": { + "SKILL.md": { + "text": "# Demo\n\nname: demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md does not start with a --- frontmatter block" + }, + { + "id": "path-512-characters", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc": { + "text": "x\n" + } + }, + "expect": "pass" + }, + { + "id": "path-513-characters", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc": { + "text": "x\n" + } + }, + "expect": "fail", + "error": "is longer than 512 characters" + }, + { + "id": "path-with-space", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + }, + "references/my guide.md": { + "text": "x\n" + } + }, + "expect": "fail", + "error": "path references/my guide.md must match ^[A-Za-z0-9._-]+(\\/[A-Za-z0-9._-]+)*$" + }, + { + "id": "skill-md-missing", + "name": "demo", + "files": { + "README.md": { + "text": "# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md is missing" + }, + { + "id": "skill-md-only-in-evals", + "name": "demo", + "files": { + "evals/SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md is missing" + }, + { + "id": "tab-in-indentation", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n\tauthor: x\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a tab in indentation" + }, + { + "id": "valid-minimal", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "pass" + }, + { + "id": "version-leading-zero", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"01.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "metadata.version 01.0.0 must be MAJOR.MINOR.PATCH" + }, + { + "id": "version-missing", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: d\nmetadata:\n author: x\n---\n" + } + }, + "expect": "fail", + "error": "metadata.version must be a string" + }, + { + "id": "version-three-part", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.2.3\"\n---\n\n# Demo\n" + } + }, + "expect": "pass" + }, + { + "id": "version-three-part-unquoted", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: 1.2.3\n---\n\n# Demo\n" + } + }, + "expect": "pass" + }, + { + "id": "version-two-part-quoted", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"2.6\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "metadata.version 2.6 must be MAJOR.MINOR.PATCH", + "stricter": true + }, + { + "id": "version-two-part-unquoted-number", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: 2.6\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "metadata.version must be a string" + }, + { + "id": "version-unquoted-1-10", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: 1.10\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "metadata.version 1.10 must be MAJOR.MINOR.PATCH", + "stricter": true + }, + { + "id": "version-unquoted-minus-0", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: -0\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "metadata.version -0 must be MAJOR.MINOR.PATCH" + }, + { + "id": "yaml-directive", + "name": "demo", + "files": { + "SKILL.md": { + "text": "---\n%YAML 1.2\nname: demo\ndescription: Use this skill when a test needs a valid skill.\nmetadata:\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + }, + "expect": "fail", + "error": "SKILL.md frontmatter uses a YAML directive" + } +] diff --git a/.github/scripts/validate_skill_publishing.py b/.github/scripts/validate_skill_publishing.py new file mode 100644 index 00000000..b0aa20a2 --- /dev/null +++ b/.github/scripts/validate_skill_publishing.py @@ -0,0 +1,976 @@ +#!/usr/bin/env python3 +"""Validate that every skill in this repository can be published. + +Skills on ``main`` are published as one set: when one skill breaks a rule +below, no skill is published. This check therefore validates every skill +folder under ``skills/`` at the head, not only the folders that a pull request +touches, and reports every error of every skill. + +Each skill folder is read with ``git archive``, so the bytes are the bytes of +the published archive of the commit. The published file set of a skill is +every file of its folder except ``evals/`` at the skill root, and +``.skilleval.yaml`` and ``CHANGELOG.md`` at the skill root. + +Rules for every skill folder at the head: + + * no symlink, hard link, absolute name, or empty, ``.`` or ``..`` segment + anywhere under ``skills/``; + * ``SKILL.md`` exists, is valid UTF-8 (a leading byte order mark is + allowed), and starts with a ``---`` frontmatter block; + * the frontmatter is YAML in a safe subset: no anchors, aliases, explicit + tags, directives, document markers, second document, map keys that are + not scalars, tabs in indentation, duplicate keys, comments without white + space before them, block scalar indicators on their own line, keep (``+``) + chomping, or line breaks other than LF and CRLF; + * ``name`` is a string of 1 to 64 characters that matches + ``^[a-z0-9]+(-[a-z0-9]+)*$`` and equals the folder name; + * ``description`` is a string of 1 to 1024 characters (UTF-16 code units); + * ``metadata.version`` is a ``MAJOR.MINOR.PATCH`` string. A skill that the + pull request does not touch can still have a two-part ``MAJOR.MINOR``, + read as ``MAJOR.MINOR.0``. Remove that allowance when no skill on + ``main`` has a two-part version; + * ``metadata.deprecated`` is a boolean, if present; + * at most 100 published files, each path at most 512 characters and + matching ``^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$``; + * the published files are at most 64 MiB in total; + * the zip of the published files is at most 983,040 bytes: 1 MiB minus a + 64 KiB safety margin, because zip sizes vary by tool. A warning shows + above 90% of that limit. + +The archive of the head commit must also stay at most 128 MiB, and at most +32 MiB when compressed with gzip. + +A published version is immutable. Thus, for every skill folder at the merge +base: + + * the folder must still exist at the head (deprecate a skill instead); + * ``metadata.version`` must not go down; + * a changed skill content hash needs a higher ``metadata.version``; + * an unchanged skill content hash must keep the same ``metadata.version``. + +The skill content hash is SHA-256 over one record per published file, sorted +by the UTF-8 bytes of the path: the path, a 0x00 byte, the lowercase hex +SHA-256 of the file bytes, and a 0x0A byte. + +Frontmatter values are typed like JSON only when the YAML scalar is plain and +the conversion loses nothing: ``true``/``false``, ``null``/``~``/empty, and +numbers that print back as the same text (``3``, ``1.5``). Any other value is +a string (``1.10``, ``2.0``, ``-0``, ``010``, ``2.6.0``). Thus an unquoted +``version: 2.6`` is a number and fails. + +Before it checks a skill, the script checks itself: the skill content hash +of a golden fixture, the SHA-256 of ``conformance/cases.json`` against +``CONFORMANCE_SHA256``, and the result of every case in that file. A change +to the corpus must therefore also change ``CONFORMANCE_SHA256`` and +``conformance/README.md``. Run ``--self-check-only`` to check only that. + +Exit code is 0 when there is no error, 1 when a skill breaks a rule, and 2 +when the self-check fails (then no skill is checked). +""" + +from __future__ import annotations + +import argparse +import base64 +import decimal +import hashlib +import io +import json +import math +import os +import re +import subprocess +import sys +import tarfile +import time +import zipfile +import zlib +from dataclasses import dataclass, field +from pathlib import Path + +import yaml +from yaml.composer import Composer +from yaml.events import ( + AliasEvent, + CollectionStartEvent, + DocumentEndEvent, + DocumentStartEvent, + ScalarEvent, +) +from yaml.nodes import MappingNode, ScalarNode, SequenceNode +from yaml.parser import Parser +from yaml.reader import Reader +from yaml.resolver import BaseResolver +from yaml.scanner import Scanner, ScannerError + +SKILLS_DIR = "skills" + +MAX_NAME_LENGTH = 64 +MAX_DESCRIPTION_LENGTH = 1024 +MAX_FILE_COUNT = 100 +MAX_PATH_LENGTH = 512 +MAX_ZIP_BYTES = 1_048_576 +ZIP_MARGIN_BYTES = 65_536 +ZIP_LIMIT_BYTES = MAX_ZIP_BYTES - ZIP_MARGIN_BYTES +ZIP_WARNING_RATIO = 0.9 +MAX_SKILL_BYTES = 64 * 1024 * 1024 +MAX_ARCHIVE_BYTES = 128 * 1024 * 1024 +MAX_COMPRESSED_ARCHIVE_BYTES = 32 * 1024 * 1024 +MAX_SAFE_INTEGER = 2**53 - 1 + +NAME_PATTERN = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +PATH_PATTERN = re.compile(r"^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$") +VERSION_PATTERN = re.compile(r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +TWO_PART_VERSION_PATTERN = re.compile(r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +INTEGER_PATTERN = re.compile(r"^-?(0|[1-9][0-9]*)$") +DECIMAL_PATTERN = re.compile(r"^-?(0|[1-9][0-9]*)\.[0-9]+$") +# \Z, not $: the frontmatter block ends at the end of the text, not before a final newline. +FRONTMATTER_PATTERN = re.compile(r"^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|\Z)") +EXCLUDED_ROOT_FILES = frozenset({".skilleval.yaml", "CHANGELOG.md"}) + +NAME_PATTERN_SOURCE = "^[a-z0-9]+(-[a-z0-9]+)*$" +PATH_PATTERN_SOURCE = "^[A-Za-z0-9._-]+(\\/[A-Za-z0-9._-]+)*$" + +REMOVED_MESSAGE = ( + "a pull request cannot remove a skill folder; set metadata.deprecated: true " + "and bump metadata.version" +) +HASH_CHANGED_MESSAGE = "published content changed, so metadata.version must go up" +VERSION_ONLY_MESSAGE = "metadata.version changed, but the published content did not" + +DOCS_HINT = 'See the "Publishing Rules" section of CONTRIBUTING.md.' + +_MISSING = object() + +GOLDEN_FILES = { + "SKILL.md": b"---\nname: example-skill\ndescription: Example skill for the content hash golden test.\nmetadata:\n version: 1.0.0\n---\n\n# Example\n", + "references/guide.md": b"Line one\r\nLine two\n", + "images/pixel.png": bytes([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0xFF]), + "assets/empty.txt": b"", +} +GOLDEN_HASH = "15f2eecb8d833caa6da288791f742204460d998c6a0d3cfbdb8bc5b51cf01822" + +CONFORMANCE_FILE = Path(__file__).resolve().parent / "conformance" / "cases.json" +CONFORMANCE_SHA256 = "df26439cc6d8f5c7eb74250b6ab5ca512ca6f6902464c8affc30b3b82ac7ed2e" +CASE_FIELDS = frozenset({"id", "name", "files", "expect", "error", "stricter"}) + + +class FrontmatterError(ValueError): + pass + + +def unsupported(feature: str) -> FrontmatterError: + return FrontmatterError( + f"SKILL.md frontmatter uses {feature}, which skills in this repository must not use" + ) + + +@dataclass +class SkillTree: + """The regular files of one skill folder at one revision, and its unsafe entries.""" + + files: dict[str, bytes] = field(default_factory=dict) + errors: list[str] = field(default_factory=list) + + +@dataclass +class HeadResult: + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + version: tuple[int, int, int] | None = None + + +@dataclass +class SkillReport: + path: str + change: str + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + base_version: str | None = None + head_version: str | None = None + + @property + def ok(self) -> bool: + return not self.errors + + @property + def notable(self) -> bool: + return self.change != "unchanged" or bool(self.errors or self.warnings) + + +class _FrontmatterLoader(Reader, Scanner, Parser, Composer, BaseResolver): + """PyYAML without a constructor, and with the YAML 1.2 indentation rules that PyYAML does not apply. + + A flow collection or a multi-line quoted scalar inside a block collection + must be indented more than that block collection, and a lone ``-`` inside + a flow collection is a block sequence entry, not a plain scalar. + """ + + def __init__(self, stream: str): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseResolver.__init__(self) + + def _line_indent(self) -> int | None: + """Leading spaces of the line, or None when text comes before the scanner on that line.""" + prefix = self.buffer[self.pointer - self.column : self.pointer] + if prefix.strip(" \t"): + return None + return len(prefix) - len(prefix.lstrip(" ")) + + def _check_flow_line(self) -> None: + if not self.flow_level or self.peek() in "]}#\0": + return + spaces = self._line_indent() + if spaces is not None and spaces <= self.indent: + raise ScannerError( + None, + None, + "Flow collection in block collection must be sufficiently indented", + self.get_mark(), + ) + + def unwind_indent(self, column): + self._check_flow_line() + super().unwind_indent(column) + + def scan_plain_spaces(self, indent, start_mark): + line = self.line + chunks = super().scan_plain_spaces(indent, start_mark) + if self.line != line: + self._check_flow_line() + return chunks + + def scan_plain(self): + token = super().scan_plain() + if self.flow_level and token.value == "-": + raise ScannerError( + None, + None, + "Block collections are not allowed within flow collections", + token.start_mark, + ) + return token + + def scan_to_next_token(self): + if self.peek() == "#" and self.pointer > 0 and self.buffer[self.pointer - 1] not in " \t\r\n\ufeff": + raise ScannerError( + None, + None, + "Comments must be separated from other tokens by white space", + self.get_mark(), + ) + return super().scan_to_next_token() + + def scan_block_scalar(self, style): + if self._line_indent() is not None: + raise ScannerError( + None, + None, + "a block scalar indicator must be on the line of its key", + self.get_mark(), + ) + return super().scan_block_scalar(style) + + def scan_block_scalar_indicators(self, start_mark): + chomping, increment = super().scan_block_scalar_indicators(start_mark) + if chomping is True: + raise unsupported("a keep (+) chomping indicator") + return chomping, increment + + def scan_flow_scalar_breaks(self, double, start_mark): + chunks = super().scan_flow_scalar_breaks(double, start_mark) + spaces = self._line_indent() + if spaces is not None and spaces <= self.indent: + raise ScannerError( + "while scanning a quoted scalar", + start_mark, + "a line of a multi-line quoted scalar must be indented more than its block collection", + self.get_mark(), + ) + return chunks + + def compose_scalar_node(self, anchor): + event = self.peek_event() + node = super().compose_scalar_node(anchor) + node.plain = event.tag is None and event.style is None + return node + + +def _one_line(text: str) -> str: + return " ".join(text.split()) + + +def _check_safe_subset(text: str) -> None: + if re.search("\r(?!\n)|[\x85\u2028\u2029]", text): + raise unsupported("a line break character") + for line in text.split("\n"): + indentation = line[: len(line) - len(line.lstrip(" \t"))] + if "\t" in indentation: + raise unsupported("a tab in indentation") + if line.startswith("%"): + raise unsupported("a YAML directive") + loader = _FrontmatterLoader(text) + documents = 0 + try: + while loader.check_event(): + event = loader.get_event() + if isinstance(event, AliasEvent): + raise unsupported("an alias") + if isinstance(event, (ScalarEvent, CollectionStartEvent)): + if event.anchor is not None: + raise unsupported("an anchor") + if event.tag is not None: + raise unsupported("an explicit tag") + if isinstance(event, DocumentStartEvent): + documents += 1 + if documents > 1: + raise unsupported("more than one document") + if event.version is not None or event.tags: + raise unsupported("a YAML directive") + if event.explicit: + raise unsupported("a document marker") + if isinstance(event, DocumentEndEvent) and event.explicit: + raise unsupported("a document marker") + finally: + loader.dispose() + + +def _check_duplicate_keys(node) -> None: + if isinstance(node, SequenceNode): + for item in node.value: + _check_duplicate_keys(item) + elif isinstance(node, MappingNode): + keys: set[str] = set() + for key, value in node.value: + if isinstance(key, ScalarNode): + if key.value in keys: + raise FrontmatterError("Map keys must be unique") + keys.add(key.value) + _check_duplicate_keys(key) + _check_duplicate_keys(value) + + +def js_number_text(value: float) -> str: + """JavaScript String(value) of a finite number, so that "prints back as the same text" matches JSON tools.""" + if value == 0: + return "0" + _, digit_tuple, exponent = decimal.Decimal(repr(abs(value))).normalize().as_tuple() + digits = "".join(map(str, digit_tuple)) + k = len(digits) + n = exponent + k + sign = "-" if value < 0 else "" + if k <= n <= 21: + return sign + digits + "0" * (n - k) + if 0 < n <= 21: + return sign + digits[:n] + "." + digits[n:] + if -6 < n <= 0: + return sign + "0." + "0" * -n + digits + e = n - 1 + mantissa = digits if k == 1 else digits[0] + "." + digits[1:] + return f"{sign}{mantissa}e{'+' if e >= 0 else '-'}{abs(e)}" + + +def plain_scalar_to_json(text: str): + if text in ("true", "false"): + return text == "true" + if text in ("null", "~", ""): + return None + if INTEGER_PATTERN.match(text): + value = int(text) + if text != "-0" and abs(value) <= MAX_SAFE_INTEGER: + return value + elif DECIMAL_PATTERN.match(text): + number = float(text) + if math.isfinite(number) and js_number_text(number) == text: + return number + return text + + +def _to_json(node): + if isinstance(node, ScalarNode): + return plain_scalar_to_json(node.value) if node.plain else node.value + if isinstance(node, SequenceNode): + return [_to_json(item) for item in node.value] + result = {} + for key, value in node.value: + if not isinstance(key, ScalarNode): + raise unsupported("a map key that is not a scalar") + result[key.value] = _to_json(value) + return result + + +def parse_frontmatter(text: str) -> dict: + """Parses the SKILL.md frontmatter to JSON values, or raises FrontmatterError.""" + match = FRONTMATTER_PATTERN.match(text.removeprefix("\ufeff")) + if match is None: + raise FrontmatterError("SKILL.md does not start with a --- frontmatter block") + # Parsed with a final newline, so a block scalar as the last key ends with a newline. + source = match.group(1) + "\n" + try: + _check_safe_subset(source) + loader = _FrontmatterLoader(source) + try: + root = loader.get_single_node() + finally: + loader.dispose() + if root is not None: + _check_duplicate_keys(root) + parsed = None if root is None else _to_json(root) + except (yaml.YAMLError, RecursionError) as exc: + reason = "nesting is too deep" if isinstance(exc, RecursionError) else _one_line(str(exc)) + raise FrontmatterError(f"SKILL.md frontmatter is not valid YAML: {reason}") from exc + except FrontmatterError as exc: + if str(exc).startswith("SKILL.md"): + raise + raise FrontmatterError(f"SKILL.md frontmatter is not valid YAML: {exc}") from exc + if not isinstance(parsed, dict): + raise FrontmatterError("SKILL.md frontmatter is not an object") + return parsed + + +def decode_skill_md(data: bytes) -> str: + """Strict UTF-8 that drops one leading byte order mark; parse_frontmatter drops one more.""" + return data.decode("utf-8").removeprefix("\ufeff") + + +def js_length(text: str) -> int: + """String length in UTF-16 code units.""" + return len(text.encode("utf-16-le", "surrogatepass")) // 2 + + +def is_published_path(path: str) -> bool: + return not path.startswith("evals/") and path not in EXCLUDED_ROOT_FILES + + +def published_files(files: dict[str, bytes]) -> dict[str, bytes]: + return {path: data for path, data in files.items() if is_published_path(path)} + + +def sorted_paths(files) -> list[str]: + return sorted(files, key=lambda path: path.encode("utf-8", "surrogateescape")) + + +def content_hash(files: dict[str, bytes]) -> str: + """The skill content hash of a published file set.""" + outer = hashlib.sha256() + for path in sorted_paths(files): + digest = hashlib.sha256(files[path]).hexdigest() + outer.update(path.encode("utf-8", "surrogateescape") + b"\0" + digest.encode("ascii") + b"\n") + return outer.hexdigest() + + +def build_zip(files: dict[str, bytes]) -> bytes: + """Entries sorted by UTF-8 path, deflate level 6, all dated 1980-01-01.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for path in sorted_paths(files): + info = zipfile.ZipInfo(path, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, files[path], compresslevel=6) + return buffer.getvalue() + + +def read_version(frontmatter: dict, allow_two_part: bool) -> tuple[int, int, int]: + metadata = frontmatter.get("metadata") + version = metadata.get("version") if isinstance(metadata, dict) else None + if not isinstance(version, str): + raise FrontmatterError("metadata.version must be a string") + if allow_two_part and TWO_PART_VERSION_PATTERN.match(version): + version += ".0" + if not VERSION_PATTERN.match(version): + raise FrontmatterError(f"metadata.version {version} must be MAJOR.MINOR.PATCH") + major, minor, patch = (int(part) for part in version.split(".")) + return major, minor, patch + + +def _field_errors(name: str, frontmatter: dict) -> list[str]: + errors = [] + declared = frontmatter.get("name") + if not isinstance(declared, str) or not 1 <= js_length(declared) <= MAX_NAME_LENGTH: + errors.append(f"name must be a string of 1 to {MAX_NAME_LENGTH} characters") + elif not NAME_PATTERN.match(declared): + errors.append(f"name must match {NAME_PATTERN_SOURCE}") + elif declared != name: + errors.append("name must equal the folder name") + description = frontmatter.get("description") + if ( + not isinstance(description, str) + or not 1 <= js_length(description) <= MAX_DESCRIPTION_LENGTH + ): + errors.append( + f"description must be a string of 1 to {MAX_DESCRIPTION_LENGTH} characters" + ) + return errors + + +def _path_errors(paths: list[str]) -> list[str]: + errors = [] + if len(paths) > MAX_FILE_COUNT: + errors.append(f"skill has {len(paths)} files; the maximum is {MAX_FILE_COUNT}") + for path in paths: + if js_length(path) > MAX_PATH_LENGTH: + errors.append(f"path {path} is longer than {MAX_PATH_LENGTH} characters") + elif not PATH_PATTERN.match(path): + errors.append(f"path {path} must match {PATH_PATTERN_SOURCE}") + return errors + + +def _size_errors(published: dict[str, bytes], result: HeadResult) -> None: + total = sum(len(data) for data in published.values()) + if total > MAX_SKILL_BYTES: + result.errors.append( + f"published files are {total} bytes in total; the maximum is {MAX_SKILL_BYTES}" + ) + return + zip_bytes = len(build_zip(published)) + if zip_bytes > ZIP_LIMIT_BYTES: + result.errors.append( + f"zip is {zip_bytes} bytes; the maximum is {ZIP_LIMIT_BYTES} " + f"(1 MiB minus a {ZIP_MARGIN_BYTES}-byte safety margin, because zip sizes vary by tool)" + ) + elif zip_bytes > ZIP_LIMIT_BYTES * ZIP_WARNING_RATIO: + result.warnings.append( + f"zip is {zip_bytes} bytes, over {int(ZIP_WARNING_RATIO * 100)}% of " + f"the maximum of {ZIP_LIMIT_BYTES}" + ) + + +def check_head(name: str, files: dict[str, bytes], strict_version: bool = True) -> HeadResult: + """The head rules for one skill folder. All errors are reported, not only the first.""" + result = HeadResult() + published = published_files(files) + skill_md = published.get("SKILL.md") + if skill_md is None: + result.errors.append("SKILL.md is missing") + else: + try: + frontmatter = parse_frontmatter(decode_skill_md(skill_md)) + except UnicodeDecodeError: + result.errors.append("SKILL.md is not valid UTF-8") + except FrontmatterError as exc: + result.errors.append(str(exc)) + else: + result.errors += _field_errors(name, frontmatter) + try: + result.version = read_version(frontmatter, allow_two_part=not strict_version) + except FrontmatterError as exc: + result.errors.append(str(exc)) + metadata = frontmatter.get("metadata") + deprecated = metadata.get("deprecated", _MISSING) if isinstance(metadata, dict) else _MISSING + if deprecated is not _MISSING and not isinstance(deprecated, bool): + result.errors.append("metadata.deprecated must be a boolean") + result.errors += _path_errors(sorted_paths(published)) + _size_errors(published, result) + return result + + +def base_version(files: dict[str, bytes]) -> tuple[int, int, int]: + skill_md = published_files(files).get("SKILL.md") + if skill_md is None: + raise FrontmatterError("SKILL.md is missing") + try: + text = decode_skill_md(skill_md) + except UnicodeDecodeError as exc: + raise FrontmatterError("SKILL.md is not valid UTF-8") from exc + return read_version(parse_frontmatter(text), allow_two_part=True) + + +def _version_text(version: tuple[int, int, int]) -> str: + return ".".join(map(str, version)) + + +def check_skill( + name: str, base: SkillTree | None, head: SkillTree | None, touched: bool = True +) -> SkillReport: + path = f"{SKILLS_DIR}/{name}" + if head is None: + report = SkillReport(path, "removed") + if base is not None: + report.errors.append(REMOVED_MESSAGE) + return report + + change = "added" if base is None else ("changed" if touched else "unchanged") + report = SkillReport(path, change) + report.errors += head.errors + head_result = check_head(name, head.files, strict_version=touched) + report.errors += head_result.errors + report.warnings += head_result.warnings + if head_result.version is not None: + report.head_version = _version_text(head_result.version) + if base is None or head_result.version is None: + return report + + try: + if base.errors: + raise FrontmatterError(base.errors[0]) + old_version = base_version(base.files) + except FrontmatterError as exc: + report.warnings.append( + f"the version rules were not checked, because the skill at the base " + f"commit is not valid: {exc}" + ) + return report + report.base_version = _version_text(old_version) + new_version = head_result.version + hash_changed = content_hash(published_files(base.files)) != content_hash( + published_files(head.files) + ) + if new_version < old_version: + report.errors.append( + f"metadata.version went down from {report.base_version} to {report.head_version}" + ) + elif hash_changed and new_version == old_version: + report.errors.append(HASH_CHANGED_MESSAGE) + elif not hash_changed and new_version != old_version: + report.errors.append(VERSION_ONLY_MESSAGE) + return report + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _git(repo_root: Path, *args: str) -> subprocess.CompletedProcess[bytes]: + env = {**os.environ, "GIT_LITERAL_PATHSPECS": "1"} + return subprocess.run(["git", *args], cwd=repo_root, capture_output=True, env=env) + + +def _git_ok(repo_root: Path, *args: str) -> bytes: + result = _git(repo_root, *args) + if result.returncode != 0: + raise SystemExit( + f"git {' '.join(args)} failed: {result.stderr.decode('utf-8', 'replace').strip()}" + ) + return result.stdout + + +def _decode_path(raw: bytes) -> str: + return raw.decode("utf-8", "surrogateescape") + + +def _diff_base(repo_root: Path, base_ref: str, head_ref: str) -> str: + result = _git(repo_root, "merge-base", base_ref, head_ref) + merge_base = result.stdout.decode().strip() + if result.returncode == 0 and merge_base: + return merge_base + print( + f"warning: no merge base between {base_ref} and {head_ref}; " + f"diffing against {base_ref} directly", + file=sys.stderr, + ) + return base_ref + + +def changed_paths(repo_root: Path, base: str, head: str) -> list[str]: + out = _git_ok(repo_root, "diff", "--name-only", "--no-renames", "-z", base, head) + return [_decode_path(raw) for raw in out.split(b"\0") if raw] + + +def touched_names(paths: list[str]) -> set[str] | None: + """Skill folder names with a change, or None when a .gitattributes change touches all of them.""" + if any(path.rsplit("/", 1)[-1] == ".gitattributes" for path in paths): + return None + return { + parts[1] + for parts in (path.split("/") for path in paths) + if len(parts) >= 3 and parts[0] == SKILLS_DIR + } + + +def _skill_folders(repo_root: Path, rev: str) -> set[str]: + out = _git_ok(repo_root, "ls-tree", "-z", rev, "--", f"{SKILLS_DIR}/") + folders = set() + for raw in out.split(b"\0"): + meta, _, name = raw.partition(b"\t") + if meta.split(b" ")[1:2] == [b"tree"]: + folders.add(_decode_path(name).split("/", 1)[1]) + return folders + + +def read_skills(repo_root: Path, rev: str) -> tuple[dict[str, SkillTree], list[SkillReport]]: + """Every skill folder at ``rev`` from one ``git archive``, and reports for unsafe entries directly in skills/.""" + folders = _skill_folders(repo_root, rev) + trees = {name: SkillTree() for name in folders} + loose: list[SkillReport] = [] + if not folders: + return trees, loose + result = _git(repo_root, "archive", "--format=tar", rev, "--", SKILLS_DIR) + if result.returncode != 0: + raise SystemExit( + f"git archive {rev} {SKILLS_DIR} failed: " + f"{result.stderr.decode('utf-8', 'replace').strip()}" + ) + prefix = f"{SKILLS_DIR}/" + with tarfile.open(fileobj=io.BytesIO(result.stdout), encoding="utf-8") as tar: + for member in tar: + entry = member.name + relative = entry.lstrip("/") + relative = relative[len(prefix):] if relative.startswith(prefix) else relative + folder, _, inner = relative.partition("/") + tree = trees.get(folder) if inner else None + if entry.startswith("/"): + error = f"Tarball entry {entry} has an absolute path" + elif any(segment in ("", ".", "..") for segment in entry.split("/")): + error = f'Tarball entry {entry} has an empty, "." or ".." path segment' + elif member.issym() or member.islnk(): + kind = "symlink" if member.issym() else "link" + error = f"Tarball entry {entry} is a {kind}; links are not allowed" + else: + if tree is not None and member.isfile(): + tree.files[inner] = tar.extractfile(member).read() + continue + if tree is not None: + tree.errors.append(error) + elif entry != SKILLS_DIR: + loose.append(SkillReport(entry, "—", errors=[error])) + return trees, loose + + +def check_archive(repo_root: Path, rev: str) -> SkillReport: + """Size of the archive of the whole commit, uncompressed and with gzip.""" + report = SkillReport("(repository archive)", "—") + raw = 0 + compressed = 0 + compressor = zlib.compressobj(6, zlib.DEFLATED, 31) + with subprocess.Popen( + ["git", "archive", "--format=tar", rev], cwd=repo_root, stdout=subprocess.PIPE + ) as process: + for chunk in iter(lambda: process.stdout.read(1 << 20), b""): + raw += len(chunk) + compressed += len(compressor.compress(chunk)) + compressed += len(compressor.flush()) + if process.returncode != 0: + raise SystemExit(f"git archive {rev} failed") + if raw > MAX_ARCHIVE_BYTES: + report.errors.append( + f"the repository archive is {raw} bytes; the maximum is {MAX_ARCHIVE_BYTES}" + ) + if compressed > MAX_COMPRESSED_ARCHIVE_BYTES: + report.errors.append( + f"the compressed repository archive is {compressed} bytes; the maximum is " + f"{MAX_COMPRESSED_ARCHIVE_BYTES}" + ) + return report + + +def _escape_data(text: str) -> str: + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def _escape_property(text: str) -> str: + return _escape_data(text).replace(":", "%3A").replace(",", "%2C") + + +def _annotate(report: SkillReport) -> None: + if not os.environ.get("GITHUB_ACTIONS"): + return + title = _escape_property(f"Skill publishing: {report.path}") + for error in report.errors: + print(f"::error title={title}::{_escape_data(f'{error} — {DOCS_HINT}')}") + for warning in report.warnings: + print(f"::warning title={title}::{_escape_data(warning)}") + + +def _versions(report: SkillReport) -> str: + return f"{report.base_version or '—'} → {report.head_version or '—'}" + + +def _write_summary(reports: list[SkillReport]) -> None: + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + shown = [r for r in reports if r.notable] + quiet = len(reports) - len(shown) + lines = ["## Skill publishing check", ""] + if shown: + lines += ["| Skill | Change | Version | Result |", "| --- | --- | --- | --- |"] + for r in shown: + result = "pass" if r.ok else f"**fail** — {len(r.errors)} error(s)" + if r.warnings: + result += f", {len(r.warnings)} warning(s)" + lines.append(f"| `{r.path}` | {r.change} | {_versions(r)} | {result} |") + lines.append("") + lines.append(f"{quiet} unchanged skill(s) pass.") + lines.append("") + for r in shown: + if r.errors or r.warnings: + lines.append(f"### `{r.path}`") + lines += [f"- error: {_one_line(e)}" for e in r.errors] + lines += [f"- warning: {_one_line(w)}" for w in r.warnings] + lines.append("") + if any(not r.ok for r in reports): + lines.append(DOCS_HINT) + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + +def run(repo_root: Path, base_ref: str, head_ref: str) -> list[SkillReport]: + base = _diff_base(repo_root, base_ref, head_ref) + touched = touched_names(changed_paths(repo_root, base, head_ref)) + base_trees, _ = read_skills(repo_root, base) + head_trees, reports = read_skills(repo_root, head_ref) + for name in sorted(base_trees.keys() | head_trees.keys()): + reports.append( + check_skill( + name, + base_trees.get(name), + head_trees.get(name), + touched=touched is None or name in touched, + ) + ) + archive = check_archive(repo_root, head_ref) + if archive.errors: + reports.append(archive) + return sorted(reports, key=lambda r: r.path) + + +def _case_files(files) -> dict[str, bytes]: + return { + path: content["text"].encode("utf-8") if "text" in content else base64.b64decode(content["base64"], validate=True) + for path, content in files.items() + } + + +def _check_case(case) -> list[str]: + case_id = case.get("id") if isinstance(case, dict) else None + label = f"conformance case {case_id}" + if ( + not isinstance(case, dict) + or not isinstance(case_id, str) + or not set(case) <= CASE_FIELDS + or not isinstance(case.get("name"), str) + or not isinstance(case.get("files"), dict) + or case.get("expect") not in ("pass", "fail") + or (case["expect"] == "fail" and not isinstance(case.get("error"), str)) + ): + return [f"{label} is malformed"] + try: + files = _case_files(case["files"]) + except (AttributeError, KeyError, TypeError, ValueError): + return [f"{label} has a malformed file entry"] + errors = check_head(case["name"], files).errors + if case["expect"] == "pass": + return [f"{label} should pass, but fails: {_one_line(errors[0])}"] if errors else [] + expected = case["error"] + if not errors: + return [f"{label} should fail with {expected!r}, but passes"] + if not any(expected in error for error in errors): + return [f"{label} should fail with {expected!r}, but fails with: {_one_line(errors[0])}"] + return [] + + +def self_check() -> tuple[list[str], int]: + """Problems found in the script itself, and the number of conformance cases run.""" + problems = [] + golden = content_hash(GOLDEN_FILES) + if golden != GOLDEN_HASH: + problems.append( + f"the skill content hash of the golden fixture is {golden}, expected {GOLDEN_HASH}" + ) + relative = "conformance/cases.json" + try: + data = CONFORMANCE_FILE.read_bytes() + except OSError as exc: + return problems + [f"cannot read {relative}: {exc}"], 0 + digest = hashlib.sha256(data).hexdigest() + if digest != CONFORMANCE_SHA256: + problems.append( + f"{relative} has SHA-256 {digest}, but CONFORMANCE_SHA256 is {CONFORMANCE_SHA256}; " + "a corpus change must also update CONFORMANCE_SHA256 and conformance/README.md" + ) + try: + cases = json.loads(data.decode("utf-8")) + except ValueError as exc: + return problems + [f"{relative} is not valid JSON: {exc}"], 0 + if not isinstance(cases, list) or not cases: + return problems + [f"{relative} must be a non-empty JSON array"], 0 + ids = [str(case.get("id")) if isinstance(case, dict) else "" for case in cases] + duplicates = sorted({case_id for case_id in ids if ids.count(case_id) > 1}) + if duplicates: + problems.append(f"{relative} has duplicate ids: {', '.join(duplicates)}") + if ids != sorted(ids): + problems.append(f"{relative} is not sorted by id") + for case in cases: + problems += _check_case(case) + return problems, len(cases) + + +def _report_self_check_failure(problems: list[str]) -> None: + for problem in problems: + print(f"::error title=Self-check failed::{_escape_data(problem)}") + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + lines = ["## Skill publishing check", "", "The self-check of the script failed, so no skill was checked:", ""] + lines += [f"- {_one_line(problem)}" for problem in problems] + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + +def main(argv: list[str] | None = None, repo_root: Path | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--base-ref", + default="origin/main", + help="Base ref of the pull request (default: origin/main).", + ) + parser.add_argument( + "--head-ref", + default="HEAD", + help="Head ref of the pull request (default: HEAD).", + ) + parser.add_argument( + "--self-check-only", + action="store_true", + help="Run only the self-check (golden hash, corpus digest, conformance cases).", + ) + args = parser.parse_args(argv) + + started = time.monotonic() + problems, case_count = self_check() + elapsed = time.monotonic() - started + if problems: + _report_self_check_failure(problems) + return 2 + print( + f"Self-check passed: golden hash, corpus digest, and {case_count} conformance " + f"cases in {elapsed:.2f} s.", + file=sys.stderr, + ) + if args.self_check_only: + return 0 + + reports = run(repo_root or _repo_root(), args.base_ref, args.head_ref) + shown = [r for r in reports if r.notable] + for r in shown: + print(f"{'PASS' if r.ok else 'FAIL'} {r.path} ({r.change}, {_versions(r)})") + for error in r.errors: + print(f" - {_one_line(error)}") + for warning in r.warnings: + print(f" ! {_one_line(warning)}") + _annotate(r) + _write_summary(reports) + + failed = [r for r in reports if not r.ok] + print( + f"\n{len(reports)} skill(s) checked: {len(reports) - len(failed)} pass, " + f"{len(failed)} fail. {len(reports) - len(shown)} unchanged skill(s) that pass " + "are not listed.", + file=sys.stderr, + ) + if failed: + print( + "\nWhen one skill on main breaks these rules, no skill is published, so " + f"merging is blocked. {DOCS_HINT}", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/validate-skill-publishing-main.yml b/.github/workflows/validate-skill-publishing-main.yml new file mode 100644 index 00000000..62d4661c --- /dev/null +++ b/.github/workflows/validate-skill-publishing-main.yml @@ -0,0 +1,44 @@ +name: Validate skill publishing on main + +# Runs the skill publishing check again after each push to `main`, with the +# previous head of `main` as the base. The pull request check is exact only for +# the base that it ran on, so two pull requests that each passed on an older +# base can break the rules together when both merge. This run reports that at +# once, so the next pull request can fix it. It only reads the repository. + +on: + push: + branches: [main] + +permissions: + contents: read + +jobs: + validate-skill-publishing-main: + name: validate-skill-publishing-main + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install PyYAML + run: python3 -m pip install pyyaml==6.0.3 + + - name: Check publishing rules of every skill + env: + BEFORE: ${{ github.event.before }} + run: | + # Without a usable previous head (the push created the branch, or a + # force push removed it), compare with the parent commit instead. + if [ -z "${BEFORE//0/}" ] || ! git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then + BEFORE="$(git rev-parse --verify --quiet HEAD^ || git hash-object -t tree /dev/null)" + fi + + python3 .github/scripts/validate_skill_publishing.py \ + --base-ref "${BEFORE}" \ + --head-ref HEAD diff --git a/.github/workflows/validate-skill-publishing.yml b/.github/workflows/validate-skill-publishing.yml new file mode 100644 index 00000000..0c3d746d --- /dev/null +++ b/.github/workflows/validate-skill-publishing.yml @@ -0,0 +1,66 @@ +name: Validate skill publishing + +# Checks that every skill in the repository can be published. Skills on `main` +# are published as one set, and one skill that breaks a rule stops the +# publishing of all skills, so the check validates the whole skills/ tree on +# every pull request. See .github/scripts/validate_skill_publishing.py for the +# rules, and validate-skill-publishing-main.yml for the same check after a merge. +# +# NOTE: a red check does not block merging on its own. An admin must add the job's +# name ("validate-skill-publishing") to the required status checks for `main` in +# branch protection. Until then this workflow is advisory. +# +# The result is exact only for the base commit that the check ran on. Two pull +# requests that each pass on an older base can break the rules together, for +# example when both bump a skill to the same version. Turn on "Require branches +# to be up to date before merging" in branch protection, or use a merge queue, +# so that every merge is checked against the current `main`. + +on: + pull_request: + # `edited` re-runs the check when the base branch of the pull request changes. + types: [opened, synchronize, reopened, ready_for_review, edited] + # Deliberately NOT filtered with `paths: skills/**`. A required status check + # that never runs is reported as pending forever, so a path filter would block + # every PR that does not touch skills/. + +permissions: + contents: read + +concurrency: + group: validate-skill-publishing-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + # The job id and its display name are the same, because the display name is the + # status check context that branch protection matches on. + validate-skill-publishing: + name: validate-skill-publishing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # Full history, so the script can find the merge base and read every + # skill at it. + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install PyYAML + run: python3 -m pip install pyyaml==6.0.3 + + - name: Check publishing rules of every skill + # Event values are passed through the environment and quoted on use, not + # interpolated into the command. + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + git fetch --no-tags --quiet origin \ + "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" + + python3 .github/scripts/validate_skill_publishing.py \ + --base-ref "origin/${BASE_REF}" \ + --head-ref HEAD diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6728905..cf917094 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,45 @@ Follow these guidelines for each tool: Make sure the frontmatter in the `SKILL.md` file includes a `metadata` block with `version`, `author` (your GitHub user) and `aws-devops-agent-skills.*` fields (see examples in existing skills) +#### Publishing Rules + +Skills on `main` are published as one set: when one skill breaks a rule below, no skill is published. The `validate-skill-publishing` check therefore applies these rules to every skill in `skills/` on each pull request, not only to the skills that the pull request changes. + +`SKILL.md` and its frontmatter: + +- `SKILL.md` is valid UTF-8 and starts with a `---` line, the YAML frontmatter, and a closing `---` line. +- `name` is 1 to 64 characters, matches `^[a-z0-9]+(-[a-z0-9]+)*$` (lowercase letters, digits, and single hyphens), and equals the folder name. +- `description` is 1 to 1024 characters. A block scalar (`>` or `|`) ends with a newline, and the newline counts. +- `metadata.version` is a quoted `MAJOR.MINOR.PATCH` string, for example `version: "1.2.0"`. An unquoted `version: 2.6` is a number, not a string, and fails. +- `metadata.deprecated`, if present, is `true` or `false` without quotes. +- The frontmatter uses plain YAML only. These are not allowed: duplicate keys, anchors (`&`), aliases (`*`), tags (`!` or `!!`), directives (`%`), document markers (`---` or `...` inside the frontmatter), map keys that are not plain values, tabs in indentation, a `#` comment without a space before it, a block scalar indicator (`|` or `>`) on its own line, the keep indicator (`|+` or `>+`), and line breaks other than LF or CRLF. + +Versions: + +- A published version is immutable. Bump `metadata.version` on every change to a published file, also a change to `README.md`. Do not change the version when no published file changes. The version must never go down. +- Do not remove a skill folder. To retire a skill, set `metadata.deprecated: true` and bump `metadata.version`. + +Files and sizes: + +- These files are not published, so a change to them alone needs no version bump: the `evals/` folder, and `.skilleval.yaml` and `CHANGELOG.md` at the skill root. Every other file in the skill folder is published, binary files too. +- A skill has at most 100 published files. Each path, relative to the skill folder, is at most 512 characters and matches `^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$`. Symlinks are not allowed. +- The published files of a skill are at most 64 MiB in total, and their zip is at most 983,040 bytes. That is 1 MiB with a safety margin, because zip sizes vary by tool. +- The archive of the whole repository is at most 128 MiB, and at most 32 MiB when compressed. + +The check is exact only for the base commit that it ran on. Two pull requests that each pass on an older base can break the rules together, for example when both bump a skill to the same version. Maintainers should turn on "Require branches to be up to date before merging" in branch protection, or use a merge queue. The `validate-skill-publishing-main` workflow runs the check again after each push to `main` and reports such a break at once. + +Run the check locally before you push (it needs `pip install pyyaml==6.0.3`). Use the `main` branch of this repository as the base. In a clone of a fork, add this repository as the `upstream` remote, fetch it, and use `upstream/main`: + +```bash +git remote add upstream https://github.com/aws/tools-for-devops-agent.git +git fetch upstream main +python3 .github/scripts/validate_skill_publishing.py --base-ref upstream/main +``` + +In a clone of this repository, use `--base-ref origin/main`. + +The script checks itself before it checks any skill: the skill content hash of a golden fixture, the digest of `.github/scripts/conformance/cases.json`, and every case in that file (see `.github/scripts/conformance/README.md`). When the self-check fails, the script exits with code 2. Run `python3 .github/scripts/validate_skill_publishing.py --self-check-only` to check only that, for example after you change the script or the corpus. + #### Test Your Skill 1. Test relevant scenarios with DevOps Agent, with and without skill, to understand what good looks like. Iterate several times and make changes as necessary. Make sure you test relevant functionalities. For example, if your skill is intended for DevOps Agent investigations related to RDS PostgreSQL, then set up relevant AWS resources, simulate issues, and then start DevOps Agent investigations and evaluate the root cause with and without skill. Another example is, if your skill is intended to generate a report using DevOps Agent chat (such as EKS operations review), then set up relevant AWS resources, simulate scenarios that you expect your report to highlight, and then start DevOps Agent chat and evaluate its response with and without skill. Some tips for what you should check when evaluating your skill: for investigation, evaluate the different investigation root cause parts in the "Root cause" investigation tab (root cause, key findings, gaps, evidence) for correctness and quality across multiple iterations with and without skill. For chat, evaluate the chat's final response for actionability, correctness and completeness, with and without skill. Compare with and without skill runs and iterate to see if there are improvements. In both chat and investigation, evaluate multiple iterations for output consistency (investigation root cause or chat final response). diff --git a/skills/analytics-opensearch-expertise/CHANGELOG.md b/skills/analytics-opensearch-expertise/CHANGELOG.md index 83e526db..ed0b5769 100644 --- a/skills/analytics-opensearch-expertise/CHANGELOG.md +++ b/skills/analytics-opensearch-expertise/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to the `analytics-opensearch-expertise` skill are documented here. +## [2.6.1] - 2026-10-07 +### Changed +- `metadata.version` in `SKILL.md` uses the full `MAJOR.MINOR.PATCH` form (`2.6` is now `2.6.1`). + ## [2.6.0] - 2026-07-24 ### Changed - Remediation Reference moved to `references/remediation-reference.md`, loaded on demand via diff --git a/skills/analytics-opensearch-expertise/SKILL.md b/skills/analytics-opensearch-expertise/SKILL.md index 6926562a..4cf2ba04 100644 --- a/skills/analytics-opensearch-expertise/SKILL.md +++ b/skills/analytics-opensearch-expertise/SKILL.md @@ -2,7 +2,7 @@ name: analytics-opensearch-expertise description: "Amazon OpenSearch Service domain health assessment. Performs read-only, API-driven checks against a customer's OpenSearch domain(s) covering cluster health, node/shard configuration, performance metrics, security posture, and cost optimization signals. Activate this skill for requests about OpenSearch or Elasticsearch domain health, cluster review, domain assessment, performance, security posture, or cost optimization. Given a domain ARN (or name + region), it produces a structured findings report with prioritized recommendations. All checks use read-only AWS control-plane APIs (es:Describe*, es:GetCompatibleVersions, es:DescribeReservedInstances, cloudwatch:GetMetricData) — no data-plane access required." metadata: - version: "2.6" + version: "2.6.1" author: genealpe --- diff --git a/skills/database-rds-devops/CHANGELOG.md b/skills/database-rds-devops/CHANGELOG.md index f2619653..c38a03e6 100644 --- a/skills/database-rds-devops/CHANGELOG.md +++ b/skills/database-rds-devops/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [1.0.1] - 2026-10-07 + +### Changed +- `metadata.version` in `SKILL.md` uses the full `MAJOR.MINOR.PATCH` form (`1.0` is now `1.0.1`). + ## [1.0.0] - 2026-07-16 ### Added diff --git a/skills/database-rds-devops/SKILL.md b/skills/database-rds-devops/SKILL.md index cb71df67..ebbb6a51 100644 --- a/skills/database-rds-devops/SKILL.md +++ b/skills/database-rds-devops/SKILL.md @@ -2,7 +2,7 @@ name: database-rds-devops description: "Database-level data-plane diagnostics for Aurora MySQL and Aurora PostgreSQL. Executes predefined read-only health check queries via RDS Data API to analyze buffer pool, connections, locks, replication, storage, performance, and index efficiency. Requires the rds-aidba MCP server for database-internal access beyond what CloudWatch and RDS APIs provide." metadata: - version: "1.0" + version: "1.0.1" author: kiranmam ---