diff --git a/.github/workflows/release-preflight.yml b/.github/workflows/release-preflight.yml index 1192045..55eadef 100644 --- a/.github/workflows/release-preflight.yml +++ b/.github/workflows/release-preflight.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: tag: - description: Proposed release tag (for example, v2.2.0) + description: Proposed release tag (for example, v2.3.0) required: true type: string push: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d78c30..9f1ee6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## Unreleased + +Release candidate for **v2.3.0**. + +- Adds `experiment-run/1.0`, a minimal observation record for one external + research-compute attempt: stable run/project identity, objective, executor/code + and environment references, effective parameters, named random seeds, lifecycle + status, artifacts, metrics, and producer identity. The file is explicitly data, + never an execution instruction or authorization. +- Adds `claim-evidence/1.0`, a reviewable claim with precise evidence links and + independent `epistemic_status` and `review_state` axes. Every explicit review + decision records `reviewed_at`; accepted claims require evidence and cannot remain + speculative; superseded claims retain history and identify their replacement. +- Adds synthetic XQ-17 examples, negative fixtures, stdlib validator dispatch, and + JSON Schema conformance coverage for both formats. +- Adds `specs/research-execution-and-evidence.md` and updates ownership/product + direction. External executors own run records; explicit human review owns claim + decisions; the thin entry neither executes code references nor approves claims. +- This is not a restoration of the broad `experiment-log/1.0` removed in v2.0.0. + `project/1.x` remains unchanged and has no `experiments` field. +- Records the implemented local Provenance ingestion of `parsed-paper/1.0`, + `reading-note/1.0`, `review/1.0`, and `lineage-graph/1.0`. Provenance ingestion + of the two new contracts remains a release gap. + ## v2.2.0 — 2026-07-20 - Replaces all public worked examples and domain-bearing invalid fixtures with a diff --git a/README.md b/README.md index acd6c70..91ebd1a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ English | [中文](README.zh.md) > The shared data contracts that let the Scriptorium suite's tools exchange files. -> **Product status: Public Alpha contract baseline.** The Public Alpha +> **Product status: Public Alpha contract release candidate.** The Public Alpha > target is Windows-first and requires at least one agent host. Codex and Claude > Code are the two first-class target choices; canonical installers now exist, while > Claude Code live `SessionEnd` golden-path parity remains a release gap. The user's Markdown workspace, @@ -51,13 +51,13 @@ The three suite tools (developed separately) are: The ownership ADR assigns Windows setup, diagnostics, the synthetic demo, and agent-task registration to the thin suite entrypoint; it does not become another data store. A local umbrella candidate now implements preview-first `init`, `doctor`, -`status`, explicit-root metadata-only `inventory`, `demo`, `pull`, canonical host -installers, and Windows CI. Adapter-specific reviewed migration execution, a -published installer/package, and external beta evidence remain gaps. +`status`, explicit-root metadata-only `inventory`, reviewed Markdown/PDF migration, +`demo`, `pull`, canonical host installers, and Windows CI. A published +installer/package, live-host acceptance, and external beta evidence remain gaps. ## Features -- **Ten exchange formats**, each a JSON Schema (Draft 2020-12): +- **Twelve exchange formats**, each a JSON Schema (Draft 2020-12): | Format | Schema | Produced by | Consumed by | |---|---|---|---| @@ -67,17 +67,20 @@ published installer/package, and external beta evidence remain gaps. | `project/1.x` | [schemas/project/v1.json](schemas/project/v1.json) | human / agent / Markdown-frontmatter adapter | Provenance portfolio/context, optional dashboards | | `note/1.x` | [schemas/note/v1.json](schemas/note/v1.json) | host sync layer or optional capture adapter | Provenance protected ingest/search | | `session-summary/1.x` | [schemas/session-summary/v1.json](schemas/session-summary/v1.json) | Codex / Claude Code host workflow | Provenance approval flow, project progress-log | - | `reading-note/1.x` | [schemas/reading-note/v1.json](schemas/reading-note/v1.json) | `read-paper` agent task | Steward renderers and file-based agent workflows; Provenance ingest is a release gap | - | `parsed-paper/1.x` | [schemas/parsed-paper/v1.json](schemas/parsed-paper/v1.json) | Steward `parse` (local GROBID) | `read-paper`, synthesis, Steward lineage | - | `lineage-graph/1.x` | [schemas/lineage-graph/v1.json](schemas/lineage-graph/v1.json) | Steward `lineage` + agent typing | Steward renderer and file-based agent workflows; Provenance ingest is a release gap | - | `review/1.x` | [schemas/review/v1.json](schemas/review/v1.json) | `synthesize-direction` agent task | Markdown/file output and agent workflows; Provenance ingest is a release gap | + | `reading-note/1.x` | [schemas/reading-note/v1.json](schemas/reading-note/v1.json) | `read-paper` agent task | Steward renderers, file-based agent workflows, Provenance | + | `parsed-paper/1.x` | [schemas/parsed-paper/v1.json](schemas/parsed-paper/v1.json) | Steward `parse` (local GROBID) | `read-paper`, synthesis, Steward lineage, Provenance | + | `lineage-graph/1.x` | [schemas/lineage-graph/v1.json](schemas/lineage-graph/v1.json) | Steward `lineage` + agent typing | Steward renderer, file-based agent workflows, Provenance | + | `review/1.x` | [schemas/review/v1.json](schemas/review/v1.json) | `synthesize-direction` agent task | Markdown/file output, agent workflows, Provenance | + | `experiment-run/1.x` | [schemas/experiment-run/v1.json](schemas/experiment-run/v1.json) | external compute executor or agent workflow | file-based agent workflows; Provenance ingest is a release gap | + | `claim-evidence/1.x` | [schemas/claim-evidence/v1.json](schemas/claim-evidence/v1.json) | agent/human review workflow | human review and file-based agent workflows; Provenance ingest is a release gap | Lectern currently consumes `handoff/1.x`; it does **not** directly consume -`library-kb/1.x`. Provenance currently ingests library/project/note/session data, -but reading-note/review/lineage ingestion has not shipped and is intentionally -listed above as a Public Alpha release gap. +`library-kb/1.x`. Provenance currently ingests library/project/note/session data +and the `parsed-paper`, `reading-note`, `review`, and `lineage-graph` research +artifacts. Ingestion of the new experiment-run and claim-evidence contracts remains +a release gap. -- **Convention specs** for versioning, the Markdown project portfolio, optional Obsidian export/layout, config-root layout, the event/sync layer (`sync-layer.md`), product direction (`product-direction.md`), literature automation (`literature-automation.md`), suite entry/ownership, and the trust model (`trust-model.md`). +- **Convention specs** for versioning, the Markdown project portfolio, optional Obsidian export/layout, config-root layout, the event/sync layer (`sync-layer.md`), product direction (`product-direction.md`), literature automation (`literature-automation.md`), research execution/evidence (`research-execution-and-evidence.md`), suite entry/ownership, and the trust model (`trust-model.md`). - **Worked examples** for every format under [`examples/`](examples), kept valid against the schemas. Every example and invalid test fixture belongs to a deliberately fictional XQ-17 demo universe. Names, papers, identifiers, paths, sessions, dates, and results do not describe real people or research. @@ -127,7 +130,9 @@ scriptorium-spec/ │ ├── reading-note/v1.json # per-paper staged interpretation (4 optional reading levels) │ ├── parsed-paper/v1.json # normalized local parse of a paper PDF (sections + refs + figures/tables) │ ├── lineage-graph/v1.json # a research direction's citation 脉络 (nodes + typed edges) -│ └── review/v1.json # direction synthesis (narrative sections + comparison table) +│ ├── review/v1.json # direction synthesis (narrative sections + comparison table) +│ ├── experiment-run/v1.json # observation of one external research-compute attempt +│ └── claim-evidence/v1.json # reviewable claim with precise evidence links ├── examples/ # valid examples per format, including compatibility variants ├── specs/ # convention documents │ ├── versioning.md # schema_version rules; ignore/preserve unknown fields @@ -140,6 +145,7 @@ scriptorium-spec/ │ ├── suite-entry-and-ownership.md # suite entrypoint + component boundaries │ ├── literature-automation.md # on-demand literature refresh (optional weekly opt-in) + digest │ ├── literature-reading.md # staged reading + direction synthesis +│ ├── research-execution-and-evidence.md # external runs + human-gated claims │ └── trust-model.md # suite safety/privacy guarantees by theme + honest limits ├── tools/ │ └── validate.py # minimal stdlib-only structural validator @@ -156,15 +162,16 @@ the entrypoint ownership ADR defines that Public Alpha release boundary. ## Status -**Public Alpha contract baseline: v2.2.0.** This baseline targets compatibility with -Scriptorium v0.1.0; it does not claim that every component tag is already published. -Cross-repository and Windows CI golden paths cover -the `init`/`doctor`/`status`/`inventory`/`demo`/`pull` entry and canonical host -installers, but adapter-specific reviewed migration execution, a packaged suite -installer, and external beta evidence remain product gaps. +**Public Alpha contract release candidate: v2.3.0.** This worktree adds the +experiment-run and claim-evidence contracts; it does not claim that a corresponding +tag or every component version is already published. Local cross-repository and +Windows acceptance paths cover the `init`/`doctor`/`status`/`inventory`/reviewed +migration/`demo`/`pull` entry, canonical host installers, and an install lifecycle, +but fresh remote CI, live-host acceptance, a packaged suite installer, and external +beta evidence remain product gaps. The event/sync-layer contracts (`note/1.0`, `session-summary/1.0`) are -implemented in Provenance; Provenance ingestion of parsed-paper/reading-note/review/ -lineage is not yet implemented. +implemented in Provenance, as is local ingestion of parsed-paper/reading-note/review/ +lineage. Provenance ingestion of experiment-run/claim-evidence is not yet implemented. ## License diff --git a/README.zh.md b/README.zh.md index ebd77ce..9364072 100644 --- a/README.zh.md +++ b/README.zh.md @@ -4,7 +4,7 @@ > 让 Scriptorium 套件各工具彼此交换文件的共享数据契约。 -> **产品状态：Public Alpha 契约基线。** Public Alpha 目标以 Windows +> **产品状态：Public Alpha 契约发布候选。** Public Alpha 目标以 Windows > 为首发平台，并要求至少选择一个 agent 宿主；Codex 与 Claude Code 是地位相同的一等 > 目标宿主；canonical installer 已实现，但 Claude Code 的 live `SessionEnd` golden > path 对等验证仍是 release gap。用户的 @@ -48,13 +48,13 @@ JSON Schema、约定文档、可用示例以及一个 所有权 ADR 将 Windows 配置、诊断、合成 demo 与 agent task 注册归于薄套件入口； 该入口本身不成为新的数据存储。本地 umbrella 候选版现已实现默认预览的 `init`、 -`doctor`、`status`、仅盘点显式来源和元数据的 `inventory`、`demo`、`pull`、 -canonical host installer 与 Windows CI；适配器级的人审迁移执行、正式发布的安装包 -以及外部 beta 证据仍是 release gap。 +`doctor`、`status`、仅盘点显式来源和元数据的 `inventory`、经人审的 Markdown/PDF +迁移、`demo`、`pull`、canonical host installer 与 Windows CI；正式发布的安装包、 +真实宿主验收以及外部 beta 证据仍是 release gap。 ## 特性 -- **十种交换格式**，每种都是一份 JSON Schema（Draft 2020-12）： +- **十二种交换格式**，每种都是一份 JSON Schema（Draft 2020-12）： | 格式 | Schema | 生产者 | 消费者 | |---|---|---|---| @@ -64,16 +64,19 @@ canonical host installer 与 Windows CI；适配器级的人审迁移执行、 | `project/1.x` | [schemas/project/v1.json](schemas/project/v1.json) | 人工 / agent / Markdown frontmatter 适配器 | Provenance 项目组合/上下文、可选看板 | | `note/1.x` | [schemas/note/v1.json](schemas/note/v1.json) | 宿主同步层或可选捕获适配器 | Provenance 受保护摄取/搜索 | | `session-summary/1.x` | [schemas/session-summary/v1.json](schemas/session-summary/v1.json) | Codex / Claude Code 宿主工作流 | Provenance 审批流、项目进展日志 | - | `reading-note/1.x` | [schemas/reading-note/v1.json](schemas/reading-note/v1.json) | `read-paper` agent task | Steward 渲染器与文件型 agent 工作流；Provenance 摄取仍是 release gap | - | `parsed-paper/1.x` | [schemas/parsed-paper/v1.json](schemas/parsed-paper/v1.json) | Steward `parse`（本地 GROBID） | `read-paper`、综述、Steward lineage | - | `lineage-graph/1.x` | [schemas/lineage-graph/v1.json](schemas/lineage-graph/v1.json) | Steward `lineage` + agent 标注关系类型 | Steward 渲染器与文件型 agent 工作流；Provenance 摄取仍是 release gap | - | `review/1.x` | [schemas/review/v1.json](schemas/review/v1.json) | `synthesize-direction` agent task | Markdown/文件输出与 agent 工作流；Provenance 摄取仍是 release gap | + | `reading-note/1.x` | [schemas/reading-note/v1.json](schemas/reading-note/v1.json) | `read-paper` agent task | Steward 渲染器、文件型 agent 工作流、Provenance | + | `parsed-paper/1.x` | [schemas/parsed-paper/v1.json](schemas/parsed-paper/v1.json) | Steward `parse`（本地 GROBID） | `read-paper`、综述、Steward lineage、Provenance | + | `lineage-graph/1.x` | [schemas/lineage-graph/v1.json](schemas/lineage-graph/v1.json) | Steward `lineage` + agent 标注关系类型 | Steward 渲染器、文件型 agent 工作流、Provenance | + | `review/1.x` | [schemas/review/v1.json](schemas/review/v1.json) | `synthesize-direction` agent task | Markdown/文件输出、agent 工作流、Provenance | + | `experiment-run/1.x` | [schemas/experiment-run/v1.json](schemas/experiment-run/v1.json) | 外部计算执行器或 agent 工作流 | 文件型 agent 工作流；Provenance 摄取仍是 release gap | + | `claim-evidence/1.x` | [schemas/claim-evidence/v1.json](schemas/claim-evidence/v1.json) | agent/人工审核工作流 | 人工审核与文件型 agent 工作流；Provenance 摄取仍是 release gap | Lectern 当前消费 `handoff/1.x`，**不会**直接消费 `library-kb/1.x`。Provenance 当前已实现 -文献库、项目、笔记和会话数据的摄取；reading-note/review/lineage 摄取尚未发布，因此在上表 -明确列为 Public Alpha release gap。 +文献库、项目、笔记和会话数据，以及 `parsed-paper`、`reading-note`、`review`、 +`lineage-graph` 研究工件的本地摄取；新增的 experiment-run/claim-evidence 摄取仍是 +Public Alpha release gap。 -- **约定文档**：覆盖版本号规则、Markdown 项目组合、可选 Obsidian 导出/布局、config-root、事件/同步层（`sync-layer.md`）、产品方向（`product-direction.md`）、文献自动化（`literature-automation.md`）、套件入口/所有权与信任模型（`trust-model.md`）。 +- **约定文档**：覆盖版本号规则、Markdown 项目组合、可选 Obsidian 导出/布局、config-root、事件/同步层（`sync-layer.md`）、产品方向（`product-direction.md`）、文献自动化（`literature-automation.md`）、研究执行与证据（`research-execution-and-evidence.md`）、套件入口/所有权与信任模型（`trust-model.md`）。 - **可用示例**：[`examples/`](examples) 下每种格式都有示例，并保持与 schema 一致有效。 所有示例和无效测试夹具均属于刻意虚构的 XQ-17 演示世界；其中的人名、论文、标识符、路径、 会话、日期与结果均不描述任何真实人物或科研工作。 @@ -120,7 +123,9 @@ scriptorium-spec/ │ ├── reading-note/v1.json # 单篇论文分级解读（4 个可选阅读层级） │ ├── parsed-paper/v1.json # 论文 PDF 的规范化本地解析（章节 + 参考文献 + 图表） │ ├── lineage-graph/v1.json # 研究方向的引用脉络（节点 + 带类型的边） -│ └── review/v1.json # 方向综述（叙事章节 + 对比表） +│ ├── review/v1.json # 方向综述（叙事章节 + 对比表） +│ ├── experiment-run/v1.json # 一次外部科研计算尝试的观察记录 +│ └── claim-evidence/v1.json # 可审核断言及其精确证据链接 ├── examples/ # 每种格式的有效示例，含兼容性版本变体 ├── specs/ # 约定文档 │ ├── versioning.md # schema_version 规则；忽略并保留未知字段 @@ -133,6 +138,7 @@ scriptorium-spec/ │ ├── suite-entry-and-ownership.md # 套件入口与组件边界 │ ├── literature-automation.md # 按需文献刷新（可选每周 opt-in）+ 库内新进展 digest │ ├── literature-reading.md # 分阶段阅读 + 方向脉络综述 +│ ├── research-execution-and-evidence.md # 外部实验记录 + 人审断言 │ └── trust-model.md # 套件安全/隐私保证（按主题）+ 诚实的边界说明 ├── tools/ │ └── validate.py # 零依赖、仅标准库的结构校验器 @@ -148,13 +154,13 @@ Steward 的源码包名为 `scriptorium-steward`，CLI 为 `steward`。Provenanc ## 状态 -**Public Alpha 契约基线：v2.2.0。** 该基线以 Scriptorium v0.1.0 为兼容目标， -不表示所有组件 tag 已经发布。跨仓与 Windows CI golden path 已覆盖 -`init`/`doctor`/`status`/`inventory`/`demo`/ -`pull` 入口及 canonical host installer，但适配器级的人审迁移执行、套件安装包与 +**Public Alpha 契约发布候选：v2.3.0。** 当前工作树新增 experiment-run 与 +claim-evidence 契约，不表示对应 tag 或所有组件版本已经发布。本地跨仓与 Windows +验收路径已覆盖 `init`/`doctor`/`status`/`inventory`/经人审迁移/`demo`/`pull`、 +canonical host installer 与安装生命周期；全新远端 CI、真实宿主验收、套件安装包与 外部 beta 证据仍是产品缺口。事件/同步层契约 -（`note/1.0`、`session-summary/1.0`）已在 Provenance 实现；parsed-paper/reading-note/ -review/lineage 摄取尚未实现。 +（`note/1.0`、`session-summary/1.0`）及 parsed-paper/reading-note/review/lineage +本地摄取已在 Provenance 实现；experiment-run/claim-evidence 摄取尚未实现。 ## License diff --git a/examples/claim-evidence.v1.example.json b/examples/claim-evidence.v1.example.json new file mode 100644 index 0000000..4365c25 --- /dev/null +++ b/examples/claim-evidence.v1.example.json @@ -0,0 +1,22 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-0001", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] The generated XQ-17 fixture produced a synthetic-error value of 0.125 in run 0001; this is not a real scientific result.", + "kind": "observation", + "epistemic_status": "supported", + "review_state": "accepted", + "evidence": [ + { + "relation": "supports", + "source_type": "experiment-run", + "source_id": "synthetic-xq17-run-0001", + "locator": "metrics[synthetic-error]", + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "note": "[SYNTHETIC] The metric is read from the generated output artifact." + } + ], + "reviewed_at": "2100-01-04T10:05:00Z", + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/examples/experiment-run.v1.example.json b/examples/experiment-run.v1.example.json new file mode 100644 index 0000000..aba4e41 --- /dev/null +++ b/examples/experiment-run.v1.example.json @@ -0,0 +1,47 @@ +{ + "schema_version": "experiment-run/1.0", + "run_id": "synthetic-xq17-run-0001", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T09:30:00Z", + "status": "succeeded", + "objective": "[SYNTHETIC] Exercise the experiment-run contract against generated XQ-17 rows.", + "execution": { + "runner": "python", + "code_ref": "git:demo0001#scripts/run_synthetic_xq17.py", + "environment_ref": "lock:synthetic-environment-0001", + "parameters": { + "generated_rows": 32, + "calibration_mode": "synthetic" + }, + "random_seeds": { + "python": 1701, + "numpy": 1701 + } + }, + "artifacts": [ + { + "artifact_id": "synthetic-xq17-input-0001", + "role": "input", + "locator": "fixtures/generated-xq17.csv", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "media_type": "text/csv" + }, + { + "artifact_id": "synthetic-xq17-metrics-0001", + "role": "output", + "locator": "results/generated-xq17-metrics.json", + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "media_type": "application/json" + } + ], + "metrics": [ + { + "name": "synthetic-error", + "value": 0.125, + "unit": "generated-units", + "direction": "minimize" + } + ], + "source_session_id": "demo-session-0001", + "generated_by": "synthetic-executor/0.1.0" +} diff --git a/pyproject.toml b/pyproject.toml index 99232dc..ac2d726 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scriptorium-spec" -version = "2.2.0" +version = "2.3.0" description = "Shared data contracts for the Scriptorium suite" requires-python = "==3.12.*" dependencies = [] diff --git a/schemas/claim-evidence/v1.json b/schemas/claim-evidence/v1.json new file mode 100644 index 0000000..1e67bf4 --- /dev/null +++ b/schemas/claim-evidence/v1.json @@ -0,0 +1,154 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/scriptorium-suite/scriptorium-spec/blob/main/schemas/claim-evidence/v1.json", + "title": "Scriptorium Claim Evidence v1", + "description": "One reviewable research claim and its precise evidence links. epistemic_status describes what the current evidence indicates; review_state separately records human governance. A draft or rejected record never becomes an authoritative project fact merely because it validates. See specs/research-execution-and-evidence.md.", + "type": "object", + "required": [ + "schema_version", + "claim_id", + "project", + "created", + "statement", + "epistemic_status", + "review_state", + "evidence", + "generated_by" + ], + "properties": { + "schema_version": { "type": "string", "pattern": "^claim-evidence/1\\.[0-9]+$" }, + "claim_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "Stable idempotency key for this claim record." + }, + "project": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "Owning project_id (project/1.x)." + }, + "created": { + "type": "string", + "format": "date-time" + }, + "statement": { + "type": "string", + "minLength": 1, + "description": "A bounded proposition that can be supported, qualified, or contradicted." + }, + "kind": { + "type": "string", + "enum": ["observation", "hypothesis", "interpretation", "limitation", "decision"] + }, + "epistemic_status": { + "type": "string", + "enum": [ + "speculative", + "partially-supported", + "supported", + "contradicted", + "mixed", + "unresolved" + ], + "description": "Evidence state, independent from review_state." + }, + "review_state": { + "type": "string", + "enum": ["draft", "accepted", "rejected", "superseded"], + "description": "Human governance state. Only an explicit review workflow may set accepted/rejected/superseded." + }, + "evidence": { + "type": "array", + "description": "Precise links into source records. Accepted claims require at least one entry.", + "items": { + "type": "object", + "required": ["relation", "source_type", "source_id", "locator"], + "properties": { + "relation": { + "type": "string", + "enum": ["supports", "contradicts", "qualifies", "context"] + }, + "source_type": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$", + "description": "Contract or source class, such as experiment-run, parsed-paper, reading-note, or artifact." + }, + "source_id": { + "type": "string", + "minLength": 1, + "description": "Stable id of the source record or artifact." + }, + "locator": { + "type": "string", + "minLength": 1, + "description": "Precise in-source location, such as a metric name, section id, page, figure, table, or paragraph." + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "Optional lowercase SHA-256 digest of the referenced immutable artifact." + }, + "note": { + "type": "string", + "description": "Short explanation of how the source bears on the claim; not a replacement for the source." + } + } + } + }, + "reviewed_at": { + "type": "string", + "format": "date-time", + "description": "Required for every explicit accepted, rejected, or superseded review decision." + }, + "superseded_by": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "Required when review_state is superseded." + }, + "generated_by": { + "type": "string", + "minLength": 1, + "description": "Producer name and version." + } + }, + "allOf": [ + { + "if": { + "properties": { + "review_state": { + "enum": ["accepted", "rejected", "superseded"] + } + }, + "required": ["review_state"] + }, + "then": { "required": ["reviewed_at"] } + }, + { + "if": { + "properties": { "review_state": { "const": "accepted" } }, + "required": ["review_state"] + }, + "then": { + "properties": { + "epistemic_status": { + "enum": [ + "partially-supported", + "supported", + "contradicted", + "mixed", + "unresolved" + ] + }, + "evidence": { "minItems": 1 } + } + } + }, + { + "if": { + "properties": { "review_state": { "const": "superseded" } }, + "required": ["review_state"] + }, + "then": { "required": ["superseded_by"] } + } + ] +} diff --git a/schemas/experiment-run/v1.json b/schemas/experiment-run/v1.json new file mode 100644 index 0000000..58a95d6 --- /dev/null +++ b/schemas/experiment-run/v1.json @@ -0,0 +1,150 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/scriptorium-suite/scriptorium-spec/blob/main/schemas/experiment-run/v1.json", + "title": "Scriptorium Experiment Run v1", + "description": "A traceable observation of one research-compute attempt. The record describes what an external executor ran and observed; it is data, never an instruction or authorization to execute code. Producers may update a running record, but a terminal succeeded/failed/cancelled record is treated as an immutable result identified by run_id. See specs/research-execution-and-evidence.md.", + "type": "object", + "required": [ + "schema_version", + "run_id", + "project", + "created", + "status", + "objective", + "execution", + "generated_by" + ], + "properties": { + "schema_version": { "type": "string", "pattern": "^experiment-run/1\\.[0-9]+$" }, + "run_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "Stable idempotency key for this run." + }, + "project": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "Owning project_id (project/1.x)." + }, + "created": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp when the run record was created." + }, + "status": { + "type": "string", + "enum": ["running", "succeeded", "failed", "cancelled"] + }, + "objective": { + "type": "string", + "minLength": 1, + "description": "The bounded question or evaluation objective for this run." + }, + "execution": { + "type": "object", + "required": ["runner", "code_ref", "environment_ref", "parameters", "random_seeds"], + "properties": { + "runner": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$", + "description": "Executor class, such as python, jupyter, container, or manual." + }, + "code_ref": { + "type": "string", + "minLength": 1, + "description": "Stable code/version reference plus an entrypoint locator; never a command to execute." + }, + "environment_ref": { + "type": "string", + "minLength": 1, + "description": "Lockfile, image digest, or other reproducible environment reference." + }, + "parameters": { + "type": "object", + "description": "Non-secret effective run parameters. Values may be nested JSON; credentials and private absolute paths are forbidden." + }, + "random_seeds": { + "type": "object", + "description": "Named random seeds used by each stochastic subsystem; use an empty object for a deterministic run.", + "propertyNames": { "pattern": "^[a-z][a-z0-9_-]*$" }, + "additionalProperties": { + "oneOf": [ + { "type": "integer" }, + { "type": "string", "minLength": 1 } + ] + } + } + } + }, + "artifacts": { + "type": "array", + "description": "Stable input/output/log references. Contract files should prefer logical ids and workspace-relative locators over absolute local paths.", + "items": { + "type": "object", + "required": ["artifact_id", "role"], + "properties": { + "artifact_id": { + "type": "string", + "minLength": 1, + "description": "Stable logical identifier for the artifact." + }, + "role": { + "type": "string", + "enum": ["input", "output", "log"] + }, + "locator": { + "type": "string", + "minLength": 1, + "description": "Optional workspace-relative or producer-defined logical locator; no secret-bearing URL." + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "Optional lowercase SHA-256 content digest." + }, + "media_type": { "type": "string" } + } + } + }, + "metrics": { + "type": "array", + "description": "Machine-observed evaluation values. A metric is evidence, not by itself an approved scientific claim.", + "items": { + "type": "object", + "required": ["name", "value"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string" }, + "direction": { + "type": "string", + "enum": ["minimize", "maximize", "target", "none"] + } + } + } + }, + "failure_reason": { + "type": "string", + "minLength": 1, + "description": "Required for a failed run; redact credentials and private paths." + }, + "source_session_id": { + "type": "string", + "description": "Optional supported agent-host session id that initiated or inspected the run." + }, + "generated_by": { + "type": "string", + "minLength": 1, + "description": "Producer name and version." + } + }, + "allOf": [ + { + "if": { + "properties": { "status": { "const": "failed" } }, + "required": ["status"] + }, + "then": { "required": ["failure_reason"] } + } + ] +} diff --git a/schemas/lineage-graph/v1.json b/schemas/lineage-graph/v1.json index d50d020..573d820 100644 --- a/schemas/lineage-graph/v1.json +++ b/schemas/lineage-graph/v1.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/scriptorium-suite/scriptorium-spec/blob/main/schemas/lineage-graph/v1.json", "title": "Scriptorium Lineage Graph v1", - "description": "A research direction's 脉络 / lineage over a SET of own-library papers, built in two passes. (1) DETERMINISTIC: Steward `lineage` reads each paper's parsed-paper/1.0, emits one `node` per paper from its metadata, and walks the parsed `references` to add a `cites` edge wherever a reference matches another paper IN THE SET (by DOI when present, else normalized title) — `evidence` is that matched reference's raw string — plus a year-ordered `timeline`. (2) AGENT-ENRICHED: the `synthesize-direction` task may add typed relations (extends / supersedes / method-of / contrasts) and `cluster` assignments. Own-library only to start (no beyond-library / OpenAlex edges). Nodes join the rest of the suite by `citekey` = the Better BibTeX id. Steward can render a Markdown projection; Obsidian enhancements are optional. Direct Provenance ingestion is a Public Alpha release gap. Additive contract — supersedes nothing. See specs/literature-reading.md §3.", + "description": "A research direction's 脉络 / lineage over a SET of own-library papers, built in two passes. (1) DETERMINISTIC: Steward `lineage` reads each paper's parsed-paper/1.0, emits one `node` per paper from its metadata, and walks the parsed `references` to add a `cites` edge wherever a reference matches another paper IN THE SET (by DOI when present, else normalized title) — `evidence` is that matched reference's raw string — plus a year-ordered `timeline`. (2) AGENT-ENRICHED: the `synthesize-direction` task may add typed relations (extends / supersedes / method-of / contrasts) and `cluster` assignments. Own-library only to start (no beyond-library / OpenAlex edges). Nodes join the rest of the suite by `citekey` = the Better BibTeX id. Steward can render a Markdown projection; Obsidian enhancements are optional. Provenance locally ingests validated files as reference-only research artifacts, not approved claims. Additive contract — supersedes nothing. See specs/literature-reading.md §3.", "type": "object", "required": ["schema_version", "direction", "nodes", "edges"], "properties": { diff --git a/schemas/reading-note/v1.json b/schemas/reading-note/v1.json index f959a7c..b4dfa1a 100644 --- a/schemas/reading-note/v1.json +++ b/schemas/reading-note/v1.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/scriptorium-suite/scriptorium-spec/blob/main/schemas/reading-note/v1.json", "title": "Scriptorium Reading Note v1", - "description": "A per-paper staged interpretation produced by the read-paper agent task as you read at the depth you need. The four OPTIONAL stages are glance / close_read / deep_read / situate, each filled on demand. Joined to the rest of the suite by `id` = the Better BibTeX citekey. Steward can render it to Markdown; Obsidian is an optional client. Direct Provenance ingestion is a Public Alpha release gap, not a shipped consumer. Additive contract — supersedes nothing. See specs/literature-reading.md §3.", + "description": "A per-paper staged interpretation produced by the read-paper agent task as you read at the depth you need. The four OPTIONAL stages are glance / close_read / deep_read / situate, each filled on demand. Joined to the rest of the suite by `id` = the Better BibTeX citekey. Steward can render it to Markdown; Obsidian is an optional client. Provenance locally ingests validated files as reference-only research artifacts, not approved claims. Additive contract — supersedes nothing. See specs/literature-reading.md §3.", "type": "object", "required": ["schema_version", "id", "created"], "properties": { diff --git a/schemas/review/v1.json b/schemas/review/v1.json index 1a191bb..f36ff65 100644 --- a/schemas/review/v1.json +++ b/schemas/review/v1.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/scriptorium-suite/scriptorium-spec/blob/main/schemas/review/v1.json", "title": "Scriptorium Review v1", - "description": "A research-direction synthesis (narrative + comparison table) produced by the synthesize-direction agent task in answer to a question/topic. Outline -> draft -> critique narrative across `sections`, an optional paper-by-paper `comparison_table`, identified `gaps`, and `priority_reads`. Anti-fabrication: the comparison table and citations MUST be grounded in parsed-paper / library-kb, never invented by the model. It may reference a lineage-graph via `lineage_ref` and be materialized as Markdown. Direct Provenance ingestion is a Public Alpha release gap. Additive contract — supersedes nothing. See specs/literature-reading.md §3.", + "description": "A research-direction synthesis (narrative + comparison table) produced by the synthesize-direction agent task in answer to a question/topic. Outline -> draft -> critique narrative across `sections`, an optional paper-by-paper `comparison_table`, identified `gaps`, and `priority_reads`. Anti-fabrication: the comparison table and citations MUST be grounded in parsed-paper / library-kb, never invented by the model. It may reference a lineage-graph via `lineage_ref` and be materialized as Markdown. Provenance locally ingests validated files as reference-only research artifacts, not approved claims. Additive contract — supersedes nothing. See specs/literature-reading.md §3.", "type": "object", "required": ["schema_version", "direction", "sections"], "properties": { diff --git a/specs/literature-reading.md b/specs/literature-reading.md index 4a9dc0f..d8f6343 100644 --- a/specs/literature-reading.md +++ b/specs/literature-reading.md @@ -1,12 +1,13 @@ # Literature-Reading Expansion — Design `[P1–P3-core + P4-core ratified & built · P3+/P4-tail remaining]` -> **Status (2026-06-22):** architecture approved; **P1–P3-core + P4-core ratified & built.** +> **Status (amended 2026-07-23):** architecture approved; **P1–P3-core + P4-core ratified & built.** > The contracts sketched here are now ratified additive schemas under `schemas/` > (`reading-note/1.0`, `parsed-paper/1.0`, `lineage-graph/1.0`, `review/1.0`), with > the Steward CLI (`parse`, `lineage`, `lineage-render`) + skills (`read-paper`, > `synthesize-direction`) shipped. **Remaining:** P3+ local-embedding recall, P4 tail -> (OpenAlex beyond-library 谱系), direct Provenance ingestion of parsed/reading/review/ -> lineage artifacts, and canonical Codex/Claude Code task installers. The current +> (OpenAlex beyond-library 谱系), parsed-section full-text/semantic indexing, and +> canonical Codex/Claude Code task installers. Provenance now locally ingests validated +> parsed/reading/review/lineage artifacts as reference-only research context. The current > skill files are usable from their existing host-specific location; host parity is > a Public Alpha release gap. This EXTENDS > the literature flow (Steward `library-kb` + `review`, see @@ -32,8 +33,8 @@ or other host sends the task context under that host's provider settings. |---|---|---| | **Read rail — local Zotero MCP** (`zotero-mcp`, read-only, local API/no-key, semantic search local-or-off) | the agent's live window into the library (metadata / full-text / annotations / notes); complements Steward's batch `library-kb` | adopt (plugin) + a registration note | | **Human reading — PDF++** (optional Obsidian integration) | in-vault PDF read + **markdown** annotations (survive the plugin; file-contract-clean) | optional adopted plugin | -| **Parse layer — GROBID** (optional richer local parser later) | PDF → structured files (sections + reference list), **local** | **Steward** (`parse`) — emits contract files; direct Provenance ingestion is not implemented | -| **Index — current + target** | current Provenance FTS5 indexes `library-kb`; parsed sections and optional local embeddings are the P3+ target | **Provenance** — parsed/reading/review/lineage ingestion and embeddings remain release gaps | +| **Parse layer — GROBID** (optional richer local parser later) | PDF → structured files (sections + reference list), **local** | **Steward** (`parse`) emits contract files; Provenance locally ingests validated files as reference-only research artifacts | +| **Index — current + target** | current Provenance indexes library/project/session data and imports parsed/reading/review/lineage records; parsed-section full-text and optional local embeddings are the P3+ target | **Provenance** — contract ingestion is implemented; richer parsed-section/semantic recall remains a release gap | | **Generation = the agent, in-session** (NO LLM client in suite code) | Codex or Claude Code writes the reading/synthesis prose; cloud egress, if any, belongs to the selected host | current host-specific tasks; the current thin entry does not yet package canonical two-host literature-task installers | | **Lineage — own-library** (GROBID reference edges → typed relations → graph file) | the direction 脉络 / lineage | **Steward** (`lineage`) | | **Agent hosts — Codex / Claude Code** | drive staged reading + synthesis over the same files and public tools | both are first-class targets; installer parity remains a release gap | @@ -137,16 +138,17 @@ edge list, so they never disagree. - **`synthesize-direction` skill** — input a question/topic; the agent scopes papers (MCP search + `library-kb`; optional local-embedding recall after P3+), retrieve→rank (PaperQA2 shape), - and builds the contract outputs. Direct Provenance archival of review/lineage files - is not implemented. *[current host-specific task]* + and builds the contract outputs. Provenance locally ingests validated review/lineage + files as reference-only research artifacts. *[current host-specific task]* - **`steward parse`** — PDF → structured file through the local GROBID backend and a pluggable parser seam; Docling/MinerU backends are not bundled. *[Steward CLI]* - **`steward lineage`** — build a `lineage-graph` from parsed reference lists (own-library). *[Steward CLI]* - **`steward lineage-render`** — deterministically project a `lineage-graph/1.0` into the self-contained `Reviews/.lineage.md` (native-Obsidian Mermaid + timeline + Dataview edge table); `--stamp-notes` opportunistically writes Breadcrumbs frontmatter onto already-materialized paper notes (merge-not-clobber). *[Steward CLI]* - **`steward read-render`** — deterministically project a `reading-note/1.0` into a browsable, self-contained Obsidian note `reading-notes/.md` (YAML frontmatter + a `## ` section per FILLED stage — 速览/精读/深读/串联定位 — + situate `lineage_refs` as `[[citekey]]` wikilinks + a `## 标注 · Annotations` section — highlight text inlines as a blockquote, a URI/path reference as a link — + a links block); a `close_read.figures[]` image path inlines as an Obsidian embed `![[path]]` (caption text otherwise — render-support only: actual figure images need a layout parser that emits image files, e.g. local MinerU; GROBID/parsed-paper carry caption text only). Optional `--kb` enriches the frontmatter with title/authors/year from `library-kb/1.x` (reading-note carries none). The `.json` stays the machine SSoT; the `.md` is re-rendered idempotently. The `[[citekey]]` library link resolves because `steward export` writes a `citekey` alias onto each `Literature/` note (from the item's `Citation Key:` extra-field line → the optional `citekey` introduced in `library-kb/1.1`; historical 1.0 snapshots remain readable). *[Steward CLI]* - **`steward read-index`** — scan a vault's `reading-notes/*.json` and emit one overview note `reading-notes/_index.md` (the reading status dashboard): status-grouped wikilink sections (To Read / In Progress / Read / Not Reading / no-status), a full table (citekey · title · year · status · stages · tags), and a Dataview block (static views serve non-plugin users). Deterministic, idempotent overwrite of `_index.md` only; the per-paper `.json` files stay the SSoT. *[Steward CLI]* -- **Index** — current FTS5 covers `library-kb`; parsed-section indexing and optional - SentenceTransformers embeddings are planned P3+ work. *[Provenance release gap]* +- **Index** — current local ingestion covers the four research-artifact contracts; + parsed-section full-text indexing and optional SentenceTransformers embeddings are + planned P3+ work. *[Provenance release gap]* - **Provider boundary** — the selected agent host owns generation egress; optional local retrieval models, when implemented, stay on-device. *[shared contract]* @@ -166,7 +168,8 @@ edge list, so they never disagree. 5. **Lineage** — GROBID reference edges among the scoped papers → typed relations → `lineage-graph/1.0`. 6. **Synthesize** — outline → draft → critique (AutoSurvey shape) → `review/1.0` (narrative + comparison table), anti-fabrication. 7. **Land** — write review + lineage contract files and optional Markdown/Obsidian - projections; link reading notes. Direct Provenance ingestion remains a release gap. + projections; link reading notes; optionally ingest the validated contract files into + Provenance as reference-only research context. ## 6. R17 revisit + trust-model impact - **Local embeddings are allowed but not shipped.** The P3+ target may use an optional @@ -190,10 +193,11 @@ edge list, so they never disagree. - **P4-core · lineage render** ✅ *(shipped 2026-06-22)* — deterministic `steward lineage-render` projects an (agent-enriched) `lineage-graph/1.0` into a self-contained, tool-owned `Reviews/.lineage.md`: a **native-Obsidian Mermaid** graph (zero plugin), a year-ordered timeline, and a Dataview edge table. Breadcrumbs/ExcaliBrain per-note frontmatter is the **opportunistic** enhancement (`--stamp-notes`, written only onto already-materialized paper notes). *(Render is a projection of the existing `lineage-graph/1.0`, not a new contract — see §3 "Lineage render".)* - **P4 (tail) · beyond-library 谱系** — (later, optional) OpenAlex for lineage edges beyond your own library. -## 8. Resolved design targets and current gaps (2026-07-15) -1. **Index home target = Provenance.** Current FTS5 covers library metadata. Parsed - sections, reading notes, reviews, lineage graphs, and optional embeddings are not yet - ingested; each remains a Public Alpha release gap. +## 8. Resolved design targets and current gaps (2026-07-23) +1. **Index home = Provenance.** Validated parsed-paper, reading-note, review, and + lineage-graph files are now ingested as reference-only research artifacts. Rich + parsed-section full-text indexing and optional embeddings remain Public Alpha gaps; + ingestion never promotes these artifacts into approved claims. 2. **Task home migration.** The current implementations live in Steward's host-specific `.claude/skills/` directory. Canonical task content and both Codex/ Claude Code literature-task installers are not yet packaged by the current umbrella entry; host folders become diff --git a/specs/product-direction.md b/specs/product-direction.md index 15b8b3d..c21daaa 100644 --- a/specs/product-direction.md +++ b/specs/product-direction.md @@ -1,8 +1,9 @@ # Scriptorium 产品方向决议（Public Alpha 阶段宪法） -> 状态：已确认 v2 · 2026-07-15 · 本文件是 Public Alpha 阶段所有细化工作的依据。 +> 状态：已确认 v3 · 2026-07-23 · 本文件是 Public Alpha 阶段所有细化工作的依据。 > 演化（2026-06-22）：**R5「实时=自动·零手动」已换挡到「agent 原生·按需 pull」**——后台守护进程/无人值守摘要器全退役，改由用户/agent 在会话里按需驱动（可用的捕获钩子保留）。 > 演化（2026-07-15）：目标从仅作者自用扩展为面向外部技术型研究者的 Public Alpha；采用薄入口、分域 master、Codex + Claude Code 双宿主、可选集成，并明确 Engram 不属于官方运行组件。 +> 演化（2026-07-23）：增加外部科研执行与断言证据层；`experiment-run/1.0` 只记录执行观察、绝不授权执行，`claim-evidence/1.0` 将证据状态与人工审核状态分离。 > 角色约定：Codex 与 Claude Code 均为受支持的 agent 宿主；对话中文、代码/标识符英文。 > 关联：[suite-entry-and-ownership.md](suite-entry-and-ownership.md)（入口、所有权与宿主 ADR）· [vault-layout.md](vault-layout.md)（Markdown workspace）· [trust-model.md](trust-model.md)（信任边界）。跨项目历史见 Provenance MCP。 > 标记：`[已确认]` = 用户已拍板；`[草案]` = PM 拟、待用户增补/否决。 @@ -15,7 +16,7 @@ - **R2 产品身份** `[已确认]` Scriptorium 是薄入口 + agent 宿主 + 可独立使用的组件集合：核心文件契约统一，运行时保持松耦合；完整 Public Alpha 必须选择 Codex、Claude Code 或同时选择二者，二者一等支持；文献、Obsidian 体验与出片按可选 capability profile 组合，不做单体应用。 ## 2. 架构 -- **R3 分域 master** `[已确认]` 每类持久数据只有一个 master：Zotero（启用时）拥有文献源记录；Markdown 文件拥有人写项目事实与获批工件；host sync layer 拥有项目笔记中的自动 progress-log marker 区；Provenance 仅是权威的跨会话记忆/检索层；Steward 产出的契约文件和 Lectern 交付物各自保持文件级 master。薄入口不拥有业务数据。 +- **R3 分域 master** `[已确认]` 每类持久数据只有一个 master：Zotero（启用时）拥有文献源记录；Markdown 文件拥有人写项目事实与获批工件；host sync layer 拥有项目笔记中的自动 progress-log marker 区；外部计算执行器拥有原始执行输出及其 `experiment-run/1.x` 记录；显式人工审核工作流拥有 `claim-evidence/1.x` 的审核决定；Provenance 仅是权威的跨会话记忆/检索层；Steward 产出的契约文件和 Lectern 交付物各自保持文件级 master。薄入口不拥有业务数据。 - **R4 工作台与边界** `[已确认]` plain Markdown workspace = 核心人机工作台；Obsidian 是可选客户端而非运行时依赖。workspace 按 master 分区：**人写区**（用户拥有，摄取进 Provenance）/ **工具派生区**（生成工具拥有，可重建）。每条信息单一 master、视图永远派生。 - **R5 同步模型** `[已确认]` 捕获适配器可事件入队，处理统一为短生命周期、显式的 on-demand pull。Claude Code 可用 enqueue-only `SessionEnd` 钩子；Codex 可用本地日志扫描；两者在 pull 后进入同一 worker 与审批流。不得宣称所有宿主具有相同的自动钩子，也不做亚秒级常驻守护进程。 @@ -32,7 +33,8 @@ - **R11 出片** `[已确认]` Lectern = 官方但可选的 Slides profile，通过 `handoff/1.x` 消费文件；核心套件不依赖它，**不纳入 PPT-Agent**。 - **R12 可选集成** `[已确认]` Obsidian、Zotero、browser extension、Lectern、GROBID、Better BibTeX、PDF++、Dataview 均按能力 profile 启用；缺失时只降级对应能力，不得使 Public Alpha core 的其余能力失效。 - **R13 文献进展摘要** `[已确认]` Steward 已提供库内 review/reading/lineage 的契约与确定性投影；Public Alpha 只承诺已有 Zotero/本地文献链。RSS、OpenAlex、微信公众号等外部发现源留待未来，且只能产出候选或可审核 proposal。 -- **R14 契约演进** `[已确认]` `note/1.0`、`session-summary/1.0` 与 Markdown workspace 所有权约定已经落地；未来格式变化继续采用版本化、加性演进，破坏性变化才提升 major。 +- **R14 契约演进** `[已确认]` `note/1.0`、`session-summary/1.0`、`experiment-run/1.0`、`claim-evidence/1.0` 与 Markdown workspace 所有权约定已经落地；未来格式变化继续采用版本化、加性演进，破坏性变化才提升 major。 +- **R19 研究执行与证据** `[已确认]` Python、Jupyter、容器、人工或领域软件等外部执行器负责实际运行与隔离；`experiment-run/1.x` 记录输入、代码/环境、有效参数、随机种子、状态、指标与产物哈希，是可追溯观察记录而非命令，成功指标也不会自动成为科研结论。agent 可生成 `claim-evidence/1.x` 草稿，但证据状态与审核状态必须分离，只有显式人工流程可以接受、拒绝或取代断言；入口、校验器和 Provenance 摄取都不得暗含执行或批准。 ## 6. 执行 - **R15 薄入口** `[已确认]` 当前 umbrella repo 只负责 `init`/`doctor`/`status`/`inventory`/`pull`/`demo`、组件兼容清单、workspace 模板与宿主安装器；`inventory` 仅盘点显式来源并给出不含路径的分类级审阅路由，不读取文件正文、不持久化清单，也不执行迁移。入口只能调用公开 CLI/MCP/契约文件，不得 import 组件内部模块或复制业务逻辑。 @@ -65,7 +67,7 @@ 注：可选外部提案源使用的暂存位置与权限由连接器自行声明，不属于 Public Alpha 核心布局。Dataview、Bases、PDF++ 或 AI 输入插件均不得成为基础验收条件。 ## 9. 数据流（见架构图） -Markdown 人写区与 capture adapters（Codex 扫描 / Claude Code enqueue / 可选 browser import）→ on-demand pull → 单 worker（混合审核、append-only）→ **Provenance 权威记忆/检索层** → MCP 回喂与派生视图。启用 Literature profile 时，Zotero → Steward → `library-kb`/reading/lineage 契约 → Provenance/Markdown；启用 Slides profile 时，Steward `handoff` → Lectern → 用户拥有的 `.pptx`。薄入口只编排，不进入数据面。 +Markdown 人写区与 capture adapters（Codex 扫描 / Claude Code enqueue / 可选 browser import）→ on-demand pull → 单 worker（混合审核、append-only）→ **Provenance 权威记忆/检索层** → MCP 回喂与派生视图。启用 Literature profile 时，Zotero → Steward → `library-kb`/reading/lineage 契约 → Provenance/Markdown；进行科研执行时，外部执行器 → `experiment-run` → agent 提议 `claim-evidence` 草稿 → 人工审核 → 后续记忆/交付；启用 Slides profile 时，Steward `handoff` → Lectern → 用户拥有的 `.pptx`。薄入口只编排，不进入数据面。 ## 10. 下一步（按 R16） 1. 在 Provenance MCP 之上增加项目级 context-capsule/resume 入口，并与不暴露内容的控制面 `status` 分开。 @@ -73,3 +75,4 @@ Markdown 人写区与 capture adapters（Codex 扫描 / Claude Code enqueue / 3. 建立 canonical schema 驱动的跨仓 E2E，补齐 Steward handoff→Lectern 交付链。 4. 用干净 Windows 环境完成源码首装、配置回退、离线 demo 与真实项目路径验收。 5. 以 CI、验收记录和截图为证据，统一 README、版本与 Public Alpha 发布说明。 +6. 在不引入内置执行器的前提下，为 `experiment-run/1.0` 与 `claim-evidence/1.0` 增加 Provenance 摄取、审批及合成跨仓 E2E。 diff --git a/specs/project-portfolio.md b/specs/project-portfolio.md index d4576af..25ca0a0 100644 --- a/specs/project-portfolio.md +++ b/specs/project-portfolio.md @@ -32,8 +32,9 @@ frontmatter updates and append-only progress-log region are governed by The current Provenance Markdown adapter uses the `Projects/*.md` convention. The thin suite entry now provides preview-first `scriptorium init` and explicit, no-clobber host installers. Its explicit-root `scriptorium inventory` reports only -suffix-based aggregate routing candidates; automatic discovery and actual migration -remain future work and MUST NOT be implied by initialization or inventory. +suffix-based aggregate routing candidates; a separate reviewed migration plan/apply/ +verify/rollback flow handles selected Markdown/PDF sources. Initialization and +inventory alone still MUST NOT imply or authorize migration. ## Frontmatter is the structured contract @@ -75,6 +76,14 @@ workflow or contract may require either plugin. Generated dashboards are disposable projections. The individual project files, not a dashboard or Provenance index, remain the authoritative project records. +## Research execution stays outside project frontmatter + +`experiment-run/1.0` and `claim-evidence/1.0` are separate JSON contract files +governed by [research-execution-and-evidence.md](research-execution-and-evidence.md). +They do not restore the removed `project.experiments` field and do not turn a +project Markdown file into a run database. The external executor owns each run +record; the explicit human review workflow owns each claim decision. + ## Provenance integration - Provenance currently ingests supported `Projects/*.md` frontmatter and exposes @@ -108,6 +117,7 @@ The intended working pattern is: 4. Stage high-value project changes for approval; append only permitted timeline data. 5. Review the optional portfolio view or query `get_portfolio()` across projects. -The thin entry implements `init`, `doctor`, `status`, `inventory`, `demo`, `pull`, -and canonical host-task installers. Adapter-specific reviewed migration execution and -external beta evidence remain release gaps. +The thin entry implements `init`, `doctor`, `status`, `inventory`, reviewed +Markdown/PDF migration, `demo`, `pull`, and canonical host-task installers. +Live-host acceptance, a published package, and external beta evidence remain +release gaps. diff --git a/specs/research-execution-and-evidence.md b/specs/research-execution-and-evidence.md new file mode 100644 index 0000000..0387244 --- /dev/null +++ b/specs/research-execution-and-evidence.md @@ -0,0 +1,150 @@ +# Research Execution and Claim Evidence (v1) + +This convention connects external research execution to Scriptorium's reviewed +project memory without turning the suite into a code runner or treating a metric +as a scientific conclusion. + +The two contracts are deliberately small: + +- `experiment-run/1.0` records what one external executor ran and observed; +- `claim-evidence/1.0` records one bounded claim, precise links to evidence, and + a separate human-review state. + +Both are ordinary JSON files. They add no daemon, scheduler, sandbox, provider, +or implicit network access. + +## 1. Ownership and producer/consumer map + +| Contract | Master / producer | Consumers | +|---|---|---| +| `experiment-run/1.0` | The external executor or agent workflow that observed the run | file-based agent workflows; Provenance ingest is a release gap | +| `claim-evidence/1.0` draft | The host review workflow that formulated the candidate claim | the human review surface; no authoritative consumer before review | +| `claim-evidence/1.0` accepted/rejected/superseded | The explicit human review workflow | file-based agent workflows; Provenance ingest is a release gap | + +The thin Scriptorium entry may validate, route, and report aggregate readiness for +these files. It does not own their contents, execute their code references, or set +their review decisions. + +## 2. `experiment-run/1.0` is an observation, not an instruction + +An experiment-run file describes a bounded attempt after an executor has started +it. It contains: + +- a stable `run_id` and owning `project`; +- an objective and lifecycle `status`; +- a reproducibility-oriented `execution` reference with the runner, code, + environment, effective parameters, and named random seeds; +- optional input/output/log artifacts and content hashes; +- optional machine-observed numeric metrics; +- producer identity in `generated_by`. + +`execution.code_ref` is a version/entrypoint locator. A consumer MUST NOT interpret +it as a shell command or permission to run code. Creating, validating, importing, +or reading an experiment-run file never authorizes: + +- command execution; +- package installation; +- network access; +- credential use; +- access to an unselected source directory. + +An executor may replace its own `running` record as the run progresses. Once the +record reaches `succeeded`, `failed`, or `cancelled`, consumers SHOULD treat that +`run_id` as immutable. A retry is a new run with a new `run_id`; it may refer to +the same code and inputs. + +A failed run requires a redacted `failure_reason`. Producers MUST remove secrets, +credential-bearing URLs, and private absolute paths from contract records and +logs before sharing them. Artifact entries should prefer stable logical ids, +workspace-relative locators, and SHA-256 hashes. + +`execution.parameters` records the effective non-secret configuration, not just +defaults from a source file. `execution.random_seeds` records each stochastic +subsystem separately (for example Python and NumPy); a deterministic run uses an +empty object. Neither field may contain credentials or secret-bearing URLs. + +Metrics are observations under a declared evaluation setup. They do not become an +approved scientific conclusion merely because the run succeeded or the file +validates. + +## 3. `claim-evidence/1.0` separates evidence from governance + +Each file contains one proposition in `statement`. Two independent fields prevent +the common failure mode where an agent's plausible sentence silently becomes a +project fact: + +- `epistemic_status` describes what the currently linked evidence indicates; +- `review_state` records the human governance decision. + +The v1 states are: + +| Axis | States | +|---|---| +| Evidence | `speculative`, `partially-supported`, `supported`, `contradicted`, `mixed`, `unresolved` | +| Review | `draft`, `accepted`, `rejected`, `superseded` | + +Validation is not approval. A well-formed `draft` remains a candidate. Only the +explicit review workflow may set `accepted`, `rejected`, or `superseded`. + +Every explicit `accepted`, `rejected`, or `superseded` decision MUST record +`reviewed_at`. In addition, an accepted claim: + +1. MUST contain at least one evidence link; +2. MUST NOT remain `speculative`. + +A superseded claim MUST identify its replacement in `superseded_by`. Rejection or +supersession keeps the historical record reviewable; it does not delete the old +statement or its evidence. + +## 4. Evidence links + +Every evidence entry identifies: + +- `relation`: `supports`, `contradicts`, `qualifies`, or `context`; +- `source_type`: an extensible lowercase class such as `experiment-run`, + `parsed-paper`, `reading-note`, or `artifact`; +- `source_id`: the stable id of that source; +- `locator`: the precise metric, section, page, figure, table, or paragraph; +- optional `sha256` and a short explanatory `note`. + +The link points to evidence; it does not copy the source into the claim record. +`note` is an explanation, not a substitute for the source. A consumer SHOULD +resolve and verify the linked source before presenting a claim as supported. + +Evidence from an experiment-run normally uses: + +```text +source_type = experiment-run +source_id = +locator = metrics[] or artifacts[] +``` + +Literature evidence should point to a stable paper/reading record and an exact +section, page, figure, table, or paragraph locator whenever available. + +## 5. Trust and lifecycle + +The minimum workflow is: + +```text +external executor + -> experiment-run/1.0 + -> agent proposes claim-evidence/1.0 (draft) + -> user reviews evidence and wording + -> accepted / rejected / superseded + -> approved project memory or downstream artifact +``` + +Consumers MUST preserve unknown additive fields. They MUST NOT: + +- promote a draft merely because its evidence looks strong; +- convert a successful run directly into an accepted claim; +- erase contradictory evidence; +- rewrite source artifacts through an evidence link; +- expose source content, private paths, or identifiers in content-free + `status`/`doctor` reports. + +The contracts record provenance and review state; they do not claim to provide +process isolation or to validate the scientific adequacy of an evaluator. The +executor remains responsible for its sandbox, resource limits, credentials, +network policy, random seeds, and domain-specific verification. diff --git a/specs/suite-entry-and-ownership.md b/specs/suite-entry-and-ownership.md index b99b55b..f3b903a 100644 --- a/specs/suite-entry-and-ownership.md +++ b/specs/suite-entry-and-ownership.md @@ -2,6 +2,7 @@ > Status: **Accepted** > Date: 2026-07-15 +> Amended: 2026-07-23 — records the external execution/evidence contract boundary > Scope: Scriptorium Public Alpha and later compatible releases ## Context @@ -147,8 +148,11 @@ the remaining iterable to calculate an exact total. | Human-authored project prose, plans, and inbox notes | Markdown workspace | Provenance ingests them | | `project/1.x` frontmatter | the corresponding Markdown project file | approved sync operations MAY update only the contract fields they own | | Tool-owned progress-log marker region | host sync layer | append-only; human-authored regions are immutable to tools | -| `parsed-paper/1.x` and deterministic `lineage-graph/1.x` | Steward-produced contract files | current Steward/agent file workflows consume them; direct Provenance ingestion is a release gap | -| Reviewed `reading-note/1.x` and `review/1.x` | contract files produced by the agent workflow | Steward renders supported projections and agents read the files; direct Provenance ingestion is a release gap | +| `parsed-paper/1.x` and deterministic `lineage-graph/1.x` | Steward-produced contract files | Steward/agent file workflows consume them; Provenance locally ingests them as reference-only research artifacts | +| Reviewed `reading-note/1.x` and `review/1.x` | contract files produced by the agent workflow | Steward renders supported projections; agents read them; Provenance locally ingests them as reference-only research artifacts | +| `experiment-run/1.x` | the external executor or agent workflow that observed the run | file-based agent workflows consume the record; Provenance ingestion is a release gap; no consumer may execute its code reference | +| Draft `claim-evidence/1.x` | host review workflow | human review surface only; validation never promotes the draft | +| Accepted/rejected/superseded `claim-evidence/1.x` | explicit human review workflow | file-based agent workflows consume the decision; Provenance ingestion is a release gap | | Agent-generated scaffold/fill or high-value claims before approval | draft owned by the host workflow | no authoritative consumer until approved | | Cross-session memory/index and protected sync state | Provenance | exposed through read-only MCP and derived views | | Raw browser/platform export | source-platform or local capture artifact | Provenance ingests and sanitizes it | @@ -199,6 +203,9 @@ surface remains the files and schemas in this repository. neither host is preferred or treated as a second-class fallback. - **Literature profile:** Steward; Zotero is optional and becomes the literature master only when this profile is enabled. +- **Research execution path:** an external Python, Jupyter, container, manual, or + domain executor MAY produce `experiment-run/1.x`; an agent/human review workflow + MAY link it through `claim-evidence/1.x`. These files do not authorize execution. - **Slides profile:** Lectern as an optional `handoff/1.x` consumer. - **Obsidian:** optional Markdown client. - **Browser extension:** optional capture front door; local agent logs and explicit @@ -248,7 +255,7 @@ workspace, Provenance memory/search/MCP, Steward's contract-driven literature pa at least one installed Codex or Claude Code adapter (with both documented and equally supported), and one verified optional Steward-to-Lectern handoff. -Public Alpha does not include a desktop shell, cloud/team sync, OpenAlex/RSS +Public Alpha does not include a built-in compute sandbox, desktop shell, cloud/team sync, OpenAlex/RSS discovery, local vector embeddings, remote-control or third-party automation connectors, or a consumer-grade installer. Those capabilities require later decisions and must not be implied by the release surface. @@ -260,7 +267,10 @@ trust boundaries above. The umbrella repository owns executable entry code, component manifests, workspace templates, demo fixtures, host installers, product documentation, screenshots, and cross-repository end-to-end CI. -No new exchange schema is introduced by this ADR. +The 2026-07-23 amendment records the ownership boundary for +`experiment-run/1.0` and `claim-evidence/1.0`; their field-level definitions and +lifecycle rules are authoritative in the schemas and +`research-execution-and-evidence.md`. ## Consequences diff --git a/tests/fixtures/invalid/claim-evidence.accepted-speculative.json b/tests/fixtures/invalid/claim-evidence.accepted-speculative.json new file mode 100644 index 0000000..da9fb02 --- /dev/null +++ b/tests/fixtures/invalid/claim-evidence.accepted-speculative.json @@ -0,0 +1,20 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-speculative", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] An accepted claim cannot remain speculative.", + "kind": "hypothesis", + "epistemic_status": "speculative", + "review_state": "accepted", + "evidence": [ + { + "relation": "context", + "source_type": "experiment-run", + "source_id": "synthetic-xq17-run-0001", + "locator": "metrics[synthetic-error]" + } + ], + "reviewed_at": "2100-01-04T10:05:00Z", + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/tests/fixtures/invalid/claim-evidence.accepted-without-evidence.json b/tests/fixtures/invalid/claim-evidence.accepted-without-evidence.json new file mode 100644 index 0000000..8b085ff --- /dev/null +++ b/tests/fixtures/invalid/claim-evidence.accepted-without-evidence.json @@ -0,0 +1,13 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-invalid", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] An accepted claim cannot omit its evidence.", + "kind": "observation", + "epistemic_status": "supported", + "review_state": "accepted", + "evidence": [], + "reviewed_at": "2100-01-04T10:05:00Z", + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/tests/fixtures/invalid/claim-evidence.accepted-without-reviewed-at.json b/tests/fixtures/invalid/claim-evidence.accepted-without-reviewed-at.json new file mode 100644 index 0000000..0f17a73 --- /dev/null +++ b/tests/fixtures/invalid/claim-evidence.accepted-without-reviewed-at.json @@ -0,0 +1,19 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-missing-review-time", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] An accepted claim must retain its review timestamp.", + "kind": "observation", + "epistemic_status": "supported", + "review_state": "accepted", + "evidence": [ + { + "relation": "supports", + "source_type": "experiment-run", + "source_id": "synthetic-xq17-run-0001", + "locator": "metrics[synthetic-error]" + } + ], + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/tests/fixtures/invalid/claim-evidence.rejected-without-reviewed-at.json b/tests/fixtures/invalid/claim-evidence.rejected-without-reviewed-at.json new file mode 100644 index 0000000..20e187c --- /dev/null +++ b/tests/fixtures/invalid/claim-evidence.rejected-without-reviewed-at.json @@ -0,0 +1,12 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-rejected", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] A rejected claim must retain its review timestamp.", + "kind": "interpretation", + "epistemic_status": "unresolved", + "review_state": "rejected", + "evidence": [], + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/tests/fixtures/invalid/claim-evidence.superseded-without-replacement.json b/tests/fixtures/invalid/claim-evidence.superseded-without-replacement.json new file mode 100644 index 0000000..00dedeb --- /dev/null +++ b/tests/fixtures/invalid/claim-evidence.superseded-without-replacement.json @@ -0,0 +1,13 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-superseded", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] A superseded claim must identify its replacement.", + "kind": "interpretation", + "epistemic_status": "mixed", + "review_state": "superseded", + "evidence": [], + "reviewed_at": "2100-01-04T10:05:00Z", + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/tests/fixtures/invalid/claim-evidence.superseded-without-reviewed-at.json b/tests/fixtures/invalid/claim-evidence.superseded-without-reviewed-at.json new file mode 100644 index 0000000..2873b98 --- /dev/null +++ b/tests/fixtures/invalid/claim-evidence.superseded-without-reviewed-at.json @@ -0,0 +1,13 @@ +{ + "schema_version": "claim-evidence/1.0", + "claim_id": "synthetic-xq17-claim-superseded-time", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T10:00:00Z", + "statement": "[SYNTHETIC] A superseded claim must retain its review timestamp.", + "kind": "interpretation", + "epistemic_status": "mixed", + "review_state": "superseded", + "evidence": [], + "superseded_by": "synthetic-xq17-claim-0001", + "generated_by": "synthetic-review-workflow/0.1.0" +} diff --git a/tests/fixtures/invalid/experiment-run.bad-artifact-hash.json b/tests/fixtures/invalid/experiment-run.bad-artifact-hash.json new file mode 100644 index 0000000..7c72301 --- /dev/null +++ b/tests/fixtures/invalid/experiment-run.bad-artifact-hash.json @@ -0,0 +1,23 @@ +{ + "schema_version": "experiment-run/1.0", + "run_id": "synthetic-xq17-run-bad-hash", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T09:30:00Z", + "status": "succeeded", + "objective": "[SYNTHETIC] Verify that an artifact digest is a lowercase SHA-256 value.", + "execution": { + "runner": "python", + "code_ref": "git:demo0001#scripts/run_synthetic_xq17.py", + "environment_ref": "lock:synthetic-environment-0001", + "parameters": {}, + "random_seeds": {} + }, + "artifacts": [ + { + "artifact_id": "synthetic-xq17-output-invalid", + "role": "output", + "sha256": "not-a-sha256" + } + ], + "generated_by": "synthetic-executor/0.1.0" +} diff --git a/tests/fixtures/invalid/experiment-run.bad-random-seed.json b/tests/fixtures/invalid/experiment-run.bad-random-seed.json new file mode 100644 index 0000000..9b90f72 --- /dev/null +++ b/tests/fixtures/invalid/experiment-run.bad-random-seed.json @@ -0,0 +1,18 @@ +{ + "schema_version": "experiment-run/1.0", + "run_id": "synthetic-xq17-run-bad-seed", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T09:30:00Z", + "status": "succeeded", + "objective": "[SYNTHETIC] Verify that named random seeds are integers or strings.", + "execution": { + "runner": "python", + "code_ref": "git:demo0001#scripts/run_synthetic_xq17.py", + "environment_ref": "lock:synthetic-environment-0001", + "parameters": {}, + "random_seeds": { + "python": true + } + }, + "generated_by": "synthetic-executor/0.1.0" +} diff --git a/tests/fixtures/invalid/experiment-run.failed-without-reason.json b/tests/fixtures/invalid/experiment-run.failed-without-reason.json new file mode 100644 index 0000000..8ebc8bf --- /dev/null +++ b/tests/fixtures/invalid/experiment-run.failed-without-reason.json @@ -0,0 +1,16 @@ +{ + "schema_version": "experiment-run/1.0", + "run_id": "synthetic-xq17-run-invalid", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T09:30:00Z", + "status": "failed", + "objective": "[SYNTHETIC] Verify that a failed run requires a redacted failure reason.", + "execution": { + "runner": "python", + "code_ref": "git:demo0001#scripts/run_synthetic_xq17.py", + "environment_ref": "lock:synthetic-environment-0001", + "parameters": {}, + "random_seeds": {} + }, + "generated_by": "synthetic-executor/0.1.0" +} diff --git a/tests/fixtures/invalid/experiment-run.missing-environment.json b/tests/fixtures/invalid/experiment-run.missing-environment.json new file mode 100644 index 0000000..1f0b4f8 --- /dev/null +++ b/tests/fixtures/invalid/experiment-run.missing-environment.json @@ -0,0 +1,15 @@ +{ + "schema_version": "experiment-run/1.0", + "run_id": "synthetic-xq17-run-missing-environment", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T09:30:00Z", + "status": "succeeded", + "objective": "[SYNTHETIC] Verify that the execution environment is explicit.", + "execution": { + "runner": "python", + "code_ref": "git:demo0001#scripts/run_synthetic_xq17.py", + "parameters": {}, + "random_seeds": {} + }, + "generated_by": "synthetic-executor/0.1.0" +} diff --git a/tests/fixtures/invalid/experiment-run.missing-parameters.json b/tests/fixtures/invalid/experiment-run.missing-parameters.json new file mode 100644 index 0000000..66fe5f6 --- /dev/null +++ b/tests/fixtures/invalid/experiment-run.missing-parameters.json @@ -0,0 +1,15 @@ +{ + "schema_version": "experiment-run/1.0", + "run_id": "synthetic-xq17-run-missing-parameters", + "project": "synthetic-xq17-calibration", + "created": "2100-01-04T09:30:00Z", + "status": "succeeded", + "objective": "[SYNTHETIC] Verify that effective parameters are explicit.", + "execution": { + "runner": "python", + "code_ref": "git:demo0001#scripts/run_synthetic_xq17.py", + "environment_ref": "lock:synthetic-environment-0001", + "random_seeds": {} + }, + "generated_by": "synthetic-executor/0.1.0" +} diff --git a/tools/validate.py b/tools/validate.py index fa1f6de..a71ed3b 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -244,6 +244,179 @@ def validate_review(doc, errors): err(errors, rp, "cells must be an array") +EXPERIMENT_RUN_STATUSES = {"running", "succeeded", "failed", "cancelled"} +EXPERIMENT_ARTIFACT_ROLES = {"input", "output", "log"} +METRIC_DIRECTIONS = {"minimize", "maximize", "target", "none"} +CLAIM_KINDS = {"observation", "hypothesis", "interpretation", "limitation", "decision"} +EPISTEMIC_STATUSES = { + "speculative", + "partially-supported", + "supported", + "contradicted", + "mixed", + "unresolved", +} +CLAIM_REVIEW_STATES = {"draft", "accepted", "rejected", "superseded"} +EVIDENCE_RELATIONS = {"supports", "contradicts", "qualifies", "context"} +SHA256_RE = re.compile(r"^[a-f0-9]{64}$") +SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +SOURCE_TYPE_RE = re.compile(r"^[a-z][a-z0-9-]*$") +SEED_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$") + + +def validate_experiment_run(doc, errors): + for field in ("run_id", "project"): + value = doc.get(field) + if not isinstance(value, str) or not SLUG_RE.match(value): + err(errors, field, f"bad {field}: {value!r}") + if not isinstance(doc.get("created"), str) or not doc["created"]: + err(errors, "created", "missing or not a string") + if doc.get("status") not in EXPERIMENT_RUN_STATUSES: + err(errors, "status", f"invalid status: {doc.get('status')!r}") + if not isinstance(doc.get("objective"), str) or not doc["objective"]: + err(errors, "objective", "missing or not a string") + + execution = doc.get("execution") + if not isinstance(execution, dict): + err(errors, "execution", "missing or not an object") + else: + runner = execution.get("runner") + if not isinstance(runner, str) or not SOURCE_TYPE_RE.match(runner): + err(errors, "execution.runner", f"invalid runner: {runner!r}") + if not isinstance(execution.get("code_ref"), str) or not execution["code_ref"]: + err(errors, "execution.code_ref", "missing or not a string") + if ( + not isinstance(execution.get("environment_ref"), str) + or not execution["environment_ref"] + ): + err(errors, "execution.environment_ref", "missing or not a string") + if not isinstance(execution.get("parameters"), dict): + err(errors, "execution.parameters", "missing or not an object") + random_seeds = execution.get("random_seeds") + if not isinstance(random_seeds, dict): + err(errors, "execution.random_seeds", "missing or not an object") + else: + for name, seed in random_seeds.items(): + path = f"execution.random_seeds.{name}" + if not isinstance(name, str) or not SEED_NAME_RE.match(name): + err(errors, path, "seed name must be lowercase") + if ( + not isinstance(seed, (int, str)) + or isinstance(seed, bool) + or (isinstance(seed, str) and not seed) + ): + err(errors, path, "seed must be an integer or non-empty string") + + artifacts = doc.get("artifacts") + if artifacts is not None: + if not isinstance(artifacts, list): + err(errors, "artifacts", "must be an array") + else: + for i, artifact in enumerate(artifacts): + path = f"artifacts[{i}]" + if not isinstance(artifact, dict): + err(errors, path, "must be an object") + continue + if not isinstance(artifact.get("artifact_id"), str) or not artifact["artifact_id"]: + err(errors, path, "missing artifact_id") + if artifact.get("role") not in EXPERIMENT_ARTIFACT_ROLES: + err(errors, path, f"invalid role: {artifact.get('role')!r}") + digest = artifact.get("sha256") + if digest is not None and ( + not isinstance(digest, str) or not SHA256_RE.match(digest) + ): + err(errors, f"{path}.sha256", "must be a lowercase SHA-256 digest") + + metrics = doc.get("metrics") + if metrics is not None: + if not isinstance(metrics, list): + err(errors, "metrics", "must be an array") + else: + for i, metric in enumerate(metrics): + path = f"metrics[{i}]" + if not isinstance(metric, dict): + err(errors, path, "must be an object") + continue + if not isinstance(metric.get("name"), str) or not metric["name"]: + err(errors, path, "missing name") + value = metric.get("value") + if not isinstance(value, (int, float)) or isinstance(value, bool): + err(errors, path, f"value must be a number: {value!r}") + direction = metric.get("direction") + if direction is not None and direction not in METRIC_DIRECTIONS: + err(errors, path, f"invalid direction: {direction!r}") + + if doc.get("status") == "failed" and ( + not isinstance(doc.get("failure_reason"), str) or not doc["failure_reason"] + ): + err(errors, "failure_reason", "required for failed status") + if not isinstance(doc.get("generated_by"), str) or not doc["generated_by"]: + err(errors, "generated_by", "missing or not a string") + + +def validate_claim_evidence(doc, errors): + for field in ("claim_id", "project"): + value = doc.get(field) + if not isinstance(value, str) or not SLUG_RE.match(value): + err(errors, field, f"bad {field}: {value!r}") + if not isinstance(doc.get("created"), str) or not doc["created"]: + err(errors, "created", "missing or not a string") + if not isinstance(doc.get("statement"), str) or not doc["statement"]: + err(errors, "statement", "missing or not a string") + kind = doc.get("kind") + if kind is not None and kind not in CLAIM_KINDS: + err(errors, "kind", f"invalid kind: {kind!r}") + if doc.get("epistemic_status") not in EPISTEMIC_STATUSES: + err( + errors, + "epistemic_status", + f"invalid epistemic_status: {doc.get('epistemic_status')!r}", + ) + review_state = doc.get("review_state") + if review_state not in CLAIM_REVIEW_STATES: + err(errors, "review_state", f"invalid review_state: {review_state!r}") + + evidence = doc.get("evidence") + if not isinstance(evidence, list): + err(errors, "evidence", "missing or not an array") + evidence = [] + else: + for i, item in enumerate(evidence): + path = f"evidence[{i}]" + if not isinstance(item, dict): + err(errors, path, "must be an object") + continue + if item.get("relation") not in EVIDENCE_RELATIONS: + err(errors, path, f"invalid relation: {item.get('relation')!r}") + source_type = item.get("source_type") + if not isinstance(source_type, str) or not SOURCE_TYPE_RE.match(source_type): + err(errors, path, f"invalid source_type: {source_type!r}") + for field in ("source_id", "locator"): + if not isinstance(item.get(field), str) or not item[field]: + err(errors, path, f"missing {field}") + digest = item.get("sha256") + if digest is not None and ( + not isinstance(digest, str) or not SHA256_RE.match(digest) + ): + err(errors, f"{path}.sha256", "must be a lowercase SHA-256 digest") + + if review_state in {"accepted", "rejected", "superseded"} and ( + not isinstance(doc.get("reviewed_at"), str) or not doc["reviewed_at"] + ): + err(errors, "reviewed_at", "required for an explicit review decision") + if review_state == "accepted": + if not evidence: + err(errors, "evidence", "accepted claims require at least one evidence link") + if doc.get("epistemic_status") == "speculative": + err(errors, "epistemic_status", "accepted claims cannot remain speculative") + if review_state == "superseded": + superseded_by = doc.get("superseded_by") + if not isinstance(superseded_by, str) or not SLUG_RE.match(superseded_by): + err(errors, "superseded_by", "required for superseded review_state") + if not isinstance(doc.get("generated_by"), str) or not doc["generated_by"]: + err(errors, "generated_by", "missing or not a string") + + FORMATS = { "library-kb": validate_library_kb, "proposal": validate_proposal, @@ -255,6 +428,8 @@ def validate_review(doc, errors): "parsed-paper": validate_parsed_paper, "lineage-graph": validate_lineage_graph, "review": validate_review, + "experiment-run": validate_experiment_run, + "claim-evidence": validate_claim_evidence, } diff --git a/uv.lock b/uv.lock index 8600d09..4bfd46c 100644 --- a/uv.lock +++ b/uv.lock @@ -128,7 +128,7 @@ wheels = [ [[package]] name = "scriptorium-spec" -version = "2.2.0" +version = "2.3.0" source = { virtual = "." } [package.dev-dependencies]