From 05d2500614cc77cecf43563d172f74e347c6b723 Mon Sep 17 00:00:00 2001 From: Bnjoroge1 Date: Mon, 5 Oct 2026 14:45:41 -0400 Subject: [PATCH 1/6] perf(control/lite): serve the ready-queue order and attempt lookups from indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `jobs_ready` carried `namespace_id` between the pool key and the priority, so SQLite sorted the last three ORDER BY terms of every claim poll (`USE TEMP B-TREE FOR LAST 3 TERMS OF ORDER BY`); Postgres already keyed its index exactly like the claim ORDER BY. The key is now `(pool_key, priority DESC, run_order, job_order)` on both backends (`SCAN j USING INDEX jobs_ready`, no sort). SQLite also gains `job_requests_attempts (run_id, job_id, request_id DESC)` — the index Postgres has — which serves the latest-attempt lookups (`WHERE run_id = ? AND job_id = ? ORDER BY request_id DESC LIMIT 1`, used by dispatch, settle, steps, the reaper and the test view) and the `jobs` -> `job_requests` cascade; the partial inflight index cannot serve settled attempts. Fresh databases get the aligned indexes; an existing database keeps working unchanged (a missing index is never an error), which is why this needs no schema-version bump and strands no data. Tests: a new `TestDb::query_plan` helper pins both plans (`jobs_ready_serves_the_claim_order_without_a_sort`, `attempt_lookups_use_the_attempts_index`), and the shared suite gains the canonical-id / microsecond-truncation coverage plus the lite shared-timeline trigger test that the schema-drift guard in the next commit documents. --- .../src/control/lite/schema.sql | 38 ++- .../src/control/lite/testview.rs | 13 ++ .../src/control/tests.rs | 217 ++++++++++++++++++ 3 files changed, 267 insertions(+), 1 deletion(-) diff --git a/crates/preloop-runner-server/src/control/lite/schema.sql b/crates/preloop-runner-server/src/control/lite/schema.sql index c47e5b7c5..012205894 100644 --- a/crates/preloop-runner-server/src/control/lite/schema.sql +++ b/crates/preloop-runner-server/src/control/lite/schema.sql @@ -21,6 +21,9 @@ CREATE TABLE namespaces ( namespace_id TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active' CHECK (state IN ('active','suspended','draining','deleted')), + -- UNUSED: `cell_generation` and `config_version` have no reader or + -- writer in any code; drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). cell_generation INTEGER NOT NULL DEFAULT 1, config_version INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), @@ -33,6 +36,9 @@ CREATE TABLE namespace_limits ( max_running_jobs INTEGER, submit_rate_per_minute INTEGER, max_jobs_per_run INTEGER, + -- RESERVED: not read by any code yet (planned: per-tenant job timeout + -- cap, ordering input, history retention window); see `RESERVED_UNUSED` + -- in control/schema_drift.rs. max_job_timeout_minutes INTEGER, priority_tier INTEGER NOT NULL DEFAULT 0, run_history_retention_days INTEGER @@ -45,6 +51,8 @@ CREATE TABLE namespace_pool_limits ( PRIMARY KEY (namespace_id, pool_key) ); +-- RESERVED: not read by any code yet (planned: hosted-tenant admission +-- policy); see `RESERVED_UNUSED` in control/schema_drift.rs. CREATE TABLE namespace_policies ( namespace_id TEXT PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, fork_pr_policy TEXT NOT NULL DEFAULT 'untrusted' CHECK (fork_pr_policy IN @@ -130,6 +138,9 @@ CREATE TABLE run_submissions ( github_context TEXT NOT NULL, workspace_snapshot TEXT, snapshot_timing TEXT, + -- UNUSED: no reader or writer in any code (secret values are resolved at + -- acquire and never stored); drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). secret_refs TEXT NOT NULL DEFAULT '{}', -- Record-level per-job maps the agreed schema has no table for -- (old backend's run_jobs): job_base_ids, job_names, job_needs, @@ -171,6 +182,8 @@ CREATE TABLE jobs ( priority INTEGER NOT NULL DEFAULT 0, run_order INTEGER NOT NULL DEFAULT 0, job_order INTEGER NOT NULL DEFAULT 0, + -- UNUSED: retry-backoff / delayed-start leftover; drop only with a + -- migration (`UNUSED` in control/schema_drift.rs). not_before INTEGER, enqueued_at INTEGER, claimed_by_runner_id INTEGER, @@ -194,7 +207,11 @@ CREATE TABLE jobs ( FOREIGN KEY (run_id, parent_job_id) REFERENCES jobs(run_id, job_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED ); -CREATE INDEX jobs_ready ON jobs(pool_key, namespace_id, priority DESC, run_order, job_order) +-- Claim order: `SELECT .. WHERE queue_state = 'ready' ORDER BY pool_key, +-- priority DESC, run_order, job_order`. The key columns are exactly the +-- ORDER BY (pg's index carries the same key), so the ready front is read in +-- order instead of sorting the whole ready set on every poll. +CREATE INDEX jobs_ready ON jobs(pool_key, priority DESC, run_order, job_order) WHERE queue_state = 'ready'; CREATE INDEX jobs_pending_expansion ON jobs(enqueued_at) WHERE queue_state = 'pending_expansion'; CREATE INDEX jobs_run_active ON jobs(run_id, queue_state) WHERE queue_state <> 'none'; @@ -281,6 +298,11 @@ CREATE TABLE job_requests ( ); CREATE UNIQUE INDEX job_requests_inflight ON job_requests(run_id, job_id) WHERE result IS NULL; CREATE INDEX job_requests_session ON job_requests(session_id) WHERE result IS NULL; +-- Latest-attempt lookups (`WHERE run_id = ? AND job_id = ? ORDER BY +-- request_id DESC LIMIT 1`) and the jobs -> job_requests cascade: the partial +-- inflight index cannot serve settled attempts. `request_id` is the rowid; +-- naming it keeps the key identical to pg's index. +CREATE INDEX job_requests_attempts ON job_requests(run_id, job_id, request_id DESC); CREATE TABLE job_leases ( request_id INTEGER PRIMARY KEY REFERENCES job_requests(request_id) ON DELETE CASCADE, @@ -345,6 +367,9 @@ CREATE TABLE log_files ( run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, plan_id TEXT NOT NULL REFERENCES job_requests(agent_job_id) ON DELETE CASCADE, log_id INTEGER NOT NULL CHECK (log_id > 0), + -- UNUSED: log sizes live in the log-segment store, not here; the table + -- only allocates (plan_id, log_id). Drop only with a migration + -- (`UNUSED` in control/schema_drift.rs). byte_count INTEGER NOT NULL DEFAULT 0, line_count INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), @@ -378,6 +403,8 @@ CREATE TABLE runner_sessions ( protocol TEXT NOT NULL CHECK (protocol IN ('broker','azdo')), client_id TEXT, verified INTEGER NOT NULL DEFAULT 0, + -- UNUSED: wake routing is in-process (`control/wake.rs`); drop only with + -- a migration (`UNUSED` in control/schema_drift.rs). engine_node_id TEXT, created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), last_seen_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)) @@ -426,6 +453,8 @@ CREATE TABLE provision_requests ( pool_key TEXT NOT NULL, labels TEXT NOT NULL, requested_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), + -- UNUSED: the provisioner queue has no lease protocol; drop only with a + -- migration (`UNUSED` in control/schema_drift.rs). leased_until INTEGER, lease_owner TEXT, PRIMARY KEY (run_id, job_id), @@ -565,6 +594,10 @@ CREATE TABLE check_run_updates ( CREATE INDEX check_run_updates_queue ON check_run_updates(installation_id, not_before); -- ── Artifacts ──────────────────────────────────────────────────────── +-- RESERVED: not read by any code yet (planned: shared artifact index; bytes +-- live in the file-backed ArtifactStore). The only production statement is +-- the run-archive `DELETE FROM artifacts`; see `RESERVED_UNUSED` in +-- control/schema_drift.rs. CREATE TABLE artifacts ( artifact_id INTEGER PRIMARY KEY AUTOINCREMENT, namespace_id TEXT NOT NULL, @@ -575,6 +608,9 @@ CREATE TABLE artifacts ( size_bytes INTEGER, digest TEXT, storage_key TEXT NOT NULL, + -- UNUSED: `upload_token_hash` and `finalized_at` have no reader or + -- writer in any code; drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). upload_token_hash BLOB, created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), finalized_at INTEGER, diff --git a/crates/preloop-runner-server/src/control/lite/testview.rs b/crates/preloop-runner-server/src/control/lite/testview.rs index 3124d2401..7210e8eea 100644 --- a/crates/preloop-runner-server/src/control/lite/testview.rs +++ b/crates/preloop-runner-server/src/control/lite/testview.rs @@ -1005,6 +1005,19 @@ impl TestDb<'_> { self.0.execute(sql, params) } + /// `EXPLAIN QUERY PLAN` detail lines for one statement, so index tests + /// can assert which access path (and whether a sort) the planner picks. + pub fn query_plan(&self, sql: &str) -> Vec { + let mut stmt = self + .0 + .prepare(&format!("EXPLAIN QUERY PLAN {sql}")) + .expect("prepare EXPLAIN QUERY PLAN"); + let rows = stmt + .query_map([], |row| row.get::<_, String>(3)) + .expect("run EXPLAIN QUERY PLAN"); + rows.map(|row| row.expect("plan row")).collect() + } + /// `job_assignments` staleness: `assigned_at`/`first_assigned_at`. pub fn set_assignment_times( &self, diff --git a/crates/preloop-runner-server/src/control/tests.rs b/crates/preloop-runner-server/src/control/tests.rs index 0f08806dc..6cfa6cb39 100644 --- a/crates/preloop-runner-server/src/control/tests.rs +++ b/crates/preloop-runner-server/src/control/tests.rs @@ -5277,6 +5277,69 @@ pub(crate) mod suite { outcome_b.run_number ); } + + /// Ids and instants have one canonical stored form on both backends: run + /// uuids lowercase (RFC 4122 text form), instants whole microseconds + /// since the Unix epoch. A sub-microsecond input is truncated toward the + /// epoch (`as_micros` / `timestamp_micros`), never rounded; nanosecond + /// `bigint` stamps keep full precision. + /// + /// The provenance of the run's `created_at` still differs and is recorded + /// here instead of hidden: SQLite stores the submitted value (truncated + /// to µs), Postgres takes the database clock (`DEFAULT now()`). Only the + /// precision rule and the nanosecond column are asserted identical. + pub(crate) async fn ids_and_timestamps_are_stored_canonically(backend: &dyn ControlBackend) { + let nanos = 1_700_000_000_123_456_789_i64; + let run_id = RunId(uuid::Uuid::parse_str("0F8FAD5B-D9CB-469F-A165-70867728950E").unwrap()); + let mut record = super::run_record(run_id); + // Upper-case spelling, deliberately: the stored form is the canonical + // lowercase one on both backends. + record.created_at = chrono::DateTime::from_timestamp_nanos(nanos); + record.fork_approval_requested_at_unix_nanos = Some(nanos); + record + .job_names + .insert(JobId("build".to_owned()), "build".to_owned()); + backend + .submit_run(SubmitRun { + namespace: "default".to_owned(), + record, + jobs: vec![submit_job(run_id, "build", 1)], + workflow_concurrency: None, + empty_concurrency_group: false, + check_hostable: false, + }) + .await + .unwrap(); + + let loaded = backend.run_record(run_id).await.unwrap(); + assert_eq!( + loaded.run_id.0.to_string(), + "0f8fad5b-d9cb-469f-a165-70867728950e", + "a run id must round-trip in the lowercase canonical form" + ); + assert_eq!( + loaded.job_names.get(&JobId("build".to_owned())), + Some(&"build".to_owned()), + "the job id round-trips verbatim: {:?}", + loaded.job_names + ); + let subsec = loaded.created_at.timestamp_subsec_nanos(); + assert_eq!( + subsec % 1000, + 0, + "microsecond columns must not keep sub-microsecond digits ({subsec} ns)" + ); + assert_eq!( + loaded.created_at, + chrono::DateTime::from_timestamp_micros(loaded.created_at.timestamp_micros()).unwrap(), + "the stored instant must be its own microsecond truncation" + ); + assert_eq!( + loaded.fork_approval_requested_at_unix_nanos, + Some(nanos), + "nanosecond-typed stamps must keep full precision" + ); + } } // ── SQLite ────────────────────────────────────────────────────────────── @@ -6454,6 +6517,14 @@ mod pg { }; suite::duplicate_run_number_is_reallocated(&backend).await; } + + #[tokio::test] + async fn ids_and_timestamps_are_stored_canonically() { + let Some((_pg, backend)) = backend().await else { + return skip_no_postgres(); + }; + suite::ids_and_timestamps_are_stored_canonically(&backend).await; + } } // ── New SQLite backend (`control::lite`) ──────────────────────────────── // @@ -7525,6 +7596,152 @@ mod lite { async fn duplicate_run_number_is_reallocated() { suite::duplicate_run_number_is_reallocated(&LiteBackend::in_memory().unwrap()).await; } + + #[tokio::test] + async fn ids_and_timestamps_are_stored_canonically() { + suite::ids_and_timestamps_are_stored_canonically(&LiteBackend::in_memory().unwrap()).await; + } + + /// `job_requests.timeline_id` is deliberately not UNIQUE: several attempts + /// of one job share a timeline (pg cannot — its column is UNIQUE and + /// `timelines` FKs to one request). The + /// `job_requests_timeline_cascade` trigger removes the timeline row only + /// when the last request referencing it is deleted. + #[tokio::test] + async fn timeline_is_shared_across_attempts_and_pruned_by_trigger() { + fn timeline_rows(tx: &crate::control::lite::TestDb<'_>, timeline_id: &str) -> i64 { + tx.0.query_row( + "SELECT count(*) FROM timelines WHERE timeline_id = ?1", + rusqlite::params![timeline_id], + |row| row.get::<_, i64>(0), + ) + .unwrap() + } + + let backend = LiteBackend::in_memory().unwrap(); + let run_id = RunId::new(); + backend + .submit_run(submit_run(run_id, vec![submit_job(run_id, "build", 1)])) + .await + .unwrap(); + let run_key = run_id.0.to_string(); + let timeline_id = uuid::Uuid::new_v4().to_string(); + let retry_agent = uuid::Uuid::new_v4().to_string(); + backend + .test_db_mutate(|tx| { + // Submission already minted one unclaimed attempt; settle it + // onto a shared timeline, then add the retry on the same + // timeline. Sharing is only meaningful once the previous + // attempt is settled — that is also what keeps the partial + // inflight index ("one open attempt per job") satisfied. + let first_agent: String = + tx.0.query_row( + "SELECT agent_job_id FROM job_requests \ + WHERE run_id = ?1 AND job_id = 'build'", + rusqlite::params![run_key], + |row| row.get(0), + ) + .unwrap(); + tx.execute( + "INSERT INTO timelines (timeline_id, change_id) VALUES (?1, 0)", + rusqlite::params![timeline_id], + ) + .unwrap(); + tx.execute( + "UPDATE job_requests SET timeline_id = ?2, result = 'failure' \ + WHERE agent_job_id = ?1", + rusqlite::params![first_agent, timeline_id], + ) + .unwrap(); + tx.execute( + "INSERT INTO job_requests (run_id, job_id, namespace_id, \ + agent_job_id, timeline_id, claimed_at) \ + VALUES (?1, 'build', 'default', ?2, ?3, 0)", + rusqlite::params![run_key, retry_agent, timeline_id], + ) + .expect("two attempts of one job may share the timeline id"); + assert_eq!( + timeline_rows(tx, &timeline_id), + 1, + "one shared timeline row" + ); + tx.execute( + "DELETE FROM job_requests WHERE agent_job_id = ?1", + rusqlite::params![retry_agent], + ) + .unwrap(); + assert_eq!( + timeline_rows(tx, &timeline_id), + 1, + "the timeline outlives the retry (the settled attempt still references it)" + ); + tx.execute( + "DELETE FROM job_requests WHERE agent_job_id = ?1", + rusqlite::params![first_agent], + ) + .unwrap(); + assert_eq!( + timeline_rows(tx, &timeline_id), + 0, + "the cascade trigger prunes the orphaned timeline" + ); + }) + .unwrap(); + } + + /// The claim query's `ORDER BY` must be served by `jobs_ready` without a + /// temp B-tree: the key order is the whole point of the index (pg carries + /// the same key). While `namespace_id` sat second, SQLite sorted the last + /// three ORDER BY terms on every poll. + #[tokio::test] + async fn jobs_ready_serves_the_claim_order_without_a_sort() { + let backend = LiteBackend::in_memory().unwrap(); + let plan = backend + .test_db_mutate(|tx| { + tx.query_plan(&format!( + "SELECT j.run_id FROM jobs j \ + WHERE j.queue_state = 'ready' AND ({}) \ + ORDER BY j.pool_key, j.priority DESC, j.run_order, j.job_order \ + LIMIT 64 OFFSET 0", + crate::control::types::NAMESPACE_ADMITS_CLAIM + )) + }) + .unwrap(); + assert!( + plan.iter().any(|detail| detail.contains("jobs_ready")), + "the claim query must use the jobs_ready index: {plan:?}" + ); + assert!( + !plan.iter().any(|detail| detail.contains("B-TREE")), + "the ready-queue order must come from the index, not a sort: {plan:?}" + ); + } + + /// Latest-attempt lookups must use `job_requests_attempts`: the partial + /// inflight index cannot serve settled attempts, and the row order must + /// come from the index. + #[tokio::test] + async fn attempt_lookups_use_the_attempts_index() { + let backend = LiteBackend::in_memory().unwrap(); + let plan = backend + .test_db_mutate(|tx| { + tx.query_plan( + "SELECT request_id FROM job_requests \ + WHERE run_id = '00000000-0000-0000-0000-000000000000' AND job_id = 'build' \ + ORDER BY request_id DESC LIMIT 1", + ) + }) + .unwrap(); + assert!( + plan.iter() + .any(|detail| detail.contains("job_requests_attempts")), + "latest-attempt lookups must use job_requests_attempts: {plan:?}" + ); + assert!( + !plan.iter().any(|detail| detail.contains("B-TREE")), + "the attempt order must come from the index, not a sort: {plan:?}" + ); + } } // ── Schema copies and legacy identity ─────────────────────────────────── From 41d6af355f9d1b5c261951ff2731287df5a0d56a Mon Sep 17 00:00:00 2001 From: Bnjoroge1 Date: Mon, 5 Oct 2026 14:45:41 -0400 Subject: [PATCH 2/6] test(control): guard the lite/pg schema drift and inventory reserved objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two control schemas are hand-maintained copies of one design, and nothing caught the drift between them. `control/schema_drift.rs` parses both DDL files (tables, column names, inline UNIQUE, foreign keys, index names and ordered index columns) and asserts they match except for `DOCUMENTED_DIFFERENCES`, where every entry carries its reason. The test fails on an undocumented difference and on a stale entry, so aligning the backends shrinks the list instead of leaving it to rot. The 14 current entries cover the partition children, the xid8 outbox columns and their reader index (owned by the durable-outbox PR), `consumer_offsets.last_txid`, the GIN `runners_labels` index, the pg-only `job_messages.job_timeout_s` extraction (aligning it needs a migration, because the reaper must not read a column an existing database does not have), the shared `timeline_id` (with the reverse FK), and the `runner_sessions.runner_id` FK that the legacy session-before-registration path makes impossible on SQLite's `PRAGMA foreign_keys = ON`. The same module inventories what no code reads: `RESERVED_UNUSED` (`namespace_policies`, the three unread `namespace_limits` columns and the `artifacts` table — placeholders for planned features) and `UNUSED` (`namespaces.cell_generation`/`config_version`, `run_submissions.secret_refs`, `jobs.not_before`, `provision_requests.lease_owner`/`leased_until`, `runner_sessions.engine_node_id`, `log_files.byte_count`/`line_count`, `artifacts.upload_token_hash`/`finalized_at`). Both DDL files carry matching `-- RESERVED:`/`-- UNUSED:` markers at each site, and a second test greps the Rust sources: the day code starts using one of these, it fails with the object, the reason and the location. Dropping any of them is a migration-backed change, not this PR: the branch has no migration runner, and the backends refuse a database stamped with any other schema version. Also stops `docs/control-schema.sql` from claiming secrets and upload tokens are stored: the file is the pg schema mirrored for readers, and both copies now mark the RESERVED/UNUSED objects at each site. --- .../preloop-runner-server/src/control/mod.rs | 2 + .../src/control/pg/schema.sql | 38 +- .../src/control/schema_drift.rs | 650 ++++++++++++++++++ docs/control-schema.sql | 38 +- 4 files changed, 712 insertions(+), 16 deletions(-) create mode 100644 crates/preloop-runner-server/src/control/schema_drift.rs diff --git a/crates/preloop-runner-server/src/control/mod.rs b/crates/preloop-runner-server/src/control/mod.rs index 507e7b841..6484258a0 100644 --- a/crates/preloop-runner-server/src/control/mod.rs +++ b/crates/preloop-runner-server/src/control/mod.rs @@ -34,6 +34,8 @@ pub(crate) mod logic; pub mod logic; pub(crate) mod pg; #[cfg(test)] +mod schema_drift; +#[cfg(test)] mod tests; #[cfg(any(test, feature = "test-support"))] pub mod testview; diff --git a/crates/preloop-runner-server/src/control/pg/schema.sql b/crates/preloop-runner-server/src/control/pg/schema.sql index 5119a5574..b76bf952e 100644 --- a/crates/preloop-runner-server/src/control/pg/schema.sql +++ b/crates/preloop-runner-server/src/control/pg/schema.sql @@ -48,6 +48,9 @@ CREATE TABLE namespaces ( -- suspended/deleted: neither (queued jobs stay queued, nothing starts). state text NOT NULL DEFAULT 'active' CHECK (state IN ('active','suspended','draining','deleted')), + -- UNUSED: `cell_generation` and `config_version` have no reader or + -- writer in any code; drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). cell_generation bigint NOT NULL DEFAULT 1, -- bumped on cell move; fences stale writers config_version bigint NOT NULL DEFAULT 0, -- last platform push applied created_at timestamptz NOT NULL DEFAULT now(), @@ -60,6 +63,9 @@ CREATE TABLE namespaces ( -- predicate (a capped claim locks this row first, so nodes cannot -- overshoot). Not yet read: max_job_timeout_minutes, priority_tier, -- run_history_retention_days. +-- RESERVED: not read by any code yet (planned: per-tenant job timeout cap, +-- ordering input, history retention window); see `RESERVED_UNUSED` in +-- control/schema_drift.rs. CREATE TABLE namespace_limits ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, max_queued_jobs integer, @@ -80,6 +86,8 @@ CREATE TABLE namespace_pool_limits ( ); -- Admission and job-build restrictions. +-- RESERVED: not read by any code yet (planned: hosted-tenant admission +-- policy); see `RESERVED_UNUSED` in control/schema_drift.rs. CREATE TABLE namespace_policies ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, fork_pr_policy text NOT NULL DEFAULT 'untrusted' CHECK (fork_pr_policy IN @@ -169,16 +177,17 @@ CREATE TABLE workflow_run_numbers ( PRIMARY KEY (namespace_id, repository, workflow_path) ); --- The request a run was created from. No secret values: `secret_refs` --- names which secrets (and scopes) the run may use; values are resolved --- from the SecretProvider when a job is acquired, never stored here. +-- The request a run was created from. No secret values are ever stored: +-- secrets are resolved from the SecretProvider when a job is acquired. CREATE TABLE run_submissions ( run_id uuid PRIMARY KEY REFERENCES runs(run_id) ON DELETE CASCADE, submission jsonb NOT NULL, -- WorkflowSubmission minus secrets github_context jsonb NOT NULL, workspace_snapshot jsonb, snapshot_timing jsonb, -- duration_ms, object_count, pack_bytes - secret_refs jsonb NOT NULL DEFAULT '{}', -- name -> {scope, version} + -- UNUSED: no reader or writer in any code; drop only with a migration + -- (`UNUSED` in control/schema_drift.rs). + secret_refs jsonb NOT NULL DEFAULT '{}', -- Record-level per-job maps the table layout has no column for (jobs -- with no `jobs` row yet — a check run minted before its matrix leg -- materializes). Mirrors lite's `run_submissions.record_details`. @@ -225,6 +234,8 @@ CREATE TABLE jobs ( priority integer NOT NULL DEFAULT 0, run_order bigint NOT NULL DEFAULT 0, job_order integer NOT NULL DEFAULT 0, + -- UNUSED: retry-backoff / delayed-start leftover; drop only with a + -- migration (`UNUSED` in control/schema_drift.rs). not_before timestamptz, enqueued_at timestamptz, claimed_by_runner_id bigint, @@ -414,12 +425,14 @@ CREATE TABLE timeline_records ( -- Per-plan log ids fit the official runner's 32-bit TaskLog.Id and remain -- stable across nodes. The unique pair arbitrates concurrent allocations. --- Content lives in file segments; this row tracks result-service counters. +-- Content and sizes live in file segments; this row only allocates ids. CREATE TABLE log_files ( log_key text PRIMARY KEY, run_id uuid NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, plan_id uuid NOT NULL REFERENCES job_requests(agent_job_id) ON DELETE CASCADE, log_id integer NOT NULL CHECK (log_id > 0), + -- UNUSED: log sizes live in the log-segment store, not here; drop only + -- with a migration (`UNUSED` in control/schema_drift.rs). byte_count bigint NOT NULL DEFAULT 0, line_count bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now(), @@ -451,7 +464,9 @@ CREATE TABLE runner_sessions ( protocol text NOT NULL CHECK (protocol IN ('broker','azdo')), client_id text, verified boolean NOT NULL DEFAULT false, - engine_node_id text, -- node holding the long-poll (wake routing) + -- UNUSED: wake routing is in-process (`control/wake.rs`); drop only with + -- a migration (`UNUSED` in control/schema_drift.rs). + engine_node_id text, created_at timestamptz NOT NULL DEFAULT now(), last_seen_at timestamptz NOT NULL DEFAULT now() ); @@ -505,6 +520,8 @@ CREATE TABLE provision_requests ( pool_key text NOT NULL, labels jsonb NOT NULL, requested_at timestamptz NOT NULL DEFAULT now(), + -- UNUSED: the provisioner queue has no lease protocol; drop only with a + -- migration (`UNUSED` in control/schema_drift.rs). leased_until timestamptz, lease_owner text, PRIMARY KEY (run_id, job_id), @@ -669,8 +686,10 @@ CREATE TABLE check_run_updates ( CREATE INDEX check_run_updates_queue ON check_run_updates(installation_id, not_before); -- ── Artifacts (replaces the artifact part of the `meta` blob) ──────── --- Blobs live in object storage; these rows are the shared index. Upload --- tokens are stored as hashes, never raw. +-- Blobs live in object storage; upload state lives in the file-backed +-- ArtifactStore. These rows are RESERVED (planned as the shared artifact +-- index): the only production statement is the run-archive DELETE. See +-- `RESERVED_UNUSED` in control/schema_drift.rs. -- -- The Actions cache is deliberately NOT here: it is a bounded cache -- colocated with the runner hosts (preloop-cache CAS: key index -> @@ -686,6 +705,9 @@ CREATE TABLE artifacts ( size_bytes bigint, digest text, storage_key text NOT NULL, + -- UNUSED: `upload_token_hash` and `finalized_at` have no reader or + -- writer in any code; drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). upload_token_hash bytea, created_at timestamptz NOT NULL DEFAULT now(), finalized_at timestamptz, diff --git a/crates/preloop-runner-server/src/control/schema_drift.rs b/crates/preloop-runner-server/src/control/schema_drift.rs new file mode 100644 index 000000000..659826b51 --- /dev/null +++ b/crates/preloop-runner-server/src/control/schema_drift.rs @@ -0,0 +1,650 @@ +//! Schema-drift guard between the two copies of the control schema, plus the +//! reserved/unused inventory the maintainers must keep honest. +//! +//! `lite/schema.sql` is the SQLite translation of `pg/schema.sql` (the +//! source of truth, mirrored into `docs/control-schema.sql`). Both must +//! expose the same tables, the same column names, the same foreign keys and +//! the same indexes — only storage types differ (uuid/text -> TEXT, +//! timestamptz -> INTEGER microseconds, jsonb -> TEXT, bytea -> BLOB, +//! identity -> INTEGER PRIMARY KEY AUTOINCREMENT). +//! +//! [`DOCUMENTED_DIFFERENCES`] is the complete list of deliberate exceptions; +//! every entry carries the reason it exists. A difference that is not listed +//! fails [`control_schema_differences_are_documented`], and a listed +//! difference that no longer exists fails too — stale entries must be +//! deleted, so the guard shrinks itself whenever the backends are aligned. +//! +//! This guard is parse-level only. It cannot see storage semantics (a column +//! UNIQUE here is a constraint there), so it does not bless any difference it +//! cannot express — see the `UNIQUE`/foreign-key checks, which are the parts +//! of the drift that matter for correctness. +//! +//! This is not a migration tool: a database created at any other stamped +//! `schema_version` is still refused at open, and changing that is a separate +//! release blocker (`lite/mod.rs` / `pg/mod.rs`). Until versioned migrations +//! exist, this module's job is to keep the *fresh-database* schemas honest +//! and to name every place the two backends disagree. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +/// The SQLite translation (see the file header for the type mapping). +const LITE_SCHEMA: &str = include_str!("lite/schema.sql"); +/// The Postgres source of truth. +const PG_SCHEMA: &str = include_str!("pg/schema.sql"); + +/// Deliberate structural differences between the two schemas, keyed exactly +/// as [`differences`] reports them. Keep the keys verbatim: a typo makes the +/// guard fail twice (undocumented difference + stale entry). +const DOCUMENTED_DIFFERENCES: &[(&str, &str)] = &[ + // ── Partitioning: Postgres-only ────────────────────────────────── + ( + "table attempt_history_default: pg only", + "partition child: pg partitions the *_history tables by run_created_at and drops whole \ + partitions; SQLite has no partitioning and keeps one table. The logical table and its \ + columns are identical.", + ), + ( + "table job_history_default: pg only", + "partition child of job_history (see attempt_history_default).", + ), + ( + "table outbox_events_default: pg only", + "partition child of outbox_events (see attempt_history_default).", + ), + ( + "table run_history_default: pg only", + "partition child of run_history (see attempt_history_default).", + ), + ( + "table step_history_default: pg only", + "partition child of step_history (see attempt_history_default).", + ), + // ── Outbox ordering: single writer vs xid8 safe point ──────────── + ( + "table outbox_events column txid: pg only", + "pg orders the outbox by (txid, event_id) below pg_snapshot_xmin to read only finished \ + transactions. SQLite has exactly one writer (BEGIN IMMEDIATE), so commit order is \ + event_id order and no safe-point column exists. Owned by the durable-outbox PR: do not \ + drop or rename on either side.", + ), + ( + "table outbox_events column origin: pg only", + "pg stamps the writing node (`preloop.origin`) so a node skips its own rows; the single \ + SQLite writer has no other node to skip.", + ), + ( + "table consumer_offsets column last_txid: pg only", + "pg bookmarks the outbox by xid8 safe point; SQLite bookmarks by last_event_id. Owned by \ + the durable-outbox PR: do not drop or rename on either side.", + ), + ( + "index outbox_events_read: pg only", + "the (txid, event_id) reader index; there is no txid column on SQLite (see above).", + ), + // ── Indexes that need an engine feature SQLite has no equivalent of ─ + ( + "index runners_labels: pg only", + "GIN index over the `labels` jsonb (label -> runner search). SQLite has no GIN; lite \ + matches labels in Rust (`runner_label_sets`) over the same JSON payload.", + ), + // ── Columns and constraints that encode a deliberate behavioral split ─ + ( + "table job_messages column job_timeout_s: pg only", + "pg extracts `timeout-minutes` once at write because the reaper reads it for every \ + unfinished attempt on every tick and detoasting the template cost ~15x the rest of the \ + query. Lite re-extracts with json_extract from a non-toasted TEXT column. Aligning the \ + two needs a schema-version bump with a tested migration (existing databases have no \ + column and the reaper must not fail on them).", + ), + ( + "table job_requests column timeline_id inline UNIQUE: pg only", + "intentional: a timeline is shared across the attempts of one job. Pg makes the column \ + UNIQUE so `timelines` can FK to it; SQLite keeps several job_requests rows on one \ + timeline_id and deletes the timeline through the `job_requests_timeline_cascade` trigger \ + when the last request goes.", + ), + ( + "table timelines foreign key timeline_id -> job_requests(timeline_id): pg only", + "intentional: the reverse-direction FK for the pg UNIQUE above cannot be expressed on a \ + shared, non-unique SQLite column; `job_requests_timeline_cascade` owns the lifecycle.", + ), + ( + "table runner_sessions foreign key runner_id -> runners(runner_id): pg only", + "intentional: the legacy AgentSession/distributedtask create may name an agent id whose \ + registration arrives later (`require_live_runner: false` in runner_lifecycle.rs). SQLite \ + runs with `PRAGMA foreign_keys = ON`, so an FK would reject that row; lite instead \ + checks liveness inside the insert transaction when the caller requires it, reads through \ + a LEFT JOIN, and deletes the sessions before the runner row (lifecycle.rs). Note: the pg \ + FK means the legacy create cannot work against Postgres — tracked as a parity finding.", + ), +]; + +/// Schema objects kept although *no* code reads them yet — placeholders for +/// features the maintainers intend to wire up. The test below greps the Rust +/// sources so the day one of them starts being used, this list (and the +/// matching `-- RESERVED:` comment in both schemas) must be revisited. +const RESERVED_UNUSED: &[Kept] = &[ + Kept { + what: "table namespace_policies", + why: "planned: hosted-tenant admission policy (fork-PR policy, token permission ceiling, \ + OIDC audiences, local-execution policy)", + needles: &[("namespace_policies", None)], + allowed_mentions: &[], + }, + Kept { + what: "column namespace_limits.max_job_timeout_minutes", + why: "planned: per-tenant job timeout cap, enforced at claim/acquire", + needles: &[("namespace_limits", Some("max_job_timeout_minutes"))], + allowed_mentions: &[], + }, + Kept { + what: "column namespace_limits.priority_tier", + why: "planned: input to the tenant ordering policy (dispatch orders by priority, \ + run_order, job_order today)", + needles: &[("namespace_limits", Some("priority_tier"))], + allowed_mentions: &[], + }, + Kept { + what: "column namespace_limits.run_history_retention_days", + why: "planned: per-tenant history retention window", + needles: &[("namespace_limits", Some("run_history_retention_days"))], + allowed_mentions: &[], + }, + Kept { + what: "table artifacts", + why: "planned: shared artifact index (bytes live in the file-backed ArtifactStore). \ + Today the only production statements are the run-archive `DELETE FROM artifacts`; \ + the two test-only statements in retention.rs are listed as allowed mentions.", + needles: &[("artifacts", None)], + allowed_mentions: &[ + "SELECT count(*) FROM artifacts WHERE run_id = ?1", + "INSERT INTO artifacts (namespace_id, run_id, job_backend_id, name", + ], + }, +]; + +/// Schema objects with no reader and no writer anywhere in this crate. They +/// are *not* dropped here: this branch has no migration runner, and a +/// database created from this file refuses any other stamped version (see +/// the module docs), so removing a column would strand existing databases. +/// Drop them only in a migration-backed change; until then this list (and the +/// `-- UNUSED:` comment at each site) is the inventory. +const UNUSED: &[Kept] = &[ + Kept { + what: "column namespaces.cell_generation", + why: "pg planned it to fence stale writers after a cell move; no code reads or writes it", + needles: &[("namespaces", Some("cell_generation"))], + allowed_mentions: &[], + }, + Kept { + what: "column namespaces.config_version", + why: "pg planned it for platform config pushes; no code reads or writes it", + needles: &[("namespaces", Some("config_version"))], + allowed_mentions: &[], + }, + Kept { + what: "column run_submissions.secret_refs", + why: "secret values are resolved at acquire and never stored; no code reads or writes the \ + column (the secret-name list travels in `submission`)", + needles: &[("run_submissions", Some("secret_refs"))], + allowed_mentions: &[], + }, + Kept { + what: "column jobs.not_before", + why: "retry-backoff / delayed-start leftover from the old scheduler; the queue orders by \ + priority, run_order, job_order", + needles: &[("jobs", Some("not_before"))], + allowed_mentions: &[], + }, + Kept { + what: "column provision_requests.lease_owner", + why: "the provisioner queue has no lease protocol; no code reads or writes it", + needles: &[("provision_requests", Some("lease_owner"))], + allowed_mentions: &[], + }, + Kept { + what: "column provision_requests.leased_until", + why: "the provisioner queue has no lease protocol; no code reads or writes it", + needles: &[("provision_requests", Some("leased_until"))], + allowed_mentions: &[], + }, + Kept { + what: "column runner_sessions.engine_node_id", + why: "long-poll wake routing is in-process (`control/wake.rs`), not persisted", + needles: &[("runner_sessions", Some("engine_node_id"))], + allowed_mentions: &[], + }, + Kept { + what: "column log_files.byte_count", + why: "log sizes live in the log-segment store; the control table only allocates \ + (plan_id, log_id)", + needles: &[("log_files", Some("byte_count"))], + allowed_mentions: &[], + }, + Kept { + what: "column log_files.line_count", + why: "log sizes live in the log-segment store; the control table only allocates \ + (plan_id, log_id)", + needles: &[("log_files", Some("line_count"))], + allowed_mentions: &[], + }, + Kept { + what: "column artifacts.upload_token_hash", + why: "the file-backed ArtifactStore owns upload state; the control table is a delete-only \ + index today", + needles: &[("artifacts", Some("upload_token_hash"))], + allowed_mentions: &[], + }, + Kept { + what: "column artifacts.finalized_at", + why: "the file-backed ArtifactStore owns upload state; the control table is a delete-only \ + index today", + needles: &[("artifacts", Some("finalized_at"))], + allowed_mentions: &[], + }, +]; + +/// One kept-but-unused schema object and the evidence it is still unused. +struct Kept { + /// Human description used in the failure message. + what: &'static str, + /// Why it is kept. Quoted in the failure message. + why: &'static str, + /// `(table, column)`: a column when set, a whole table otherwise. + needles: &'static [(&'static str, Option<&'static str>)], + /// Statement fragments that are known to mention the object and are + /// accepted (test-only helpers); every other match fails the test. + allowed_mentions: &'static [&'static str], +} + +#[test] +fn control_schema_differences_are_documented() { + let differences = differences(); + let documented: BTreeMap<&str, &str> = DOCUMENTED_DIFFERENCES.iter().copied().collect(); + let mut problems = Vec::new(); + for difference in &differences { + if !documented.contains_key(difference.as_str()) { + problems.push(format!("undocumented difference: {difference}")); + } + } + for (difference, _reason) in DOCUMENTED_DIFFERENCES { + if !differences.contains(*difference) { + problems.push(format!( + "stale entry, delete it from DOCUMENTED_DIFFERENCES: {difference}" + )); + } + } + assert!( + problems.is_empty(), + "the SQLite and Postgres control schemas drifted ({} difference(s)):\n{}", + differences.len(), + problems.join("\n") + ); +} + +#[test] +fn reserved_and_unused_objects_stay_unreferenced() { + let sources = rust_sources(); + let mut findings = Vec::new(); + for kept in RESERVED_UNUSED.iter().chain(UNUSED) { + for (table, column) in kept.needles { + for (path, text) in &sources { + if path.ends_with("schema_drift.rs") { + continue; + } + for statement in sql_mentions(text, table, *column) { + if kept + .allowed_mentions + .iter() + .any(|allowed| statement.contains(allowed)) + { + continue; + } + findings.push(format!( + "{} ({}): {statement}\n in {}", + kept.what, + kept.why, + path.display() + )); + } + } + } + } + assert!( + findings.is_empty(), + "these schema objects are declared RESERVED/UNUSED but the code references them now:\n{}\n\ + If this is a real use, remove the object from RESERVED_UNUSED/UNUSED in \ + control/schema_drift.rs, delete its `-- RESERVED:`/`-- UNUSED:` comment in both schemas, \ + and (for a wired-up feature) drop the `unused` claim from the PR.", + findings.join("\n") + ); +} + +// ── The parsed schemas ────────────────────────────────────────────────── + +#[derive(Debug, Default, Clone)] +struct Table { + columns: BTreeSet, + /// Columns declared with an inline `UNIQUE`. + unique_columns: BTreeSet, + /// `column -> table(columns)` for inline `REFERENCES`, `a,b -> table(x,y)` + /// for table-level `FOREIGN KEY` clauses. The `ON DELETE` action is not + /// part of the key (it is behavioral, not structural, and both schemas + /// pick the action to match their own cascade rules). + foreign_keys: BTreeSet, +} + +#[derive(Debug, Default)] +struct Schema { + tables: BTreeMap, + /// Index name -> ordered column list (ASC/DESC and the access method are + /// not part of the comparison; the guard still catches column-order and + /// column-set drift because the list is ordered). + indexes: BTreeMap>, +} + +fn differences() -> BTreeSet { + let lite = parse(LITE_SCHEMA); + let pg = parse(PG_SCHEMA); + let mut out = BTreeSet::new(); + for name in lite.tables.keys() { + if !pg.tables.contains_key(name) { + out.insert(format!("table {name}: lite only")); + } + } + for name in pg.tables.keys() { + if !lite.tables.contains_key(name) { + out.insert(format!("table {name}: pg only")); + } + } + for (name, lite_table) in &lite.tables { + let Some(pg_table) = pg.tables.get(name) else { + continue; + }; + diff_members( + &lite_table.columns, + &pg_table.columns, + |item, side| format!("table {name} column {item}: {side} only"), + &mut out, + ); + diff_members( + &lite_table.unique_columns, + &pg_table.unique_columns, + |item, side| format!("table {name} column {item} inline UNIQUE: {side} only"), + &mut out, + ); + diff_members( + &lite_table.foreign_keys, + &pg_table.foreign_keys, + |item, side| format!("table {name} foreign key {item}: {side} only"), + &mut out, + ); + } + for name in lite.indexes.keys() { + match pg.indexes.get(name) { + None => { + out.insert(format!("index {name}: lite only")); + } + Some(pg_columns) => { + let lite_columns = &lite.indexes[name]; + if lite_columns != pg_columns { + out.insert(format!( + "index {name} columns: lite {lite_columns:?} vs pg {pg_columns:?}" + )); + } + } + } + } + for name in pg.indexes.keys() { + if !lite.indexes.contains_key(name) { + out.insert(format!("index {name}: pg only")); + } + } + out +} + +fn diff_members( + lite: &BTreeSet, + pg: &BTreeSet, + key: impl Fn(&str, &str) -> String, + out: &mut BTreeSet, +) { + for item in lite.difference(pg) { + out.insert(key(item, "lite")); + } + for item in pg.difference(lite) { + out.insert(key(item, "pg")); + } +} + +/// Extract the printable schema from one DDL file: tables (with columns, +/// inline UNIQUE flags and foreign keys) and indexes. Comments and string +/// literals never span a line in these files; `--` always starts a comment. +fn parse(sql: &str) -> Schema { + let mut schema = Schema::default(); + let mut current: Option<(String, Table)> = None; + let mut depth: i32 = 0; + for raw in sql.lines() { + let trimmed = strip_comment(raw).trim(); + if trimmed.is_empty() { + continue; + } + if current.is_some() { + if depth == 1 && !trimmed.starts_with(')') { + let table = &mut current.as_mut().expect("checked above").1; + parse_table_body_line(trimmed, table); + } + depth += paren_delta(trimmed); + if depth <= 0 { + let (name, table) = current.take().expect("checked above"); + schema.tables.insert(name, table); + } + continue; + } + if let Some(rest) = trimmed.strip_prefix("CREATE TABLE ") { + let (name, tail) = split_ident(rest); + let depth_after_header = paren_delta(&tail); + let table = Table::default(); + if depth_after_header > 0 { + depth = depth_after_header; + current = Some((name, table)); + } else { + // `CREATE TABLE x PARTITION OF y ...` (no body) or a + // single-line table. + schema.tables.insert(name, table); + } + continue; + } + let index = trimmed + .strip_prefix("CREATE UNIQUE INDEX ") + .or_else(|| trimmed.strip_prefix("CREATE INDEX ")); + if let Some(rest) = index + && let Some((name, columns)) = parse_index(rest) + { + schema.indexes.insert(name, columns); + } + } + schema +} + +/// One line inside a `CREATE TABLE (...)` body: a column definition, a +/// table-level constraint, or a continuation line (`ON DELETE ...`). +fn parse_table_body_line(line: &str, table: &mut Table) { + let (word, rest) = split_ident(line); + let keyword = word.to_ascii_uppercase(); + if matches!( + keyword.as_str(), + "PRIMARY" + | "FOREIGN" + | "UNIQUE" + | "CHECK" + | "CONSTRAINT" + | "ON" + | "DEFERRABLE" + | "INITIALLY" + ) { + if keyword == "FOREIGN" + && let Some(foreign_key) = table_foreign_key(&rest) + { + table.foreign_keys.insert(foreign_key); + } + return; + } + if !table.columns.insert(word.clone()) { + return; + } + if find_word(&rest, "UNIQUE").is_some() { + table.unique_columns.insert(word.clone()); + } + if let Some(target) = reference_target(&rest) { + table.foreign_keys.insert(format!("{word} -> {target}")); + } +} + +fn table_foreign_key(rest: &str) -> Option { + let open = rest.find('(')?; + let close = rest[open..].find(')')? + open; + let columns = normalize_columns(&rest[open + 1..close]); + let target = reference_target(&rest[close + 1..])?; + Some(format!("{columns} -> {target}")) +} + +fn reference_target(text: &str) -> Option { + let at = find_word(text, "REFERENCES")?; + let after = text[at + "REFERENCES".len()..].trim_start(); + let open = after.find('(')?; + let close = after[open..].find(')')? + open; + let table = after[..open].trim(); + let columns = normalize_columns(&after[open + 1..close]); + Some(format!("{table}({columns})")) +} + +fn parse_index(rest: &str) -> Option<(String, Vec)> { + let (name, tail) = split_ident(rest); + let open = tail.find('(')?; + let close = tail[open..].find(')')? + open; + let columns = tail[open + 1..close] + .split(',') + .map(|column| { + let column = column.trim(); + let column = column + .strip_suffix("DESC") + .or_else(|| column.strip_suffix("ASC")) + .unwrap_or(column); + column.trim().to_ascii_lowercase() + }) + .filter(|column| !column.is_empty()) + .collect(); + Some((name, columns)) +} + +// ── Text helpers ──────────────────────────────────────────────────────── + +fn strip_comment(line: &str) -> &str { + match line.find("--") { + Some(at) => &line[..at], + None => line, + } +} + +fn paren_delta(line: &str) -> i32 { + line.matches('(').count() as i32 - line.matches(')').count() as i32 +} + +/// `("CREATE TABLE", "namespaces (")`-style split on the first run of +/// whitespace. +fn split_ident(text: &str) -> (String, String) { + let text = text.trim(); + match text.find(char::is_whitespace) { + Some(at) => (text[..at].to_owned(), text[at..].trim().to_owned()), + None => (text.to_owned(), String::new()), + } +} + +fn normalize_columns(columns: &str) -> String { + columns + .split(',') + .map(str::trim) + .collect::>() + .join(",") +} + +/// Byte offset of `needle` in `haystack` when bounded by non-identifier +/// characters on both sides. +fn find_word(haystack: &str, needle: &str) -> Option { + for (at, _) in haystack.match_indices(needle) { + let before = haystack[..at].chars().next_back(); + let after = haystack[at + needle.len()..].chars().next(); + let is_boundary = |ch: Option| !ch.is_some_and(|c| c.is_alphanumeric() || c == '_'); + if is_boundary(before) && is_boundary(after) { + return Some(at); + } + } + None +} + +// ── Source grep for the reserved/unused inventory ─────────────────────── + +fn rust_sources() -> Vec<(PathBuf, String)> { + fn walk(dir: &Path, out: &mut Vec<(PathBuf, String)>) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") + && let Ok(text) = std::fs::read_to_string(&path) + { + out.push((path, text)); + } + } + } + let mut out = Vec::new(); + walk(&Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), &mut out); + out +} + +/// Every mention of `table` (or `table.column` when `column` is set) inside a +/// SQL read/write statement. SQL is written across several Rust string lines, +/// so any statement verb within 240 bytes before the mention counts as the +/// same statement. `DELETE`/`DROP` are deliberately not verbs: the +/// `artifacts` table is delete-only by design, and deletes are not reads. +/// +/// The window only looks backwards, so a mention that puts the table name +/// after the column (some `SELECT` shapes) can be missed — this guard is a +/// prompt, not a proof. +fn sql_mentions(text: &str, table: &str, column: Option<&str>) -> Vec { + const VERBS: &[&str] = &["SELECT", "INSERT", "UPDATE", "JOIN"]; + let needle = column.unwrap_or(table); + let mut mentions = Vec::new(); + let mut from = 0; + while let Some(at) = find_word(&text[from..], needle) { + let at = from + at; + from = at + 1; + let start = char_window_start(text, at, 240); + let window = &text[start..at]; + if !VERBS.iter().any(|verb| find_word(window, verb).is_some()) { + continue; + } + if column.is_some() && find_word(window, table).is_none() { + continue; + } + let line_start = text[..at].rfind('\n').map_or(0, |newline| newline + 1); + let line_end = text[at..].find('\n').map_or(text.len(), |end| at + end); + mentions.push(text[line_start..line_end].trim().to_owned()); + } + mentions +} + +/// Start of the `max_chars`-long window that ends at `at`, on a char +/// boundary. +fn char_window_start(text: &str, at: usize, max_chars: usize) -> usize { + text[..at] + .char_indices() + .rev() + .nth(max_chars) + .map_or(0, |(index, _)| index) +} diff --git a/docs/control-schema.sql b/docs/control-schema.sql index ef400c3c5..902bc842d 100644 --- a/docs/control-schema.sql +++ b/docs/control-schema.sql @@ -47,6 +47,9 @@ CREATE TABLE namespaces ( -- suspended/deleted: neither (queued jobs stay queued, nothing starts). state text NOT NULL DEFAULT 'active' CHECK (state IN ('active','suspended','draining','deleted')), + -- UNUSED: `cell_generation` and `config_version` have no reader or + -- writer in any code; drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). cell_generation bigint NOT NULL DEFAULT 1, -- bumped on cell move; fences stale writers config_version bigint NOT NULL DEFAULT 0, -- last platform push applied created_at timestamptz NOT NULL DEFAULT now(), @@ -59,6 +62,9 @@ CREATE TABLE namespaces ( -- predicate (a capped claim locks this row first, so nodes cannot -- overshoot). Not yet read: max_job_timeout_minutes, priority_tier, -- run_history_retention_days. +-- RESERVED: not read by any code yet (planned: per-tenant job timeout cap, +-- ordering input, history retention window); see `RESERVED_UNUSED` in +-- control/schema_drift.rs. CREATE TABLE namespace_limits ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, max_queued_jobs integer, @@ -79,6 +85,8 @@ CREATE TABLE namespace_pool_limits ( ); -- Admission and job-build restrictions. +-- RESERVED: not read by any code yet (planned: hosted-tenant admission +-- policy); see `RESERVED_UNUSED` in control/schema_drift.rs. CREATE TABLE namespace_policies ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, fork_pr_policy text NOT NULL DEFAULT 'untrusted' CHECK (fork_pr_policy IN @@ -168,16 +176,17 @@ CREATE TABLE workflow_run_numbers ( PRIMARY KEY (namespace_id, repository, workflow_path) ); --- The request a run was created from. No secret values: `secret_refs` --- names which secrets (and scopes) the run may use; values are resolved --- from the SecretProvider when a job is acquired, never stored here. +-- The request a run was created from. No secret values are ever stored: +-- secrets are resolved from the SecretProvider when a job is acquired. CREATE TABLE run_submissions ( run_id uuid PRIMARY KEY REFERENCES runs(run_id) ON DELETE CASCADE, submission jsonb NOT NULL, -- WorkflowSubmission minus secrets github_context jsonb NOT NULL, workspace_snapshot jsonb, snapshot_timing jsonb, -- duration_ms, object_count, pack_bytes - secret_refs jsonb NOT NULL DEFAULT '{}', -- name -> {scope, version} + -- UNUSED: no reader or writer in any code; drop only with a migration + -- (`UNUSED` in control/schema_drift.rs). + secret_refs jsonb NOT NULL DEFAULT '{}', -- Record-level per-job maps the table layout has no column for (jobs -- with no `jobs` row yet — a check run minted before its matrix leg -- materializes). Mirrors lite's `run_submissions.record_details`. @@ -224,6 +233,8 @@ CREATE TABLE jobs ( priority integer NOT NULL DEFAULT 0, run_order bigint NOT NULL DEFAULT 0, job_order integer NOT NULL DEFAULT 0, + -- UNUSED: retry-backoff / delayed-start leftover; drop only with a + -- migration (`UNUSED` in control/schema_drift.rs). not_before timestamptz, enqueued_at timestamptz, claimed_by_runner_id bigint, @@ -413,12 +424,14 @@ CREATE TABLE timeline_records ( -- Per-plan log ids fit the official runner's 32-bit TaskLog.Id and remain -- stable across nodes. The unique pair arbitrates concurrent allocations. --- Content lives in file segments; this row tracks result-service counters. +-- Content and sizes live in file segments; this row only allocates ids. CREATE TABLE log_files ( log_key text PRIMARY KEY, run_id uuid NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, plan_id uuid NOT NULL REFERENCES job_requests(agent_job_id) ON DELETE CASCADE, log_id integer NOT NULL CHECK (log_id > 0), + -- UNUSED: log sizes live in the log-segment store, not here; drop only + -- with a migration (`UNUSED` in control/schema_drift.rs). byte_count bigint NOT NULL DEFAULT 0, line_count bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now(), @@ -450,7 +463,9 @@ CREATE TABLE runner_sessions ( protocol text NOT NULL CHECK (protocol IN ('broker','azdo')), client_id text, verified boolean NOT NULL DEFAULT false, - engine_node_id text, -- node holding the long-poll (wake routing) + -- UNUSED: wake routing is in-process (`control/wake.rs`); drop only with + -- a migration (`UNUSED` in control/schema_drift.rs). + engine_node_id text, created_at timestamptz NOT NULL DEFAULT now(), last_seen_at timestamptz NOT NULL DEFAULT now() ); @@ -504,6 +519,8 @@ CREATE TABLE provision_requests ( pool_key text NOT NULL, labels jsonb NOT NULL, requested_at timestamptz NOT NULL DEFAULT now(), + -- UNUSED: the provisioner queue has no lease protocol; drop only with a + -- migration (`UNUSED` in control/schema_drift.rs). leased_until timestamptz, lease_owner text, PRIMARY KEY (run_id, job_id), @@ -668,8 +685,10 @@ CREATE TABLE check_run_updates ( CREATE INDEX check_run_updates_queue ON check_run_updates(installation_id, not_before); -- ── Artifacts (replaces the artifact part of the `meta` blob) ──────── --- Blobs live in object storage; these rows are the shared index. Upload --- tokens are stored as hashes, never raw. +-- Blobs live in object storage; upload state lives in the file-backed +-- ArtifactStore. These rows are RESERVED (planned as the shared artifact +-- index): the only production statement is the run-archive DELETE. See +-- `RESERVED_UNUSED` in control/schema_drift.rs. -- -- The Actions cache is deliberately NOT here: it is a bounded cache -- colocated with the runner hosts (preloop-cache CAS: key index -> @@ -685,6 +704,9 @@ CREATE TABLE artifacts ( size_bytes bigint, digest text, storage_key text NOT NULL, + -- UNUSED: `upload_token_hash` and `finalized_at` have no reader or + -- writer in any code; drop only with a migration (`UNUSED` in + -- control/schema_drift.rs). upload_token_hash bytea, created_at timestamptz NOT NULL DEFAULT now(), finalized_at timestamptz, From 2084f223fa34a90dbf9ae3ee8a23b3a85fffc50f Mon Sep 17 00:00:00 2001 From: Bnjoroge1 Date: Mon, 5 Oct 2026 14:45:41 -0400 Subject: [PATCH 3/6] docs(changelog): record the drift guard, the index alignment and the inventory Unreleased: the schema-drift guard and reserved/unused inventory, the canonical-id / microsecond-truncation and shared-timeline coverage, and the SQLite index alignment (`jobs_ready` key, new `job_requests_attempts`) with its before/after EXPLAIN QUERY PLAN. --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 527ccb97f..263c0b26f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Releases before v0.27.0 predate the changelog. ## [Unreleased] +### Added + +- A schema-drift guard keeps the SQLite (`control/lite/schema.sql`) and + Postgres (`control/pg/schema.sql`) control schemas aligned: same tables, + column names, foreign keys and indexes, with every deliberate difference + listed and reasoned in `control/schema_drift.rs` (partition children, the + xid8 outbox columns and their ordering index, the GIN label index, the + pg-only `job_messages.job_timeout_s`, the shared `timeline_id`, and the + `runner_sessions.runner_id` FK). The guard fails on an undocumented + difference and on a stale entry, so aligning the backends shrinks the list + instead of letting it rot. The same module inventories the schema objects + kept without readers — `RESERVED` placeholders (`namespace_policies`, three + unread `namespace_limits` columns, `artifacts`) and `UNUSED` leftovers + (`namespaces.cell_generation`/`config_version`, + `run_submissions.secret_refs`, `jobs.not_before`, + `provision_requests.lease_owner`/`leased_until`, + `runner_sessions.engine_node_id`, `log_files.byte_count`/`line_count`, + `artifacts.upload_token_hash`/`finalized_at`) — and a grep test fails when + code starts using one, so the inventory cannot silently stale. Nothing is + dropped and no schema version is bumped: this branch has no migration + runner, so dropping a column would strand every existing control database. +- Coverage for id and time storage on both backends: a run id and job + round-trip in lowercase-canonical form, a sub-microsecond instant reads + back as whole microseconds (nanosecond `bigint` stamps keep full + precision), SQLite accepts several attempts of one job sharing a + `timeline_id` (the deliberate pg-`UNIQUE` divergence) and its + `job_requests_timeline_cascade` trigger prunes the shared timeline with the + last request. + +### Changed + +- SQLite's `jobs_ready` key is now `(pool_key, priority DESC, run_order, + job_order)` — exactly the claim / ready-queue `ORDER BY`, the key Postgres + already carries. `namespace_id` sat between the pool key and the priority, + so SQLite sorted the last three ORDER BY terms on every poll; the claim + query now reads the ready front in index order (`EXPLAIN QUERY PLAN` goes + from `USE TEMP B-TREE FOR LAST 3 TERMS OF ORDER BY` to no sort). SQLite + also gains `job_requests_attempts (run_id, job_id, request_id DESC)`, the + same index Postgres has, which serves the latest-attempt lookups and the + `jobs` -> `job_requests` cascade that the partial inflight index cannot. + Both are fresh-database changes: an existing database keeps working + unchanged, because a missing index is never an error. ### Security From 878f5421cb480f9712d095ed211844da861911ea Mon Sep 17 00:00:00 2001 From: Bill Njoroge Date: Wed, 7 Oct 2026 01:36:42 -0400 Subject: [PATCH 4/6] test(control): count a DELETE predicate on an inventoried column as a use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sql_mentions` treated `DELETE` as a non-verb, so `DELETE FROM provision_requests WHERE lease_owner = ?` — a real read of an inventoried column — was invisible to `reserved_and_unused_objects_stay_unreferenced`, and the inventory test would have stayed green while that column was in use. `DELETE` now counts when the needle is a column; a table-only `DELETE` stays ignored, so the delete-only `artifacts` cleanup is still not a use. A new unit test pins both halves. --- .../src/control/schema_drift.rs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/preloop-runner-server/src/control/schema_drift.rs b/crates/preloop-runner-server/src/control/schema_drift.rs index 659826b51..6fe386819 100644 --- a/crates/preloop-runner-server/src/control/schema_drift.rs +++ b/crates/preloop-runner-server/src/control/schema_drift.rs @@ -321,6 +321,24 @@ fn reserved_and_unused_objects_stay_unreferenced() { ); } +/// The inventory grep must see a column used in a `DELETE` predicate: that is +/// a real read of the column, and `DELETE` alone used to be invisible to it. +/// A table-only `DELETE` stays invisible — the `artifacts` table is +/// delete-only by design, so its cleanup statements are not uses. +#[test] +fn delete_predicates_count_as_column_uses() { + let delete = "DELETE FROM provision_requests WHERE lease_owner = ?1"; + assert_eq!( + sql_mentions(delete, "provision_requests", Some("lease_owner")).len(), + 1, + "a DELETE predicate reads the column" + ); + assert!( + sql_mentions(delete, "provision_requests", None).is_empty(), + "a table-only DELETE is not a read" + ); +} + // ── The parsed schemas ────────────────────────────────────────────────── #[derive(Debug, Default, Clone)] @@ -610,8 +628,10 @@ fn rust_sources() -> Vec<(PathBuf, String)> { /// Every mention of `table` (or `table.column` when `column` is set) inside a /// SQL read/write statement. SQL is written across several Rust string lines, /// so any statement verb within 240 bytes before the mention counts as the -/// same statement. `DELETE`/`DROP` are deliberately not verbs: the -/// `artifacts` table is delete-only by design, and deletes are not reads. +/// same statement. `DROP` is deliberately not a verb, and a bare `DELETE` of +/// a *table* is not a read (the `artifacts` table is delete-only by design), +/// but `DELETE ... WHERE ` does read that column, so `DELETE` counts +/// when the needle is a column. /// /// The window only looks backwards, so a mention that puts the table name /// after the column (some `SELECT` shapes) can be missed — this guard is a @@ -626,7 +646,8 @@ fn sql_mentions(text: &str, table: &str, column: Option<&str>) -> Vec { from = at + 1; let start = char_window_start(text, at, 240); let window = &text[start..at]; - if !VERBS.iter().any(|verb| find_word(window, verb).is_some()) { + let delete_reads_a_column = column.is_some() && find_word(window, "DELETE").is_some(); + if !delete_reads_a_column && !VERBS.iter().any(|verb| find_word(window, verb).is_some()) { continue; } if column.is_some() && find_word(window, table).is_none() { From baa7c21cc71cb87a25ff0691b2a49c96be965e2c Mon Sep 17 00:00:00 2001 From: Bill Njoroge Date: Wed, 7 Oct 2026 02:52:48 -0400 Subject: [PATCH 5/6] chore(ci): re-push for a run on this branch's own merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine's run for 878f5421 checked out the stale test merge 23e7361b ("Merge 2084f223 into 992e3b81") — the previous head's merge — so those checks, although green, never exercised the DELETE-predicate fix. No content change: the tree is identical, and this commit only gives GitHub a fresh merge to compute (its ^2 is this head) so the next run tests the right tree. From 7e7a4b922f66fc529cc7b9fc547abd696dec21a0 Mon Sep 17 00:00:00 2001 From: Bill Njoroge Date: Thu, 8 Oct 2026 11:44:44 -0400 Subject: [PATCH 6/6] refactor(control): drop unused schema objects and the inventory test Drop run_submissions.secret_refs, jobs.not_before, provision_requests.lease_owner/leased_until, runner_sessions.engine_node_id, log_files.byte_count/line_count, artifacts.upload_token_hash and pg's runners_labels GIN index: no code referenced any of them (the only labels SQL is jsonb_array_elements_text, which a GIN index cannot serve). Existing databases that still have the columns keep working. Namespace platform columns and namespace_policies stay, marked platform-owned. Remove the RESERVED/UNUSED inventory and its source-grep tests. --- CHANGELOG.md | 28 +- .../src/control/lite/schema.sql | 43 +-- .../src/control/pg/schema.sql | 41 +-- .../src/control/schema_drift.rs | 272 +----------------- docs/control-schema.sql | 41 +-- 5 files changed, 40 insertions(+), 385 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24c4d4e11..5bde0f873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,21 +14,11 @@ Releases before v0.27.0 predate the changelog. Postgres (`control/pg/schema.sql`) control schemas aligned: same tables, column names, foreign keys and indexes, with every deliberate difference listed and reasoned in `control/schema_drift.rs` (partition children, the - xid8 outbox columns and their ordering index, the GIN label index, the - pg-only `job_messages.job_timeout_s`, the shared `timeline_id`, and the + xid8 outbox columns and their ordering index, the pg-only + `job_messages.job_timeout_s`, the shared `timeline_id`, and the `runner_sessions.runner_id` FK). The guard fails on an undocumented difference and on a stale entry, so aligning the backends shrinks the list - instead of letting it rot. The same module inventories the schema objects - kept without readers — `RESERVED` placeholders (`namespace_policies`, three - unread `namespace_limits` columns, `artifacts`) and `UNUSED` leftovers - (`namespaces.cell_generation`/`config_version`, - `run_submissions.secret_refs`, `jobs.not_before`, - `provision_requests.lease_owner`/`leased_until`, - `runner_sessions.engine_node_id`, `log_files.byte_count`/`line_count`, - `artifacts.upload_token_hash`/`finalized_at`) — and a grep test fails when - code starts using one, so the inventory cannot silently stale. Nothing is - dropped and no schema version is bumped: this branch has no migration - runner, so dropping a column would strand every existing control database. + instead of letting it rot. - Coverage for id and time storage on both backends: a run id and job round-trip in lowercase-canonical form, a sub-microsecond instant reads back as whole microseconds (nanosecond `bigint` stamps keep full @@ -50,6 +40,18 @@ Releases before v0.27.0 predate the changelog. `jobs` -> `job_requests` cascade that the partial inflight index cannot. Both are fresh-database changes: an existing database keeps working unchanged, because a missing index is never an error. +- The control schema no longer carries objects nothing uses: + `run_submissions.secret_refs`, `jobs.not_before`, + `provision_requests.lease_owner`/`leased_until`, + `runner_sessions.engine_node_id`, `log_files.byte_count`/`line_count`, + `artifacts.upload_token_hash` and Postgres's `runners_labels` GIN index + (label matching reads `runners.labels` in Rust and uses no operator a GIN + index serves). No code referenced any of them, so an existing control + database that still has the columns keeps working. The namespace + platform columns (`cell_generation`, `config_version`, + `namespace_limits.max_job_timeout_minutes`/`priority_tier`/ + `run_history_retention_days`, `namespace_policies`) stay and are marked + platform-owned in the schema. ### Security diff --git a/crates/preloop-runner-server/src/control/lite/schema.sql b/crates/preloop-runner-server/src/control/lite/schema.sql index 012205894..29efb7306 100644 --- a/crates/preloop-runner-server/src/control/lite/schema.sql +++ b/crates/preloop-runner-server/src/control/lite/schema.sql @@ -21,9 +21,8 @@ CREATE TABLE namespaces ( namespace_id TEXT PRIMARY KEY, state TEXT NOT NULL DEFAULT 'active' CHECK (state IN ('active','suspended','draining','deleted')), - -- UNUSED: `cell_generation` and `config_version` have no reader or - -- writer in any code; drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). + -- Platform-owned: cell fencing and config push bookkeeping; the engine + -- neither reads nor writes these two. cell_generation INTEGER NOT NULL DEFAULT 1, config_version INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), @@ -36,9 +35,8 @@ CREATE TABLE namespace_limits ( max_running_jobs INTEGER, submit_rate_per_minute INTEGER, max_jobs_per_run INTEGER, - -- RESERVED: not read by any code yet (planned: per-tenant job timeout - -- cap, ordering input, history retention window); see `RESERVED_UNUSED` - -- in control/schema_drift.rs. + -- Platform-owned: written by the hosted platform; the engine does not + -- enforce these three yet. max_job_timeout_minutes INTEGER, priority_tier INTEGER NOT NULL DEFAULT 0, run_history_retention_days INTEGER @@ -51,8 +49,8 @@ CREATE TABLE namespace_pool_limits ( PRIMARY KEY (namespace_id, pool_key) ); --- RESERVED: not read by any code yet (planned: hosted-tenant admission --- policy); see `RESERVED_UNUSED` in control/schema_drift.rs. +-- Platform-owned: the hosted platform writes per-tenant admission policy +-- here; the engine does not read it yet. CREATE TABLE namespace_policies ( namespace_id TEXT PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, fork_pr_policy TEXT NOT NULL DEFAULT 'untrusted' CHECK (fork_pr_policy IN @@ -138,10 +136,6 @@ CREATE TABLE run_submissions ( github_context TEXT NOT NULL, workspace_snapshot TEXT, snapshot_timing TEXT, - -- UNUSED: no reader or writer in any code (secret values are resolved at - -- acquire and never stored); drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). - secret_refs TEXT NOT NULL DEFAULT '{}', -- Record-level per-job maps the agreed schema has no table for -- (old backend's run_jobs): job_base_ids, job_names, job_needs, -- job_check_run_ids, caller_plans, jobs_list, reusable_calls, @@ -182,9 +176,6 @@ CREATE TABLE jobs ( priority INTEGER NOT NULL DEFAULT 0, run_order INTEGER NOT NULL DEFAULT 0, job_order INTEGER NOT NULL DEFAULT 0, - -- UNUSED: retry-backoff / delayed-start leftover; drop only with a - -- migration (`UNUSED` in control/schema_drift.rs). - not_before INTEGER, enqueued_at INTEGER, claimed_by_runner_id INTEGER, claimed_at INTEGER, @@ -367,11 +358,6 @@ CREATE TABLE log_files ( run_id TEXT NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, plan_id TEXT NOT NULL REFERENCES job_requests(agent_job_id) ON DELETE CASCADE, log_id INTEGER NOT NULL CHECK (log_id > 0), - -- UNUSED: log sizes live in the log-segment store, not here; the table - -- only allocates (plan_id, log_id). Drop only with a migration - -- (`UNUSED` in control/schema_drift.rs). - byte_count INTEGER NOT NULL DEFAULT 0, - line_count INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), UNIQUE (plan_id, log_id) ); @@ -403,9 +389,6 @@ CREATE TABLE runner_sessions ( protocol TEXT NOT NULL CHECK (protocol IN ('broker','azdo')), client_id TEXT, verified INTEGER NOT NULL DEFAULT 0, - -- UNUSED: wake routing is in-process (`control/wake.rs`); drop only with - -- a migration (`UNUSED` in control/schema_drift.rs). - engine_node_id TEXT, created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), last_seen_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)) ); @@ -453,10 +436,6 @@ CREATE TABLE provision_requests ( pool_key TEXT NOT NULL, labels TEXT NOT NULL, requested_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), - -- UNUSED: the provisioner queue has no lease protocol; drop only with a - -- migration (`UNUSED` in control/schema_drift.rs). - leased_until INTEGER, - lease_owner TEXT, PRIMARY KEY (run_id, job_id), FOREIGN KEY (run_id, job_id) REFERENCES jobs(run_id, job_id) ON DELETE CASCADE ); @@ -594,10 +573,8 @@ CREATE TABLE check_run_updates ( CREATE INDEX check_run_updates_queue ON check_run_updates(installation_id, not_before); -- ── Artifacts ──────────────────────────────────────────────────────── --- RESERVED: not read by any code yet (planned: shared artifact index; bytes --- live in the file-backed ArtifactStore). The only production statement is --- the run-archive `DELETE FROM artifacts`; see `RESERVED_UNUSED` in --- control/schema_drift.rs. +-- Artifact catalog rows. Bytes live in the file-backed ArtifactStore; the +-- only statement today is the run-archive `DELETE FROM artifacts`. CREATE TABLE artifacts ( artifact_id INTEGER PRIMARY KEY AUTOINCREMENT, namespace_id TEXT NOT NULL, @@ -608,10 +585,6 @@ CREATE TABLE artifacts ( size_bytes INTEGER, digest TEXT, storage_key TEXT NOT NULL, - -- UNUSED: `upload_token_hash` and `finalized_at` have no reader or - -- writer in any code; drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). - upload_token_hash BLOB, created_at INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000000 AS INTEGER)), finalized_at INTEGER, expires_at INTEGER, diff --git a/crates/preloop-runner-server/src/control/pg/schema.sql b/crates/preloop-runner-server/src/control/pg/schema.sql index b76bf952e..30fd0ac58 100644 --- a/crates/preloop-runner-server/src/control/pg/schema.sql +++ b/crates/preloop-runner-server/src/control/pg/schema.sql @@ -48,9 +48,8 @@ CREATE TABLE namespaces ( -- suspended/deleted: neither (queued jobs stay queued, nothing starts). state text NOT NULL DEFAULT 'active' CHECK (state IN ('active','suspended','draining','deleted')), - -- UNUSED: `cell_generation` and `config_version` have no reader or - -- writer in any code; drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). + -- Platform-owned: cell fencing and config push bookkeeping; the engine + -- neither reads nor writes these two. cell_generation bigint NOT NULL DEFAULT 1, -- bumped on cell move; fences stale writers config_version bigint NOT NULL DEFAULT 0, -- last platform push applied created_at timestamptz NOT NULL DEFAULT now(), @@ -63,9 +62,8 @@ CREATE TABLE namespaces ( -- predicate (a capped claim locks this row first, so nodes cannot -- overshoot). Not yet read: max_job_timeout_minutes, priority_tier, -- run_history_retention_days. --- RESERVED: not read by any code yet (planned: per-tenant job timeout cap, --- ordering input, history retention window); see `RESERVED_UNUSED` in --- control/schema_drift.rs. +-- Those three are platform-owned: written by the hosted platform, not yet +-- enforced by the engine. CREATE TABLE namespace_limits ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, max_queued_jobs integer, @@ -86,8 +84,8 @@ CREATE TABLE namespace_pool_limits ( ); -- Admission and job-build restrictions. --- RESERVED: not read by any code yet (planned: hosted-tenant admission --- policy); see `RESERVED_UNUSED` in control/schema_drift.rs. +-- Platform-owned: the hosted platform writes per-tenant admission policy +-- here; the engine does not read it yet. CREATE TABLE namespace_policies ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, fork_pr_policy text NOT NULL DEFAULT 'untrusted' CHECK (fork_pr_policy IN @@ -185,9 +183,6 @@ CREATE TABLE run_submissions ( github_context jsonb NOT NULL, workspace_snapshot jsonb, snapshot_timing jsonb, -- duration_ms, object_count, pack_bytes - -- UNUSED: no reader or writer in any code; drop only with a migration - -- (`UNUSED` in control/schema_drift.rs). - secret_refs jsonb NOT NULL DEFAULT '{}', -- Record-level per-job maps the table layout has no column for (jobs -- with no `jobs` row yet — a check run minted before its matrix leg -- materializes). Mirrors lite's `run_submissions.record_details`. @@ -234,9 +229,6 @@ CREATE TABLE jobs ( priority integer NOT NULL DEFAULT 0, run_order bigint NOT NULL DEFAULT 0, job_order integer NOT NULL DEFAULT 0, - -- UNUSED: retry-backoff / delayed-start leftover; drop only with a - -- migration (`UNUSED` in control/schema_drift.rs). - not_before timestamptz, enqueued_at timestamptz, claimed_by_runner_id bigint, claimed_at timestamptz, @@ -431,10 +423,6 @@ CREATE TABLE log_files ( run_id uuid NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, plan_id uuid NOT NULL REFERENCES job_requests(agent_job_id) ON DELETE CASCADE, log_id integer NOT NULL CHECK (log_id > 0), - -- UNUSED: log sizes live in the log-segment store, not here; drop only - -- with a migration (`UNUSED` in control/schema_drift.rs). - byte_count bigint NOT NULL DEFAULT 0, - line_count bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now(), UNIQUE (plan_id, log_id) ); @@ -456,7 +444,6 @@ CREATE TABLE runners ( registered_at timestamptz NOT NULL DEFAULT now(), last_seen_at timestamptz ); -CREATE INDEX runners_labels ON runners USING gin (labels); CREATE TABLE runner_sessions ( session_id uuid PRIMARY KEY, @@ -464,9 +451,6 @@ CREATE TABLE runner_sessions ( protocol text NOT NULL CHECK (protocol IN ('broker','azdo')), client_id text, verified boolean NOT NULL DEFAULT false, - -- UNUSED: wake routing is in-process (`control/wake.rs`); drop only with - -- a migration (`UNUSED` in control/schema_drift.rs). - engine_node_id text, created_at timestamptz NOT NULL DEFAULT now(), last_seen_at timestamptz NOT NULL DEFAULT now() ); @@ -520,10 +504,6 @@ CREATE TABLE provision_requests ( pool_key text NOT NULL, labels jsonb NOT NULL, requested_at timestamptz NOT NULL DEFAULT now(), - -- UNUSED: the provisioner queue has no lease protocol; drop only with a - -- migration (`UNUSED` in control/schema_drift.rs). - leased_until timestamptz, - lease_owner text, PRIMARY KEY (run_id, job_id), FOREIGN KEY (run_id, job_id) REFERENCES jobs(run_id, job_id) ON DELETE CASCADE ); @@ -687,9 +667,8 @@ CREATE INDEX check_run_updates_queue ON check_run_updates(installation_id, not_b -- ── Artifacts (replaces the artifact part of the `meta` blob) ──────── -- Blobs live in object storage; upload state lives in the file-backed --- ArtifactStore. These rows are RESERVED (planned as the shared artifact --- index): the only production statement is the run-archive DELETE. See --- `RESERVED_UNUSED` in control/schema_drift.rs. +-- ArtifactStore. These rows are the catalog; the only statement today is +-- the run-archive DELETE. -- -- The Actions cache is deliberately NOT here: it is a bounded cache -- colocated with the runner hosts (preloop-cache CAS: key index -> @@ -705,10 +684,6 @@ CREATE TABLE artifacts ( size_bytes bigint, digest text, storage_key text NOT NULL, - -- UNUSED: `upload_token_hash` and `finalized_at` have no reader or - -- writer in any code; drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). - upload_token_hash bytea, created_at timestamptz NOT NULL DEFAULT now(), finalized_at timestamptz, expires_at timestamptz, diff --git a/crates/preloop-runner-server/src/control/schema_drift.rs b/crates/preloop-runner-server/src/control/schema_drift.rs index 6fe386819..d638cb68b 100644 --- a/crates/preloop-runner-server/src/control/schema_drift.rs +++ b/crates/preloop-runner-server/src/control/schema_drift.rs @@ -1,5 +1,4 @@ -//! Schema-drift guard between the two copies of the control schema, plus the -//! reserved/unused inventory the maintainers must keep honest. +//! Schema-drift guard between the two copies of the control schema. //! //! `lite/schema.sql` is the SQLite translation of `pg/schema.sql` (the //! source of truth, mirrored into `docs/control-schema.sql`). Both must @@ -26,7 +25,6 @@ //! and to name every place the two backends disagree. use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; /// The SQLite translation (see the file header for the type mapping). const LITE_SCHEMA: &str = include_str!("lite/schema.sql"); @@ -82,12 +80,6 @@ const DOCUMENTED_DIFFERENCES: &[(&str, &str)] = &[ "index outbox_events_read: pg only", "the (txid, event_id) reader index; there is no txid column on SQLite (see above).", ), - // ── Indexes that need an engine feature SQLite has no equivalent of ─ - ( - "index runners_labels: pg only", - "GIN index over the `labels` jsonb (label -> runner search). SQLite has no GIN; lite \ - matches labels in Rust (`runner_label_sets`) over the same JSON payload.", - ), // ── Columns and constraints that encode a deliberate behavioral split ─ ( "table job_messages column job_timeout_s: pg only", @@ -120,144 +112,6 @@ const DOCUMENTED_DIFFERENCES: &[(&str, &str)] = &[ ), ]; -/// Schema objects kept although *no* code reads them yet — placeholders for -/// features the maintainers intend to wire up. The test below greps the Rust -/// sources so the day one of them starts being used, this list (and the -/// matching `-- RESERVED:` comment in both schemas) must be revisited. -const RESERVED_UNUSED: &[Kept] = &[ - Kept { - what: "table namespace_policies", - why: "planned: hosted-tenant admission policy (fork-PR policy, token permission ceiling, \ - OIDC audiences, local-execution policy)", - needles: &[("namespace_policies", None)], - allowed_mentions: &[], - }, - Kept { - what: "column namespace_limits.max_job_timeout_minutes", - why: "planned: per-tenant job timeout cap, enforced at claim/acquire", - needles: &[("namespace_limits", Some("max_job_timeout_minutes"))], - allowed_mentions: &[], - }, - Kept { - what: "column namespace_limits.priority_tier", - why: "planned: input to the tenant ordering policy (dispatch orders by priority, \ - run_order, job_order today)", - needles: &[("namespace_limits", Some("priority_tier"))], - allowed_mentions: &[], - }, - Kept { - what: "column namespace_limits.run_history_retention_days", - why: "planned: per-tenant history retention window", - needles: &[("namespace_limits", Some("run_history_retention_days"))], - allowed_mentions: &[], - }, - Kept { - what: "table artifacts", - why: "planned: shared artifact index (bytes live in the file-backed ArtifactStore). \ - Today the only production statements are the run-archive `DELETE FROM artifacts`; \ - the two test-only statements in retention.rs are listed as allowed mentions.", - needles: &[("artifacts", None)], - allowed_mentions: &[ - "SELECT count(*) FROM artifacts WHERE run_id = ?1", - "INSERT INTO artifacts (namespace_id, run_id, job_backend_id, name", - ], - }, -]; - -/// Schema objects with no reader and no writer anywhere in this crate. They -/// are *not* dropped here: this branch has no migration runner, and a -/// database created from this file refuses any other stamped version (see -/// the module docs), so removing a column would strand existing databases. -/// Drop them only in a migration-backed change; until then this list (and the -/// `-- UNUSED:` comment at each site) is the inventory. -const UNUSED: &[Kept] = &[ - Kept { - what: "column namespaces.cell_generation", - why: "pg planned it to fence stale writers after a cell move; no code reads or writes it", - needles: &[("namespaces", Some("cell_generation"))], - allowed_mentions: &[], - }, - Kept { - what: "column namespaces.config_version", - why: "pg planned it for platform config pushes; no code reads or writes it", - needles: &[("namespaces", Some("config_version"))], - allowed_mentions: &[], - }, - Kept { - what: "column run_submissions.secret_refs", - why: "secret values are resolved at acquire and never stored; no code reads or writes the \ - column (the secret-name list travels in `submission`)", - needles: &[("run_submissions", Some("secret_refs"))], - allowed_mentions: &[], - }, - Kept { - what: "column jobs.not_before", - why: "retry-backoff / delayed-start leftover from the old scheduler; the queue orders by \ - priority, run_order, job_order", - needles: &[("jobs", Some("not_before"))], - allowed_mentions: &[], - }, - Kept { - what: "column provision_requests.lease_owner", - why: "the provisioner queue has no lease protocol; no code reads or writes it", - needles: &[("provision_requests", Some("lease_owner"))], - allowed_mentions: &[], - }, - Kept { - what: "column provision_requests.leased_until", - why: "the provisioner queue has no lease protocol; no code reads or writes it", - needles: &[("provision_requests", Some("leased_until"))], - allowed_mentions: &[], - }, - Kept { - what: "column runner_sessions.engine_node_id", - why: "long-poll wake routing is in-process (`control/wake.rs`), not persisted", - needles: &[("runner_sessions", Some("engine_node_id"))], - allowed_mentions: &[], - }, - Kept { - what: "column log_files.byte_count", - why: "log sizes live in the log-segment store; the control table only allocates \ - (plan_id, log_id)", - needles: &[("log_files", Some("byte_count"))], - allowed_mentions: &[], - }, - Kept { - what: "column log_files.line_count", - why: "log sizes live in the log-segment store; the control table only allocates \ - (plan_id, log_id)", - needles: &[("log_files", Some("line_count"))], - allowed_mentions: &[], - }, - Kept { - what: "column artifacts.upload_token_hash", - why: "the file-backed ArtifactStore owns upload state; the control table is a delete-only \ - index today", - needles: &[("artifacts", Some("upload_token_hash"))], - allowed_mentions: &[], - }, - Kept { - what: "column artifacts.finalized_at", - why: "the file-backed ArtifactStore owns upload state; the control table is a delete-only \ - index today", - needles: &[("artifacts", Some("finalized_at"))], - allowed_mentions: &[], - }, -]; - -/// One kept-but-unused schema object and the evidence it is still unused. -struct Kept { - /// Human description used in the failure message. - what: &'static str, - /// Why it is kept. Quoted in the failure message. - why: &'static str, - /// `(table, column)`: a column when set, a whole table otherwise. - needles: &'static [(&'static str, Option<&'static str>)], - /// Statement fragments that are known to mention the object and are - /// accepted (test-only helpers); every other match fails the test. - allowed_mentions: &'static [&'static str], -} - #[test] fn control_schema_differences_are_documented() { let differences = differences(); @@ -283,62 +137,6 @@ fn control_schema_differences_are_documented() { ); } -#[test] -fn reserved_and_unused_objects_stay_unreferenced() { - let sources = rust_sources(); - let mut findings = Vec::new(); - for kept in RESERVED_UNUSED.iter().chain(UNUSED) { - for (table, column) in kept.needles { - for (path, text) in &sources { - if path.ends_with("schema_drift.rs") { - continue; - } - for statement in sql_mentions(text, table, *column) { - if kept - .allowed_mentions - .iter() - .any(|allowed| statement.contains(allowed)) - { - continue; - } - findings.push(format!( - "{} ({}): {statement}\n in {}", - kept.what, - kept.why, - path.display() - )); - } - } - } - } - assert!( - findings.is_empty(), - "these schema objects are declared RESERVED/UNUSED but the code references them now:\n{}\n\ - If this is a real use, remove the object from RESERVED_UNUSED/UNUSED in \ - control/schema_drift.rs, delete its `-- RESERVED:`/`-- UNUSED:` comment in both schemas, \ - and (for a wired-up feature) drop the `unused` claim from the PR.", - findings.join("\n") - ); -} - -/// The inventory grep must see a column used in a `DELETE` predicate: that is -/// a real read of the column, and `DELETE` alone used to be invisible to it. -/// A table-only `DELETE` stays invisible — the `artifacts` table is -/// delete-only by design, so its cleanup statements are not uses. -#[test] -fn delete_predicates_count_as_column_uses() { - let delete = "DELETE FROM provision_requests WHERE lease_owner = ?1"; - assert_eq!( - sql_mentions(delete, "provision_requests", Some("lease_owner")).len(), - 1, - "a DELETE predicate reads the column" - ); - assert!( - sql_mentions(delete, "provision_requests", None).is_empty(), - "a table-only DELETE is not a read" - ); -} - // ── The parsed schemas ────────────────────────────────────────────────── #[derive(Debug, Default, Clone)] @@ -601,71 +399,3 @@ fn find_word(haystack: &str, needle: &str) -> Option { } None } - -// ── Source grep for the reserved/unused inventory ─────────────────────── - -fn rust_sources() -> Vec<(PathBuf, String)> { - fn walk(dir: &Path, out: &mut Vec<(PathBuf, String)>) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - walk(&path, out); - } else if path.extension().is_some_and(|ext| ext == "rs") - && let Ok(text) = std::fs::read_to_string(&path) - { - out.push((path, text)); - } - } - } - let mut out = Vec::new(); - walk(&Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), &mut out); - out -} - -/// Every mention of `table` (or `table.column` when `column` is set) inside a -/// SQL read/write statement. SQL is written across several Rust string lines, -/// so any statement verb within 240 bytes before the mention counts as the -/// same statement. `DROP` is deliberately not a verb, and a bare `DELETE` of -/// a *table* is not a read (the `artifacts` table is delete-only by design), -/// but `DELETE ... WHERE ` does read that column, so `DELETE` counts -/// when the needle is a column. -/// -/// The window only looks backwards, so a mention that puts the table name -/// after the column (some `SELECT` shapes) can be missed — this guard is a -/// prompt, not a proof. -fn sql_mentions(text: &str, table: &str, column: Option<&str>) -> Vec { - const VERBS: &[&str] = &["SELECT", "INSERT", "UPDATE", "JOIN"]; - let needle = column.unwrap_or(table); - let mut mentions = Vec::new(); - let mut from = 0; - while let Some(at) = find_word(&text[from..], needle) { - let at = from + at; - from = at + 1; - let start = char_window_start(text, at, 240); - let window = &text[start..at]; - let delete_reads_a_column = column.is_some() && find_word(window, "DELETE").is_some(); - if !delete_reads_a_column && !VERBS.iter().any(|verb| find_word(window, verb).is_some()) { - continue; - } - if column.is_some() && find_word(window, table).is_none() { - continue; - } - let line_start = text[..at].rfind('\n').map_or(0, |newline| newline + 1); - let line_end = text[at..].find('\n').map_or(text.len(), |end| at + end); - mentions.push(text[line_start..line_end].trim().to_owned()); - } - mentions -} - -/// Start of the `max_chars`-long window that ends at `at`, on a char -/// boundary. -fn char_window_start(text: &str, at: usize, max_chars: usize) -> usize { - text[..at] - .char_indices() - .rev() - .nth(max_chars) - .map_or(0, |(index, _)| index) -} diff --git a/docs/control-schema.sql b/docs/control-schema.sql index 902bc842d..17b14c0b6 100644 --- a/docs/control-schema.sql +++ b/docs/control-schema.sql @@ -47,9 +47,8 @@ CREATE TABLE namespaces ( -- suspended/deleted: neither (queued jobs stay queued, nothing starts). state text NOT NULL DEFAULT 'active' CHECK (state IN ('active','suspended','draining','deleted')), - -- UNUSED: `cell_generation` and `config_version` have no reader or - -- writer in any code; drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). + -- Platform-owned: cell fencing and config push bookkeeping; the engine + -- neither reads nor writes these two. cell_generation bigint NOT NULL DEFAULT 1, -- bumped on cell move; fences stale writers config_version bigint NOT NULL DEFAULT 0, -- last platform push applied created_at timestamptz NOT NULL DEFAULT now(), @@ -62,9 +61,8 @@ CREATE TABLE namespaces ( -- predicate (a capped claim locks this row first, so nodes cannot -- overshoot). Not yet read: max_job_timeout_minutes, priority_tier, -- run_history_retention_days. --- RESERVED: not read by any code yet (planned: per-tenant job timeout cap, --- ordering input, history retention window); see `RESERVED_UNUSED` in --- control/schema_drift.rs. +-- Those three are platform-owned: written by the hosted platform, not yet +-- enforced by the engine. CREATE TABLE namespace_limits ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, max_queued_jobs integer, @@ -85,8 +83,8 @@ CREATE TABLE namespace_pool_limits ( ); -- Admission and job-build restrictions. --- RESERVED: not read by any code yet (planned: hosted-tenant admission --- policy); see `RESERVED_UNUSED` in control/schema_drift.rs. +-- Platform-owned: the hosted platform writes per-tenant admission policy +-- here; the engine does not read it yet. CREATE TABLE namespace_policies ( namespace_id text PRIMARY KEY REFERENCES namespaces(namespace_id) ON DELETE CASCADE, fork_pr_policy text NOT NULL DEFAULT 'untrusted' CHECK (fork_pr_policy IN @@ -184,9 +182,6 @@ CREATE TABLE run_submissions ( github_context jsonb NOT NULL, workspace_snapshot jsonb, snapshot_timing jsonb, -- duration_ms, object_count, pack_bytes - -- UNUSED: no reader or writer in any code; drop only with a migration - -- (`UNUSED` in control/schema_drift.rs). - secret_refs jsonb NOT NULL DEFAULT '{}', -- Record-level per-job maps the table layout has no column for (jobs -- with no `jobs` row yet — a check run minted before its matrix leg -- materializes). Mirrors lite's `run_submissions.record_details`. @@ -233,9 +228,6 @@ CREATE TABLE jobs ( priority integer NOT NULL DEFAULT 0, run_order bigint NOT NULL DEFAULT 0, job_order integer NOT NULL DEFAULT 0, - -- UNUSED: retry-backoff / delayed-start leftover; drop only with a - -- migration (`UNUSED` in control/schema_drift.rs). - not_before timestamptz, enqueued_at timestamptz, claimed_by_runner_id bigint, claimed_at timestamptz, @@ -430,10 +422,6 @@ CREATE TABLE log_files ( run_id uuid NOT NULL REFERENCES runs(run_id) ON DELETE CASCADE, plan_id uuid NOT NULL REFERENCES job_requests(agent_job_id) ON DELETE CASCADE, log_id integer NOT NULL CHECK (log_id > 0), - -- UNUSED: log sizes live in the log-segment store, not here; drop only - -- with a migration (`UNUSED` in control/schema_drift.rs). - byte_count bigint NOT NULL DEFAULT 0, - line_count bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now(), UNIQUE (plan_id, log_id) ); @@ -455,7 +443,6 @@ CREATE TABLE runners ( registered_at timestamptz NOT NULL DEFAULT now(), last_seen_at timestamptz ); -CREATE INDEX runners_labels ON runners USING gin (labels); CREATE TABLE runner_sessions ( session_id uuid PRIMARY KEY, @@ -463,9 +450,6 @@ CREATE TABLE runner_sessions ( protocol text NOT NULL CHECK (protocol IN ('broker','azdo')), client_id text, verified boolean NOT NULL DEFAULT false, - -- UNUSED: wake routing is in-process (`control/wake.rs`); drop only with - -- a migration (`UNUSED` in control/schema_drift.rs). - engine_node_id text, created_at timestamptz NOT NULL DEFAULT now(), last_seen_at timestamptz NOT NULL DEFAULT now() ); @@ -519,10 +503,6 @@ CREATE TABLE provision_requests ( pool_key text NOT NULL, labels jsonb NOT NULL, requested_at timestamptz NOT NULL DEFAULT now(), - -- UNUSED: the provisioner queue has no lease protocol; drop only with a - -- migration (`UNUSED` in control/schema_drift.rs). - leased_until timestamptz, - lease_owner text, PRIMARY KEY (run_id, job_id), FOREIGN KEY (run_id, job_id) REFERENCES jobs(run_id, job_id) ON DELETE CASCADE ); @@ -686,9 +666,8 @@ CREATE INDEX check_run_updates_queue ON check_run_updates(installation_id, not_b -- ── Artifacts (replaces the artifact part of the `meta` blob) ──────── -- Blobs live in object storage; upload state lives in the file-backed --- ArtifactStore. These rows are RESERVED (planned as the shared artifact --- index): the only production statement is the run-archive DELETE. See --- `RESERVED_UNUSED` in control/schema_drift.rs. +-- ArtifactStore. These rows are the catalog; the only statement today is +-- the run-archive DELETE. -- -- The Actions cache is deliberately NOT here: it is a bounded cache -- colocated with the runner hosts (preloop-cache CAS: key index -> @@ -704,10 +683,6 @@ CREATE TABLE artifacts ( size_bytes bigint, digest text, storage_key text NOT NULL, - -- UNUSED: `upload_token_hash` and `finalized_at` have no reader or - -- writer in any code; drop only with a migration (`UNUSED` in - -- control/schema_drift.rs). - upload_token_hash bytea, created_at timestamptz NOT NULL DEFAULT now(), finalized_at timestamptz, expires_at timestamptz,