diff --git a/README.md b/README.md index e0c28f5..2f8ad9e 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,11 @@ zombie-bite spawn -d /tmp/base_path --apply-upgrade - `--para-cores =` overrides how many cores a parachain gets (defaults mirror the live networks, e.g. asset-hub takes 3 for elastic scaling). The relay's validator count follows the total. - `--keep-messaging-state` keeps the inherited HRMP/DMP state instead of clearing it. Only correct when the relay's parachains are exactly the ones being bitten, so both snapshots agree on channel heads; on a shared relay the mismatch makes cumulus panic with `HRMP head mismatch`. +- `--publish-bootnodes` writes the spawned network's own node addresses into the published chain-specs, so the artifacts are dialable from another machine (specs otherwise ship with `bootNodes: []`). + +#### Bundle manifest + +A bite writes a `manifest.json` next to `ready.json` recording, per chain, the bite block, source RPC, spec and snapshot files with sizes, any carried upgrade, and the `doppelganger` versions that produced the snapshots. A later `spawn` warns when the local binaries differ, because a snapshot from a newer node fails to restore in ways that otherwise look like corruption. #### Overrides are checked against the runtime diff --git a/src/cli.rs b/src/cli.rs index f5405c1..2f6298b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -58,6 +58,10 @@ pub enum Commands { /// Can be set multiple times, once per para. #[arg(long = "para-cores", verbatim_doc_comment)] para_cores: Vec, + /// Write the spawned network's own node addresses into the published + /// chain-specs, so the artifacts are dialable from outside the box. + #[arg(long, default_value_t = false, verbatim_doc_comment)] + publish_bootnodes: bool, /// If provided we will _bite_ the live network at the supplied block hieght #[arg(long = "rc-bite-at", verbatim_doc_comment)] relay_bite_at: Option, @@ -102,6 +106,10 @@ pub enum Commands { /// and wait until it enacts. #[arg(long, default_value_t = false, verbatim_doc_comment)] apply_upgrade: bool, + /// Write the spawned network's own node addresses into the published + /// chain-specs, so the artifacts are dialable from outside the box. + #[arg(long, default_value_t = false, verbatim_doc_comment)] + publish_bootnodes: bool, }, /// [Helper] Generate artifacts to be used by the next step (only 'spawn' and 'post' allowed) GenerateArtifacts { @@ -166,6 +174,7 @@ pub struct ResolvedBiteConfig { pub base_path: PathBuf, pub and_spawn: bool, pub apply_upgrade: bool, + pub publish_bootnodes: bool, pub opts: BiteOptions, } @@ -174,6 +183,7 @@ pub struct ResolvedSpawnConfig { pub base_path: PathBuf, pub with_monitor: bool, pub apply_upgrade: bool, + pub publish_bootnodes: bool, } #[allow(clippy::too_many_arguments)] @@ -191,6 +201,7 @@ pub fn resolve_bite_config( apply_upgrade: bool, keep_messaging_state: bool, para_cores: Vec, + publish_bootnodes: bool, ) -> Result { // Load config file if provided let config_file = if let Some(path) = config_path { @@ -364,6 +375,7 @@ pub fn resolve_bite_config( base_path: resolved_base_path, and_spawn: resolved_and_spawn, apply_upgrade: resolved_apply_upgrade, + publish_bootnodes, opts: BiteOptions { upgrades, cores, @@ -377,6 +389,7 @@ pub fn resolve_spawn_config( base_path: Option, with_monitor: bool, apply_upgrade: bool, + publish_bootnodes: bool, ) -> Result { // Load config file if provided let config_file = if let Some(path) = config_path { @@ -413,6 +426,7 @@ pub fn resolve_spawn_config( base_path: resolved_base_path, with_monitor: resolved_with_monitor, apply_upgrade: resolved_apply_upgrade, + publish_bootnodes, }) } diff --git a/src/doppelganger.rs b/src/doppelganger.rs index 39cc7a9..90a579e 100644 --- a/src/doppelganger.rs +++ b/src/doppelganger.rs @@ -40,6 +40,7 @@ use crate::utils::{ use crate::config::{ get_assigned_cores, get_state_pruning_config, BiteOptions, Context, Parachain, Relaychain, Step, }; +use crate::manifest::{self, ChainEntry, Manifest}; use crate::metadata::ChainMetadata; use crate::overrides::{generate_default_overrides_for_para, generate_default_overrides_for_rc}; use crate::sync::{sync_para, sync_relay_only}; @@ -329,8 +330,8 @@ pub async fn doppelganger_inner( }; let config = generate_config( - relay_artifacts, - para_artifacts, + relay_artifacts.clone(), + para_artifacts.clone(), Some(global_base_dir.clone()), database, req_cores, @@ -441,11 +442,80 @@ pub async fn doppelganger_inner( ) .await; + let manifest = build_manifest( + &relay_chain, + ¶s_to, + &ready_content, + &relay_artifacts, + ¶_artifacts, + ) + .await; + manifest.write(&global_base_dir).await?; + clean_up_dir_for_step(global_base_dir, Step::Bite, &relay_chain, ¶s_to).await?; Ok(()) } +async fn build_manifest( + relay_chain: &Relaychain, + paras_to: &[Parachain], + ready: &serde_json::Value, + relay_artifacts: &ChainArtifact, + para_artifacts: &[ChainArtifact], +) -> Manifest { + let file_name = |path: &str| { + Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + }; + + let relay = ChainEntry { + chain: relay_chain.as_chain_string(), + para_id: None, + bite_block: ready["rc_start_block"].as_u64(), + source_rpc: ready["rc_source_rpc"].as_str().map(str::to_string), + spec_file: file_name(&relay_artifacts.spec_path), + snapshot_file: file_name(&relay_artifacts.snap_path), + snapshot_bytes: manifest::file_size(&relay_artifacts.snap_path).await, + upgrade_file: ready["rc_upgrade_wasm"].as_str().map(str::to_string), + upgrade_hash: ready["rc_upgrade_hash"].as_str().map(str::to_string), + }; + + let mut parachains = vec![]; + for (index, para) in paras_to.iter().enumerate() { + let id = para.id(); + let artifact = para_artifacts.get(index); + parachains.push(ChainEntry { + chain: para.as_chain_string(&relay_chain.as_chain_string()), + para_id: Some(id), + bite_block: ready[format!("para_{id}_start_block")].as_u64(), + source_rpc: ready[format!("para_{id}_source_rpc")] + .as_str() + .map(str::to_string), + spec_file: artifact.and_then(|a| file_name(&a.spec_path)), + snapshot_file: artifact.and_then(|a| file_name(&a.snap_path)), + snapshot_bytes: match artifact { + Some(a) => manifest::file_size(&a.snap_path).await, + None => None, + }, + upgrade_file: ready[format!("para_{id}_upgrade_wasm")] + .as_str() + .map(str::to_string), + upgrade_hash: ready[format!("para_{id}_upgrade_hash")] + .as_str() + .map(str::to_string), + }); + } + + Manifest { + created_at: manifest::now_unix(), + relay, + parachains, + binaries: manifest::binary_versions().await, + } +} + async fn copy_upgrade_blob(from: &str, base_dir: &str, blob_name: &str) -> (String, String) { let wasm = fs::read(from) .await @@ -947,6 +1017,70 @@ async fn validate_parachain_specs( Ok(()) } +/// Write the spawned network's own node addresses into the chain-specs the +/// artifacts are published from. +/// +/// `generate_chain_spec` clears `bootNodes`, which is safe but leaves a +/// published spec unusable off the box: the peer wiring only exists in the +/// spawned nodes' arguments. Filling in the running nodes' multiaddrs makes the +/// artifacts dialable without every consumer patching the specs itself. +pub async fn publish_bootnodes( + network: &Network, + base_path: &Path, + step: Step, +) -> Result<(), anyhow::Error> { + let spec_dir = format!("{}/{}", base_path.to_string_lossy(), step.dir_from()); + + let relay_nodes: Vec = network + .relaychain() + .nodes() + .iter() + .map(|node| node.multiaddr().to_string()) + .collect(); + write_bootnodes( + &format!("{spec_dir}/{}-spec.json", network.relaychain().chain()), + &relay_nodes, + ) + .await?; + + for para in network.parachains() { + let Some(chain_id) = para.chain_id() else { + warn!( + "para {}: no chain id, can't tell which spec to write bootnodes into", + para.para_id() + ); + continue; + }; + let nodes: Vec = para + .collators() + .iter() + .map(|node| node.multiaddr().to_string()) + .collect(); + write_bootnodes(&format!("{spec_dir}/{chain_id}-spec.json"), &nodes).await?; + } + + Ok(()) +} + +async fn write_bootnodes(spec_path: &str, nodes: &[String]) -> Result<(), anyhow::Error> { + if nodes.is_empty() { + warn!("{spec_path}: no running nodes to use as bootnodes"); + return Ok(()); + } + let Ok(content) = fs::read_to_string(spec_path).await else { + warn!("{spec_path}: chain-spec not found, skipping bootnodes"); + return Ok(()); + }; + let mut spec: serde_json::Value = serde_json::from_str(&content)?; + spec["bootNodes"] = json!(nodes); + fs::write(spec_path, serde_json::to_string_pretty(&spec)?).await?; + info!( + "{spec_path}: bootNodes set to the spawned network ({})", + nodes.len() + ); + Ok(()) +} + async fn generate_snap(data_path: &str, snap_path: &str) -> Result<(), anyhow::Error> { info!("\nšŸ“ Generating snapshot file {snap_path} with data_path {data_path}..."); diff --git a/src/main.rs b/src/main.rs index caf26a6..c983a0f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use zombienet_sdk::{LocalFileSystem, Network, NetworkNode}; mod cli; mod config; mod doppelganger; +mod manifest; mod metadata; mod monit; mod overrides; @@ -168,6 +169,7 @@ async fn main() -> Result<(), anyhow::Error> { apply_upgrade, keep_messaging_state, para_cores, + publish_bootnodes, } => { if with_monitor && !and_spawn { bail!("--with-monitor can only be used with --and-spawn"); @@ -187,6 +189,7 @@ async fn main() -> Result<(), anyhow::Error> { apply_upgrade, keep_messaging_state, para_cores, + publish_bootnodes, )?; if resolved_config.apply_upgrade && !resolved_config.and_spawn { @@ -225,6 +228,15 @@ async fn main() -> Result<(), anyhow::Error> { verify::verify_fork(&network, resolved_config.base_path.as_path()).await?; + if resolved_config.publish_bootnodes { + doppelganger::publish_bootnodes( + &network, + resolved_config.base_path.as_path(), + step, + ) + .await?; + } + if resolved_config.apply_upgrade { upgrade::apply_from_ready(&network, resolved_config.base_path.as_path()) .await?; @@ -242,9 +254,15 @@ async fn main() -> Result<(), anyhow::Error> { with_monitor, step, apply_upgrade, + publish_bootnodes, } => { - let resolved_config = - resolve_spawn_config(config, base_path, with_monitor, apply_upgrade)?; + let resolved_config = resolve_spawn_config( + config, + base_path, + with_monitor, + apply_upgrade, + publish_bootnodes, + )?; let step: Step = step.into(); let base_path_str = resolved_config.base_path.to_string_lossy(); @@ -260,6 +278,8 @@ async fn main() -> Result<(), anyhow::Error> { resolve_if_dir_exist(&resolved_config.base_path, step).await; + manifest::warn_on_binary_mismatch(resolved_config.base_path.as_path()).await; + let network = doppelganger::spawn(step, resolved_config.base_path.as_path(), None, None) .await @@ -269,6 +289,15 @@ async fn main() -> Result<(), anyhow::Error> { verify::verify_fork(&network, resolved_config.base_path.as_path()).await?; + if resolved_config.publish_bootnodes { + doppelganger::publish_bootnodes( + &network, + resolved_config.base_path.as_path(), + step, + ) + .await?; + } + if resolved_config.apply_upgrade { upgrade::apply_from_ready(&network, resolved_config.base_path.as_path()).await?; } diff --git a/src/manifest.rs b/src/manifest.rs new file mode 100644 index 0000000..7368705 --- /dev/null +++ b/src/manifest.rs @@ -0,0 +1,137 @@ +//! Bundle manifest. +//! +//! A bite is often produced in CI and restored hours later on another machine, +//! so the artifacts have to describe themselves: which block each chain was +//! bitten at, where the state came from, and which binaries produced it. The +//! last one matters because a snapshot written by a newer node fails to restore +//! in ways that look like corruption. + +use std::{path::Path, time::SystemTime, time::UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; +use tokio::{fs, process::Command}; +use tracing::{info, warn}; + +pub const MANIFEST_FILE: &str = "manifest.json"; + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct ChainEntry { + pub chain: String, + pub para_id: Option, + /// Block the state was captured at. + pub bite_block: Option, + /// Network the state came from. + pub source_rpc: Option, + pub spec_file: Option, + pub snapshot_file: Option, + pub snapshot_bytes: Option, + /// Runtime carried as an authorized upgrade, if any. + pub upgrade_file: Option, + pub upgrade_hash: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Manifest { + pub created_at: u64, + pub relay: ChainEntry, + pub parachains: Vec, + /// `--version` of the binaries that produced the snapshots. + pub binaries: Vec<(String, String)>, +} + +async fn binary_version(cmd: &str) -> Option { + let out = Command::new(cmd).arg("--version").output().await.ok()?; + let version = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (!version.is_empty()).then_some(version) +} + +pub async fn binary_versions() -> Vec<(String, String)> { + let mut versions = vec![]; + for cmd in ["doppelganger", "doppelganger-parachain"] { + if let Some(version) = binary_version(cmd).await { + versions.push((cmd.to_string(), version)); + } + } + versions +} + +pub async fn file_size(path: &str) -> Option { + fs::metadata(path).await.ok().map(|m| m.len()) +} + +pub fn now_unix() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or_default() +} + +impl Manifest { + pub async fn write(&self, base_path: &Path) -> Result<(), anyhow::Error> { + let path = base_path.join(MANIFEST_FILE); + fs::write(&path, serde_json::to_string_pretty(self)?).await?; + info!("šŸ“„ manifest written to {}", path.to_string_lossy()); + Ok(()) + } + + pub async fn read(base_path: &Path) -> Option { + let content = fs::read_to_string(base_path.join(MANIFEST_FILE)) + .await + .ok()?; + serde_json::from_str(&content).ok() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn manifest_round_trips() { + let dir = Path::new("/tmp/zb-manifest-test"); + fs::create_dir_all(dir).await.unwrap(); + + let manifest = Manifest { + created_at: 1, + relay: ChainEntry { + chain: "kusama".into(), + bite_block: Some(42), + source_rpc: Some("wss://example".into()), + snapshot_bytes: Some(123), + ..Default::default() + }, + parachains: vec![ChainEntry { + chain: "asset-hub-kusama".into(), + para_id: Some(1000), + bite_block: Some(7), + ..Default::default() + }], + binaries: vec![("doppelganger".into(), "1.2.3".into())], + }; + manifest.write(dir).await.unwrap(); + + let read = Manifest::read(dir).await.expect("manifest should be there"); + assert_eq!(read.relay.bite_block, Some(42)); + assert_eq!(read.parachains[0].para_id, Some(1000)); + assert_eq!(read.binaries[0].1, "1.2.3"); + } +} + +/// Compare the binaries that produced the bundle with the ones on this machine. +/// A mismatch is a warning, not an error: it usually still restores, and when it +/// does not the failure otherwise looks like a corrupt snapshot. +pub async fn warn_on_binary_mismatch(base_path: &Path) { + let Some(manifest) = Manifest::read(base_path).await else { + return; + }; + let local = binary_versions().await; + for (cmd, bundled) in &manifest.binaries { + match local.iter().find(|(name, _)| name == cmd) { + Some((_, current)) if current == bundled => {} + Some((_, current)) => warn!( + "{cmd}: bundle was produced with '{bundled}' but this machine has '{current}'; a snapshot from a newer node can fail to restore in ways that look like corruption" + ), + None => warn!("{cmd}: not found locally, can't compare with the bundle's '{bundled}'"), + } + } +}