feat: multi-box deployment

.github/workflows/check.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        host: [finney, kingfisher]
+        host: [finney, kingfisher, albatross]
     steps:
       - uses: actions/checkout@v4
       - uses: cachix/install-nix-action@v30

flake.nix

@@ -57,8 +57,14 @@
     {
       nixosConfigurations = {
         finney = mkHost "finney" [ ];
-        kingfisher = mkHost "kingfisher" [ roost.nixosModules.default ];
-        albatross = mkHost "albatross" [ roost.nixosModules.default ];
+        kingfisher = mkHost "kingfisher" [
+          roost.nixosModules.default
+          roost.nixosModules.wireguard-mesh
+        ];
+        albatross = mkHost "albatross" [
+          roost.nixosModules.frigate-edge
+          roost.nixosModules.wireguard-mesh
+        ];
       };
 
       formatter = forAllSystems (system: nixpkgs.legacyPackages.${system}.nixfmt-tree);

hosts/_mesh.nix

@@ -0,0 +1,28 @@
+{ ... }:
+
+# Shared peer definitions for the private WireGuard mesh between
+# kingfisher (the all-in-one frigate node) and albatross (the
+# frigate-edge consumer). Imported by both hosts' wireguard.nix so the
+# `peers` block is one place. Adding a third node is a one-line edit
+# here plus that node's own `thisHost` + private-key wiring.
+#
+# Endpoint IPs are kingfisher's and albatross's public addresses. WG
+# resolves these once at interface setup; if either box's IP rotates
+# the matching entry below has to update too. The private keys
+# matching the public keys below live in secrets/wireguard-<host>.age,
+# encrypted to both `josie` and the host's own SSH key.
+
+{
+  services.roost.wireguard-mesh.peers = {
+    kingfisher = {
+      publicKey = "65eBW/IfinjLj7Q9HBnw+CBeEAx/6zaMVDejs+Vxb2o=";
+      endpoint = "136.243.9.246:51820";
+      meshIp = "10.42.0.1";
+    };
+    albatross = {
+      publicKey = "BZFpBTwYt3RUPFkIMQIrXZgkMDGryaae/empkoEiehE=";
+      endpoint = "46.62.185.45:51820";
+      meshIp = "10.42.0.2";
+    };
+  };
+}

hosts/albatross/default.nix

@@ -11,6 +11,7 @@
     ./hardware-configuration.nix
     ./gpu.nix
     ./frigate.nix
+    ./wireguard.nix
   ];
 
   networking.hostName = "albatross";

hosts/albatross/frigate.nix

@@ -6,61 +6,40 @@
 }:
 
 {
-  # Own hostname for now (rather than sharing kingfisher's
-  # frigate.2140.dev). Each backend self-issues its own cert via
-  # HTTP-01 — no DNS-01 wiring and no shared-cert coordination.
-  # Future load-balancing work will move both back behind a single
-  # hostname, fronted by a third host doing TCP/SNI passthrough.
-  services.public-frigate = {
+  # Edge-mode Frigate: TLS termination + ACME live here, the bitcoind /
+  # fulcrum / ZMQ stack runs on kingfisher and is consumed over the
+  # private WireGuard mesh (see ./wireguard.nix and ../_mesh.nix).
+  # This sidesteps the storage-capacity problem on albatross's
+  # ~866 GB pool — frigate's own DuckDB index is the only local data.
+  services.frigate-edge = {
     enable = true;
     host = "albatross.2140.dev";
     tls.acmeEmail = "josie@2140.dev";
-  };
 
-  # Add josie to the `bitcoin` group so `bitcoin-cli` works directly
-  # without `sudo -u bitcoin`. Operator name varies per box, so this
-  # stays per-host.
-  nix-bitcoin.operator = {
-    enable = true;
-    name = "josie";
+    backend = {
+      bitcoind = {
+        rpcUrl = "http://10.42.0.1:8332";
+        authCredentialFile = config.age.secrets.bitcoind-rpc-creds.path;
+        zmqSequenceEndpoint = "tcp://10.42.0.1:28336";
+      };
+      electrumUrl = "tcp://10.42.0.1:60001";
+    };
   };
 
-  # 48-thread Xeon Gold 5412U with 256 GB RAM and a Blackwell RTX PRO
-  # 6000 (96 GB VRAM). The roost preset's default dbCache of 4 GB
-  # underuses this box badly during IBD; bump it for faster initial
-  # sync. Drop back after sync if memory pressure shows up elsewhere.
-  services.bitcoind.dbCache = lib.mkForce 16384;
+  # `user:password` consumed by frigate via systemd LoadCredential. The
+  # matching `rpcauth=user:salt$hash` line lives on kingfisher in its
+  # `exposeBackends.rpcAuth.passwordHMAC` setting. Mode 0440 + owner
+  # `frigate` so the frigate user (declared by the bare frigate module)
+  # can read it for LoadCredential to pick up.
+  age.secrets.bitcoind-rpc-creds = {
+    file = ../../secrets/bitcoind-rpc-creds.age;
+    owner = "frigate";
+    mode = "0440";
+  };
 
   # systemd starts services with a stripped environment that does not
   # inherit NixOS's interactive-shell GPU library path. Without this,
   # frigate's JVM dlopen of libOpenCL.so.1 fails and DuckDB's ufsecp
   # extension silently falls back to CPU.
   systemd.services.frigate.environment.LD_LIBRARY_PATH = "/run/opengl-driver/lib";
-
-  # batchSize tuning is deferred: this GPU is roughly an order of
-  # magnitude more capable than kingfisher's RTX 4000 SFF Ada and the
-  # right value will only be visible after a real scan. The module
-  # default (300_000) is the starting point.
-
-  # BOOTSTRAP: keep the data-bearing services from autostarting on the
-  # first boot so /var/lib/{bitcoind,fulcrum,frigate} can be populated
-  # via `zfs recv` from kingfisher without racing live writes. The
-  # users/groups still get created (those come from the modules' user
-  # definitions, independent of wantedBy), so the post-recv chown
-  # step has somebody to chown to.
-  #
-  # Also disable autoUpgrade for the duration: the seeding workflow
-  # destroys the placeholder datasets before recv, which would leave
-  # mount units in a failed state until recv lands. An hourly rebuild
-  # firing during that window risks compounding the breakage and
-  # killed albatross on the first attempt.
-  #
-  # Remove this block once the import is complete and push; the next
-  # nixos-rebuild will land services in multi-user.target normally,
-  # they will start with the imported state, and autoUpgrade resumes
-  # polling.
-  systemd.services.bitcoind.wantedBy = lib.mkForce [ ];
-  systemd.services.fulcrum.wantedBy = lib.mkForce [ ];
-  systemd.services.frigate.wantedBy = lib.mkForce [ ];
-  system.autoUpgrade.enable = lib.mkForce false;
 }

hosts/albatross/hardware-configuration.nix

@@ -1,14 +1,25 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
 
 {
-  imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
-    ];
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
 
-  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ];
+  boot.initrd.availableKernelModules = [
+    "xhci_pci"
+    "ahci"
+    "nvme"
+    "usbhid"
+  ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];

hosts/albatross/wireguard.nix

@@ -0,0 +1,20 @@
+{ config, ... }:
+
+{
+  imports = [ ../_mesh.nix ];
+
+  # Encrypted private key. Agenix decrypts to /run/agenix/wireguard-albatross
+  # at activation; the wireguard module reads it as `privateKeyFile`. Mode
+  # 0400 keeps it root-only on disk.
+  age.secrets.wireguard-albatross = {
+    file = ../../secrets/wireguard-albatross.age;
+    mode = "0400";
+  };
+
+  services.roost.wireguard-mesh = {
+    enable = true;
+    thisHost = "albatross";
+    privateKeyFile = config.age.secrets.wireguard-albatross.path;
+    meshCidr = "10.42.0.0/24";
+  };
+}

hosts/kingfisher/default.nix

@@ -11,6 +11,7 @@
     ./hardware-configuration.nix
     ./gpu.nix
     ./frigate.nix
+    ./wireguard.nix
   ];
 
   networking.hostName = "kingfisher";

hosts/kingfisher/frigate.nix

@@ -14,6 +14,26 @@
     enable = true;
     host = "frigate.2140.dev";
     tls.acmeEmail = "josie@2140.dev";
+
+    # Expose bitcoind RPC + ZMQ + fulcrum on the WireGuard mesh interface
+    # so albatross can run frigate-edge against this stack instead of
+    # carrying its own ~950 GB chain copy. Interface-scoped firewall
+    # keeps these ports unreachable from the public internet.
+    #
+    # The HMAC below is committed (one-way derived from the password);
+    # the plaintext lives in secrets/bitcoind-rpc-creds.age on albatross.
+    # See modules/wireguard-mesh.nix for the mesh topology and
+    # hosts/_mesh.nix for the peer registry.
+    exposeBackends = {
+      enable = true;
+      bindAddress = "10.42.0.1";
+      interface = "wg0";
+      allowedPeers = [ "10.42.0.2/32" ];
+      rpcAuth = {
+        user = "frigate-edge";
+        passwordHMAC = "bec2842f5d4d3451316cc22f5db6560c$804448c1fd845e4160f5e6cc182b8250d5324679b9372e817fdb37c42ea71cc9";
+      };
+    };
   };
 
   # Operator pattern: add josie to the `bitcoin` group so `bitcoin-cli`

hosts/kingfisher/wireguard.nix

@@ -0,0 +1,20 @@
+{ config, ... }:
+
+{
+  imports = [ ../_mesh.nix ];
+
+  # Encrypted private key. Agenix decrypts to /run/agenix/wireguard-kingfisher
+  # at activation; the wireguard module reads it as `privateKeyFile`. Mode
+  # 0400 keeps it root-only on disk.
+  age.secrets.wireguard-kingfisher = {
+    file = ../../secrets/wireguard-kingfisher.age;
+    mode = "0400";
+  };
+
+  services.roost.wireguard-mesh = {
+    enable = true;
+    thisHost = "kingfisher";
+    privateKeyFile = config.age.secrets.wireguard-kingfisher.path;
+    meshCidr = "10.42.0.0/24";
+  };
+}

secrets/bitcoind-rpc-creds.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> X25519 HmHFvOhvEcpO8m1QxQOapgtRxbzYBy/Ep/wtZ2YFsRs
+eku0xiNGs2F6FYpwsjKJCJ9u66c5opg22mZAl/aWKoM
+-> ssh-ed25519 gk326w I6Yz6OCVP/J4xYDKIaKKPrUITRpwnmfbuYj3mcgDkzI
+pr67JMb6f7zIkNr8OeK1FVn3Uq3J/IanazqIzjN4PQA
+--- DgrxI4XH1OG1OUFvxHSK8jPfBjs7dvXDxfRrbDNA57g
+¦ÌÛ[t¹øù'ë>õ.ƒÌFþ
+5n9<!‰UZ5¥«r›õ<¦Í¸Dù˜ÏL׊5÷ž¡ú-×á/°
+ÐA!†Uݽì1”’­zi¦ÔàXª›HhüªvY

secrets/secrets.nix

@@ -11,15 +11,32 @@
 #        nix run github:ryantm/agenix -- --identity edit-key
 #      and paste its public form into `josie` below.
 #   4. Encrypt secrets:
-#        cd secrets && nix run github:ryantm/agenix -- -e bitcoind-rpcauth.age
+#        cd secrets && nix run github:ryantm/agenix -- -e <name>.age
 #   5. Reference them from a host module via `age.secrets.<name>.file = ./secrets/<name>.age;`.
 let
-  josie = "age1...REPLACE_ME_WITH_YOUR_AGE_PUBKEY";
+  josie = "age1jf8np2gw2wkd0k46x4z3plr47jz0kqvjker63jh2xqqjqpszcedsg2e6ug";
   finney = "ssh-ed25519 AAAA...REPLACE_ME_WITH_HOST_KEY_AFTER_FIRST_BOOT";
-  kingfisher = "ssh-ed25519 AAAA...REPLACE_ME_WITH_HOST_KEY_AFTER_FIRST_BOOT";
+  kingfisher = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB2j+A4rvxr+5JIP4XrRqAI3uHUOriAPpiDSc8F+izAG root@kingfisher";
+  albatross = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAPnn6DYBcz7nkpnOniTfwLtncQ8JlzYSjkFLd5uL5o3 root@albatross";
 in
 {
-  # Examples — uncomment and create the .age files when wiring services:
-  # "bitcoind-rpcauth.age".publicKeys = [ josie kingfisher ];
-  # "wireguard-finney.age".publicKeys = [ josie finney ];
+  # `user:password` for the bitcoind RPC user that frigate-edge on
+  # albatross uses to authenticate to kingfisher's bitcoind. The
+  # corresponding rpcauth HMAC lives in
+  # hosts/kingfisher/frigate.nix (services.public-frigate.exposeBackends.rpcAuth.passwordHMAC).
+  "bitcoind-rpc-creds.age".publicKeys = [
+    josie
+    albatross
+  ];
+
+  # Per-host WireGuard private keys. Each one only needs to decrypt on
+  # its own host plus josie (so josie can re-encrypt if needed).
+  "wireguard-kingfisher.age".publicKeys = [
+    josie
+    kingfisher
+  ];
+  "wireguard-albatross.age".publicKeys = [
+    josie
+    albatross
+  ];
 }

secrets/wireguard-albatross.age

@@ -0,0 +1,8 @@
+age-encryption.org/v1
+-> X25519 vuYGyBlUd9JWV2NmLJh1/y15DPqQCF0jx41c/6Rq/Rk
+ys9jOAiTgWsZ6JLqxJgaPr8s3uTfbws1P04W6VAWfH0
+-> ssh-ed25519 gk326w PeBwSV+7Ip6wTChUOSBQvq8/4vTaK47s9RnbT4FtEUc
+Rz0l+0w68Pn/xkV3YjktyBjORrhwWwolxLlTgSZjuwI
+--- 2PZ/h/XsrH2acXB4THqEZW+0qYNQcY90k75l5L1RwTY
+xðŽC¸Ñ„ì’
+Êo§,’Âñ°|Ù `_%3çÕOôp>þÕ’=Òø#ÞWZ²ù_déÓÛ»Æ#fvkŏcÒoÄ‰/º¡­
a°

secrets/wireguard-kingfisher.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> X25519 D1Ve7RKEJVkOsoblZrgyTd6k9c1fpQQrUnMkfmlYSyM
+/8/jYX7P0Iw/Js4/TqivBXz1q4g3XhR5uHpTrFDjRy0
+-> ssh-ed25519 8eNJZw 3Bo+7+XQgVpwZSY1KvT9P0GOgUHL6YmfakBHUdD1vQc
+GM3Od97IzBefnZ99Fc+iUWtvjcoE5baK55FEYRsRfsc
+--- XPebRbSZsB1LpCm27WstMLbWgwpzsgitiekJN/8gcPE
+yÏ]Úuòyl~ò55uª£­Þ}-S&7ã´2T^	Ÿüå– c|k[½ïe¦Õ'Ó>Åõ›,þ|>¢²c@0$ÏÔ¬Š`ãº