fix(pki): let a rotated host certificate re-record its own serial

docs/configuration.md

@@ -320,6 +320,21 @@ auth:
   # certs and on-node loopback are unaffected. Default false. This flag is the
   # enforcement + kill switch. See docs/auth.md.
   strict_mtls_identity: false
+  # Trust rotated peer certs — RECOVERY switch, not a steady-state setting.
+  #
+  # Peer trust binds a live host row to the certificate serial recorded in it. A
+  # node re-records its own serial at startup, so an ordinary certificate rotation
+  # converges by replication and needs nothing here. But if those recorded serials
+  # have ALREADY gone stale fleet-wide, every daemon refuses every peer
+  # ("replication RPC requires peer mTLS") and the cluster cannot repair itself:
+  # the correction has to replicate, and replication is what is being refused.
+  #
+  # Set true on EVERY node to break that deadlock — a mismatch is then logged and
+  # admitted for CA-issued HOST certificates instead of refused — wait for the
+  # fleet to replicate, then set it back to false. It never relaxes the removal
+  # tombstone (a decommissioned host stays out) and never lets a distributable
+  # client certificate act as a peer. Default false.
+  trust_rotated_peer_certs: false
   # Forwarded identity: when true (and the forwarded_identity_v1 capability is
   # active cluster-wide), the owning node re-authenticates a forwarded user's
   # bearer and runs RBAC + audit as the real user instead of the peer=admin

internal/corrosion/hosts.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"strings"
 	"time"
 )
@@ -149,6 +150,19 @@ func AdmitHost(ctx context.Context, c *Client, h HostRecord) error {
 // are idempotent. A re-added machine may clear its local tombstone only when the
 // certificate actually installed on disk has a different serial; peers still
 // require the operator's AdmitHost mutation and the matching certificate.
+//
+// A LIVE row whose serial disagrees with the certificate on disk is RE-RECORDED,
+// because this is the one caller entitled to do that: the daemon passes its own
+// name and the serial it just read from its own PKI directory, so the write only
+// ever touches the node's own row, and anyone able to change what that node
+// presents already holds its private key.
+//
+// It used to error instead, and nothing else wrote the column — AdmitHost refuses
+// a live row, no CLI sets it. So a reissued host certificate left the row stale
+// forever, and since peer trust binds a live row to its recorded serial, every
+// daemon refused every peer and replication stopped fleet-wide. There was no way
+// back in-product: the correction has to reach the PEER, and the stale serial is
+// what blocks the peer channel. Rotation now converges by ordinary replication.
 func RegisterHost(ctx context.Context, c *Client, h HostRecord) error {
 	rows, err := c.Query(ctx, `SELECT cert_serial, deleted_at FROM hosts WHERE name = ?`, h.Name)
 	if err != nil {
@@ -158,11 +172,25 @@ func RegisterHost(ctx context.Context, c *Client, h HostRecord) error {
 		return InsertHost(ctx, c, h)
 	}
 	if rows[0].String("deleted_at") == "" {
-		if rows[0].String("cert_serial") == "" ||
-			strings.EqualFold(rows[0].String("cert_serial"), h.CertSerial) {
+		recorded := rows[0].String("cert_serial")
+		if recorded == "" || strings.EqualFold(recorded, h.CertSerial) {
 			return nil
 		}
-		return fmt.Errorf("active host %q has a different certificate serial", h.Name)
+		// The daemon substitutes "unknown" when it cannot read its own
+		// certificate. Recording that would blind the pin for this host on every
+		// peer — a local file-permission problem becoming a cluster-wide trust
+		// downgrade — so the recorded serial stands.
+		if h.CertSerial == "" || h.CertSerial == "unknown" {
+			slog.Warn("could not read this host's own certificate serial; leaving the recorded one in place",
+				"host", h.Name, "recorded_serial", recorded)
+			return nil
+		}
+		slog.Warn("this host's certificate has been reissued since it was admitted; re-recording its serial "+
+			"so peers accept it (rotation converges by replication)",
+			"host", h.Name, "recorded_serial", recorded, "installed_serial", h.CertSerial)
+		return c.Execute(ctx,
+			`UPDATE hosts SET cert_serial = ?, updated_at = ? WHERE name = ? AND deleted_at IS NULL`,
+			h.CertSerial, c.NowTS(), h.Name)
 	}
 	return AdmitHost(ctx, c, h)
 }

internal/corrosion/hosts_test.go

@@ -132,6 +132,68 @@ func TestRegisterHost_ClearsOnlyItsOldCertificateTombstone(t *testing.T) {
 	}
 }
 
+// TestRegisterHost_ReRecordsItsOwnRotatedCertificate is the regression for a
+// cluster-wide control-plane partition.
+//
+// Peer trust binds a live host row to the serial recorded in it. Nothing wrote
+// that serial when a host certificate was reissued — RegisterHost ERRORED on the
+// mismatch and carried on, AdmitHost refuses a live row outright, and no CLI
+// writes it. So on a cluster whose certificates had been rotated, every daemon
+// refused every peer ("replication RPC requires peer mTLS") and there was no
+// in-product way back: the correction has to reach the PEER, and the peer channel
+// is exactly what the stale serial blocks.
+//
+// A node is authoritative for its OWN certificate — it reads it off its own disk,
+// and anyone able to change what it presents already holds that node's private
+// key. So startup re-records its own row. Rotation then converges by ordinary
+// replication instead of by hand.
+func TestRegisterHost_ReRecordsItsOwnRotatedCertificate(t *testing.T) {
+	c := testClient(t)
+	ctx := context.Background()
+	original := HostRecord{
+		Name: "node5", Address: "10.0.0.5", SSHUser: "root", SSHPort: 22,
+		GRPCPort: 7443, State: "active", CertSerial: "aaaa",
+	}
+	if err := InsertHost(ctx, c, original); err != nil {
+		t.Fatal(err)
+	}
+	rotated := original
+	rotated.CertSerial = "bbbb"
+	if err := RegisterHost(ctx, c, rotated); err != nil {
+		t.Fatalf("a node restarting with a reissued certificate: %v — this is the lockout: "+
+			"the row keeps the old serial and every peer refuses it", err)
+	}
+	got, _ := GetHost(ctx, c, "node5")
+	if got == nil || got.CertSerial != "bbbb" {
+		t.Fatalf("row = %#v, want the on-disk serial re-recorded", got)
+	}
+}
+
+// TestRegisterHost_NeverRecordsAnUnreadableCertificate: the daemon substitutes
+// "unknown" when it cannot read its own certificate. Writing that over a good
+// serial would blind the pin for this host on every peer, turning a local file
+// permission problem into a cluster-wide trust downgrade.
+func TestRegisterHost_NeverRecordsAnUnreadableCertificate(t *testing.T) {
+	c := testClient(t)
+	ctx := context.Background()
+	h := HostRecord{
+		Name: "node6", Address: "10.0.0.6", SSHUser: "root", SSHPort: 22,
+		GRPCPort: 7443, State: "active", CertSerial: "aaaa",
+	}
+	if err := InsertHost(ctx, c, h); err != nil {
+		t.Fatal(err)
+	}
+	blind := h
+	blind.CertSerial = "unknown"
+	if err := RegisterHost(ctx, c, blind); err != nil {
+		t.Fatalf("registration with an unreadable certificate: %v", err)
+	}
+	got, _ := GetHost(ctx, c, "node6")
+	if got == nil || got.CertSerial != "aaaa" {
+		t.Fatalf("row = %#v, want the recorded serial left intact", got)
+	}
+}
+
 func TestListHosts_WithLabels(t *testing.T) {
 	c := testClient(t)
 	ctx := context.Background()

internal/daemon/config.go

@@ -273,6 +273,19 @@ type AuthConfig struct {
 	// requires the StrictMTLSIdentityV1 capability active cluster-wide. This flag
 	// is the enforcement + kill switch — set false to disable regardless of latch.
 	StrictMTLSIdentity bool `yaml:"strict_mtls_identity,omitempty"`
+	// TrustRotatedPeerCerts is the RECOVERY switch for the peer certificate-serial
+	// pin. Peer trust binds a live host row to the serial recorded in it; when
+	// those recorded serials go stale (host certificates reissued), every daemon
+	// refuses every peer and the cluster stops replicating — and the correction
+	// cannot be replicated, because replication is what is being refused.
+	//
+	// Set true on EVERY node to break that deadlock: a mismatch is then logged and
+	// admitted for CA-issued host certificates rather than refused. Leave it on
+	// only until the fleet has replicated the serials each node re-records for
+	// itself at startup, then set it back to false. It never relaxes the removal
+	// tombstone, and never lets a distributable client certificate act as a peer.
+	// Default false; an ordinary rotation on a healthy cluster does not need it.
+	TrustRotatedPeerCerts bool `yaml:"trust_rotated_peer_certs,omitempty"`
 	// ForwardedIdentity, when true, makes this node (as the owner of a resource)
 	// re-authenticate the forwarded user's session bearer relayed by the entry
 	// node and run RBAC + audit as the real user, instead of the peer=admin

internal/daemon/daemon.go

@@ -753,6 +753,7 @@ func (d *Daemon) Run(ctx context.Context) error {
 	reconciler.SetFirmwarePaths(firmwarePaths)
 	svc.SetSessionTimeouts(parseDurationOr(d.cfg.Auth.SessionIdleTimeout, 0), parseDurationOr(d.cfg.Auth.SessionHardExpiry, 0))
 	svc.SetStrictMTLSIdentity(d.cfg.Auth.StrictMTLSIdentity)
+	svc.SetTrustRotatedPeerCerts(d.cfg.Auth.TrustRotatedPeerCerts)
 	svc.SetForwardedIdentity(d.cfg.Auth.ForwardedIdentity)
 	svc.SetRBACRealm(d.cfg.Auth.RBACRealm)
 	// Split-brain-family enforcement kill-switches — so the HA monitor drives the

internal/grpcapi/auth.go

@@ -116,6 +116,18 @@ func (s *Server) SetStrictMTLSIdentity(on bool) { s.strictMTLSIdentity = on }
 // session bearer is authenticated as the real user. The flag is the kill switch.
 func (s *Server) SetForwardedIdentity(on bool) { s.forwardedIdentity = on }
 
+// SetTrustRotatedPeerCerts sets this node's RECOVERY switch for the peer
+// certificate-serial pin. When true, a live host row whose recorded serial
+// disagrees with the presented certificate no longer refuses it, provided the
+// certificate is a CA-issued HOST certificate; the mismatch is logged instead.
+//
+// This exists because a fleet whose recorded serials have gone stale locks itself
+// out completely — every daemon refuses every peer, and the correction cannot be
+// replicated because replication is what is refused. Leave it false in steady
+// state: with RegisterHost re-recording each node's own serial at startup, an
+// ordinary rotation converges without it.
+func (s *Server) SetTrustRotatedPeerCerts(on bool) { s.trustRotatedPeerCerts = on }
+
 // SetRBACRealm sets this node's opt-in for realm-aware role-binding grammar.
 // The flag is the reversible kill switch; realm enforcement in GrantRole is
 // gated by this flag AND the RBACRealmV1 latch (see rbacRealmConfigured /
@@ -315,9 +327,28 @@ func (s *Server) isTrustedHostCN(ctx context.Context, cn string) bool {
 		// Real peer calls carry the leaf certificate. Bind an active name to the
 		// exact CA-issued identity recorded at admission, so re-admitting a name
 		// cannot reopen a lagging peer to that name's old certificate.
-		if cert := peerLeafCert(ctx); cert != nil && cert.SerialNumber != nil &&
-			rows[0].String("cert_serial") != "" {
-			return strings.EqualFold(rows[0].String("cert_serial"), cert.SerialNumber.Text(16))
+		//
+		// The pin only holds while the recorded serial keeps up with reality.
+		// RegisterHost re-records each node's own serial at startup, so an ordinary
+		// rotation converges by replication; trustRotatedPeerCerts is the recovery
+		// switch for a fleet that ALREADY stopped replicating, where the correction
+		// cannot travel because the stale serial is what refuses it. In that mode
+		// a mismatch falls back to the host-vs-client discriminator and is logged
+		// — it never relaxes the tombstone above, so a removed host stays removed.
+		recorded := rows[0].String("cert_serial")
+		if cert := peerLeafCert(ctx); cert != nil && cert.SerialNumber != nil && recorded != "" {
+			presented := cert.SerialNumber.Text(16)
+			if strings.EqualFold(recorded, presented) {
+				return true
+			}
+			if !s.trustRotatedPeerCerts {
+				return false
+			}
+			slog.Warn("peer presented a certificate other than the one recorded at admission; "+
+				"admitting it because trust_rotated_peer_certs recovery mode is on — turn it back off "+
+				"once the fleet has replicated its re-recorded serials",
+				"cn", cn, "recorded_serial", recorded, "presented_serial", presented)
+			return callerCertHasServerAuth(ctx)
 		}
 		return true
 	}

internal/grpcapi/peer_bootstrap_test.go

@@ -188,3 +188,55 @@ func TestPeerTrust_AnUnreadableHostRowRefusesThePeer(t *testing.T) {
 			"falling through here re-admits a decommissioned node whenever the read fails")
 	}
 }
+
+// TestPeerTrust_RecoveryFlagUnblocksARotatedFleet.
+//
+// Self-recording (corrosion.RegisterHost) makes a rotation converge on a HEALTHY
+// cluster, but it cannot rescue one that has already stopped replicating: the
+// corrected row has to reach the PEER, and the stale serial is precisely what
+// blocks the peer channel — in both directions, since a pull is refused by the
+// same check. That deadlock previously had no in-product exit; it was resolved by
+// hand-editing every node's database.
+//
+// So there is a deliberate, default-OFF recovery switch. Turned on fleet-wide it
+// downgrades the pin to trust-and-log for CA-issued HOST certificates, long
+// enough for the self-recorded serials to replicate; then it is turned back off
+// and the pin is enforcing again against correct data.
+func TestPeerTrust_RecoveryFlagUnblocksARotatedFleet(t *testing.T) {
+	ctx := context.Background()
+	s := trustFixture(t)
+	if err := corrosion.InsertHost(ctx, s.db, corrosion.HostRecord{
+		Name: "node-rot", Address: "10.0.0.12", State: "active", CertSerial: "bb",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	rotated := certSerialCtx("node-rot", big.NewInt(0xaa),
+		x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth)
+
+	// DEFAULT: the pin is enforcing, so a serial it does not recognise is refused.
+	if s.isTrustedHostCN(rotated, "node-rot") {
+		t.Fatal("the serial pin is not enforcing by default")
+	}
+
+	s.SetTrustRotatedPeerCerts(true)
+	if !s.isTrustedHostCN(rotated, "node-rot") {
+		t.Fatal("recovery mode did not admit a CA-issued HOST certificate whose serial had rotated —" +
+			" a fleet locked out by stale serials has no way back without editing databases by hand")
+	}
+
+	// Recovery mode relaxes the SERIAL check only. The discriminator that keeps a
+	// distributable operator certificate out of the cluster is untouched.
+	clientCert := certSerialCtx("node-rot", big.NewInt(0xcc), x509.ExtKeyUsageClientAuth)
+	if s.isTrustedHostCN(clientCert, "node-rot") {
+		t.Fatal("recovery mode accepted a distributable CLIENT certificate as a peer")
+	}
+
+	// And a REMOVED host stays removed: the tombstone outranks recovery mode, so
+	// this cannot be used to resurrect a decommissioned node.
+	if err := corrosion.DeleteHost(ctx, s.db, "node-rot"); err != nil {
+		t.Fatal(err)
+	}
+	if s.isTrustedHostCN(rotated, "node-rot") {
+		t.Fatal("recovery mode re-admitted a decommissioned host")
+	}
+}

internal/grpcapi/server.go

@@ -33,8 +33,8 @@ import (
 type Server struct {
 	pb.UnimplementedLiteVirtServer
 
-	hostName   string
-	dataDir    string
+	hostName string
+	dataDir  string
 	// containersRoot is where per-container state (and the owner-epoch marker)
 	// lives — <dataDir>/containers in production, injected by the daemon so the
 	// runtime-inventory collector can read markers. Empty disables marker reads
@@ -82,6 +82,15 @@ type Server struct {
 	// being active cluster-wide; the flag is also the kill switch. Default false.
 	strictMTLSIdentity bool
 
+	// trustRotatedPeerCerts, when true, downgrades the peer certificate-serial pin
+	// to trust-and-log for CA-issued HOST certificates. It is the RECOVERY switch
+	// for a fleet already locked out by stale recorded serials: self-recording
+	// converges a rotation on a healthy cluster, but cannot rescue one that has
+	// stopped replicating, because the corrected row has to travel over the very
+	// channel the stale serial blocks. Default false — turn it on fleet-wide, let
+	// the self-recorded serials replicate, then turn it back off.
+	trustRotatedPeerCerts bool
+
 	// forwardedIdentity, when true, is this node's enforcement switch for owner-
 	// side promotion of a forwarded user identity (x-litevirt-fwd-bearer). Gated
 	// by this flag AND the ForwardedIdentityV1 capability active cluster-wide.
@@ -274,7 +283,6 @@ type Server struct {
 	vmLocksMu sync.Mutex
 	vmLocks   map[string]*sync.Mutex
 
-
 	// activeBackups tracks VMs this daemon is *currently* backing up. It's
 	// in-memory, so it's empty after a restart — which is exactly what lets
 	// the reconciler tell a genuinely-in-flight backup apart from a