diff --git a/docs/configuration.md b/docs/configuration.md index 69859c1..95420a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -320,6 +320,21 @@ auth: # certs and on-node loopback are unaffected. Default false. This flag is the # enforcement + kill switch. See docs/auth.md. strict_mtls_identity: false + # Trust rotated peer certs — RECOVERY switch, not a steady-state setting. + # + # Peer trust binds a live host row to the certificate serial recorded in it. A + # node re-records its own serial at startup, so an ordinary certificate rotation + # converges by replication and needs nothing here. But if those recorded serials + # have ALREADY gone stale fleet-wide, every daemon refuses every peer + # ("replication RPC requires peer mTLS") and the cluster cannot repair itself: + # the correction has to replicate, and replication is what is being refused. + # + # Set true on EVERY node to break that deadlock — a mismatch is then logged and + # admitted for CA-issued HOST certificates instead of refused — wait for the + # fleet to replicate, then set it back to false. It never relaxes the removal + # tombstone (a decommissioned host stays out) and never lets a distributable + # client certificate act as a peer. Default false. + trust_rotated_peer_certs: false # Forwarded identity: when true (and the forwarded_identity_v1 capability is # active cluster-wide), the owning node re-authenticates a forwarded user's # bearer and runs RBAC + audit as the real user instead of the peer=admin diff --git a/internal/corrosion/hosts.go b/internal/corrosion/hosts.go index a1815a6..b4327fc 100644 --- a/internal/corrosion/hosts.go +++ b/internal/corrosion/hosts.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "strings" "time" ) @@ -149,6 +150,19 @@ func AdmitHost(ctx context.Context, c *Client, h HostRecord) error { // are idempotent. A re-added machine may clear its local tombstone only when the // certificate actually installed on disk has a different serial; peers still // require the operator's AdmitHost mutation and the matching certificate. +// +// A LIVE row whose serial disagrees with the certificate on disk is RE-RECORDED, +// because this is the one caller entitled to do that: the daemon passes its own +// name and the serial it just read from its own PKI directory, so the write only +// ever touches the node's own row, and anyone able to change what that node +// presents already holds its private key. +// +// It used to error instead, and nothing else wrote the column — AdmitHost refuses +// a live row, no CLI sets it. So a reissued host certificate left the row stale +// forever, and since peer trust binds a live row to its recorded serial, every +// daemon refused every peer and replication stopped fleet-wide. There was no way +// back in-product: the correction has to reach the PEER, and the stale serial is +// what blocks the peer channel. Rotation now converges by ordinary replication. func RegisterHost(ctx context.Context, c *Client, h HostRecord) error { rows, err := c.Query(ctx, `SELECT cert_serial, deleted_at FROM hosts WHERE name = ?`, h.Name) if err != nil { @@ -158,11 +172,25 @@ func RegisterHost(ctx context.Context, c *Client, h HostRecord) error { return InsertHost(ctx, c, h) } if rows[0].String("deleted_at") == "" { - if rows[0].String("cert_serial") == "" || - strings.EqualFold(rows[0].String("cert_serial"), h.CertSerial) { + recorded := rows[0].String("cert_serial") + if recorded == "" || strings.EqualFold(recorded, h.CertSerial) { return nil } - return fmt.Errorf("active host %q has a different certificate serial", h.Name) + // The daemon substitutes "unknown" when it cannot read its own + // certificate. Recording that would blind the pin for this host on every + // peer — a local file-permission problem becoming a cluster-wide trust + // downgrade — so the recorded serial stands. + if h.CertSerial == "" || h.CertSerial == "unknown" { + slog.Warn("could not read this host's own certificate serial; leaving the recorded one in place", + "host", h.Name, "recorded_serial", recorded) + return nil + } + slog.Warn("this host's certificate has been reissued since it was admitted; re-recording its serial "+ + "so peers accept it (rotation converges by replication)", + "host", h.Name, "recorded_serial", recorded, "installed_serial", h.CertSerial) + return c.Execute(ctx, + `UPDATE hosts SET cert_serial = ?, updated_at = ? WHERE name = ? AND deleted_at IS NULL`, + h.CertSerial, c.NowTS(), h.Name) } return AdmitHost(ctx, c, h) } diff --git a/internal/corrosion/hosts_test.go b/internal/corrosion/hosts_test.go index fc21443..142b367 100644 --- a/internal/corrosion/hosts_test.go +++ b/internal/corrosion/hosts_test.go @@ -132,6 +132,68 @@ func TestRegisterHost_ClearsOnlyItsOldCertificateTombstone(t *testing.T) { } } +// TestRegisterHost_ReRecordsItsOwnRotatedCertificate is the regression for a +// cluster-wide control-plane partition. +// +// Peer trust binds a live host row to the serial recorded in it. Nothing wrote +// that serial when a host certificate was reissued — RegisterHost ERRORED on the +// mismatch and carried on, AdmitHost refuses a live row outright, and no CLI +// writes it. So on a cluster whose certificates had been rotated, every daemon +// refused every peer ("replication RPC requires peer mTLS") and there was no +// in-product way back: the correction has to reach the PEER, and the peer channel +// is exactly what the stale serial blocks. +// +// A node is authoritative for its OWN certificate — it reads it off its own disk, +// and anyone able to change what it presents already holds that node's private +// key. So startup re-records its own row. Rotation then converges by ordinary +// replication instead of by hand. +func TestRegisterHost_ReRecordsItsOwnRotatedCertificate(t *testing.T) { + c := testClient(t) + ctx := context.Background() + original := HostRecord{ + Name: "node5", Address: "10.0.0.5", SSHUser: "root", SSHPort: 22, + GRPCPort: 7443, State: "active", CertSerial: "aaaa", + } + if err := InsertHost(ctx, c, original); err != nil { + t.Fatal(err) + } + rotated := original + rotated.CertSerial = "bbbb" + if err := RegisterHost(ctx, c, rotated); err != nil { + t.Fatalf("a node restarting with a reissued certificate: %v — this is the lockout: "+ + "the row keeps the old serial and every peer refuses it", err) + } + got, _ := GetHost(ctx, c, "node5") + if got == nil || got.CertSerial != "bbbb" { + t.Fatalf("row = %#v, want the on-disk serial re-recorded", got) + } +} + +// TestRegisterHost_NeverRecordsAnUnreadableCertificate: the daemon substitutes +// "unknown" when it cannot read its own certificate. Writing that over a good +// serial would blind the pin for this host on every peer, turning a local file +// permission problem into a cluster-wide trust downgrade. +func TestRegisterHost_NeverRecordsAnUnreadableCertificate(t *testing.T) { + c := testClient(t) + ctx := context.Background() + h := HostRecord{ + Name: "node6", Address: "10.0.0.6", SSHUser: "root", SSHPort: 22, + GRPCPort: 7443, State: "active", CertSerial: "aaaa", + } + if err := InsertHost(ctx, c, h); err != nil { + t.Fatal(err) + } + blind := h + blind.CertSerial = "unknown" + if err := RegisterHost(ctx, c, blind); err != nil { + t.Fatalf("registration with an unreadable certificate: %v", err) + } + got, _ := GetHost(ctx, c, "node6") + if got == nil || got.CertSerial != "aaaa" { + t.Fatalf("row = %#v, want the recorded serial left intact", got) + } +} + func TestListHosts_WithLabels(t *testing.T) { c := testClient(t) ctx := context.Background() diff --git a/internal/corrosion/stmtledger_generated.go b/internal/corrosion/stmtledger_generated.go index ec65175..4543779 100644 --- a/internal/corrosion/stmtledger_generated.go +++ b/internal/corrosion/stmtledger_generated.go @@ -35,6 +35,7 @@ var stmtLedger = map[string]LedgerEntry{ "stmtshape/v1:249379e73590ab6841a6d89ae957983f736393dfb7f4cc39532037879dc4533f": {Fingerprint: "stmtshape/v1:249379e73590ab6841a6d89ae957983f736393dfb7f4cc39532037879dc4533f", Kind: "update", Table: "user_2fa", Disposition: DispBulkUpdate, Category: CatPerRowLWW}, "stmtshape/v1:24fa23b631b1488801c985cfa0801d14e39a0f35b4d05314c5b2a5763a0b1dbe": {Fingerprint: "stmtshape/v1:24fa23b631b1488801c985cfa0801d14e39a0f35b4d05314c5b2a5763a0b1dbe", Kind: "update", Table: "image_hosts", Disposition: DispFullPKUpdate}, "stmtshape/v1:253921a1ed48c83c43726ce19d95c6fefe362b31bf34475bb25518c3bf8127b0": {Fingerprint: "stmtshape/v1:253921a1ed48c83c43726ce19d95c6fefe362b31bf34475bb25518c3bf8127b0", Kind: "update", Table: "recovery_code_sets", Disposition: DispFullPKUpdate}, + "stmtshape/v1:2737d6f42d42eebb457f938f036d7a72f95b47cbf45e8b52c32194e69115b6ff": {Fingerprint: "stmtshape/v1:2737d6f42d42eebb457f938f036d7a72f95b47cbf45e8b52c32194e69115b6ff", Kind: "update", Table: "hosts", Disposition: DispFullPKUpdate}, "stmtshape/v1:275a18bc506db94bc2620c18f87d52af404536f7f5348f9b8ff46b929aa4a329": {Fingerprint: "stmtshape/v1:275a18bc506db94bc2620c18f87d52af404536f7f5348f9b8ff46b929aa4a329", Kind: "update", Table: "project_authority_epochs", Disposition: DispCustomMerge}, "stmtshape/v1:280b3e84bacc37cb5ab999a68d0602b3f2a7e22289f2e6878684312c94e8f999": {Fingerprint: "stmtshape/v1:280b3e84bacc37cb5ab999a68d0602b3f2a7e22289f2e6878684312c94e8f999", Kind: "update", Table: "recovery_codes", Disposition: DispBulkUpdate, Category: CatPerRowLWW}, "stmtshape/v1:28c6e49aed00c9fdba1c7db299695289a3d08cb6250a25d9ddfcda591a854cd9": {Fingerprint: "stmtshape/v1:28c6e49aed00c9fdba1c7db299695289a3d08cb6250a25d9ddfcda591a854cd9", Kind: "insert", Table: "users", Disposition: DispPlainInsert}, diff --git a/internal/daemon/config.go b/internal/daemon/config.go index af23cd6..a8b7ae8 100644 --- a/internal/daemon/config.go +++ b/internal/daemon/config.go @@ -273,6 +273,19 @@ type AuthConfig struct { // requires the StrictMTLSIdentityV1 capability active cluster-wide. This flag // is the enforcement + kill switch — set false to disable regardless of latch. StrictMTLSIdentity bool `yaml:"strict_mtls_identity,omitempty"` + // TrustRotatedPeerCerts is the RECOVERY switch for the peer certificate-serial + // pin. Peer trust binds a live host row to the serial recorded in it; when + // those recorded serials go stale (host certificates reissued), every daemon + // refuses every peer and the cluster stops replicating — and the correction + // cannot be replicated, because replication is what is being refused. + // + // Set true on EVERY node to break that deadlock: a mismatch is then logged and + // admitted for CA-issued host certificates rather than refused. Leave it on + // only until the fleet has replicated the serials each node re-records for + // itself at startup, then set it back to false. It never relaxes the removal + // tombstone, and never lets a distributable client certificate act as a peer. + // Default false; an ordinary rotation on a healthy cluster does not need it. + TrustRotatedPeerCerts bool `yaml:"trust_rotated_peer_certs,omitempty"` // ForwardedIdentity, when true, makes this node (as the owner of a resource) // re-authenticate the forwarded user's session bearer relayed by the entry // node and run RBAC + audit as the real user, instead of the peer=admin diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9caa249..6c1fbf8 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -753,6 +753,7 @@ func (d *Daemon) Run(ctx context.Context) error { reconciler.SetFirmwarePaths(firmwarePaths) svc.SetSessionTimeouts(parseDurationOr(d.cfg.Auth.SessionIdleTimeout, 0), parseDurationOr(d.cfg.Auth.SessionHardExpiry, 0)) svc.SetStrictMTLSIdentity(d.cfg.Auth.StrictMTLSIdentity) + svc.SetTrustRotatedPeerCerts(d.cfg.Auth.TrustRotatedPeerCerts) svc.SetForwardedIdentity(d.cfg.Auth.ForwardedIdentity) svc.SetRBACRealm(d.cfg.Auth.RBACRealm) // Split-brain-family enforcement kill-switches — so the HA monitor drives the diff --git a/internal/grpcapi/auth.go b/internal/grpcapi/auth.go index 6123ae8..e7cddd7 100644 --- a/internal/grpcapi/auth.go +++ b/internal/grpcapi/auth.go @@ -116,6 +116,18 @@ func (s *Server) SetStrictMTLSIdentity(on bool) { s.strictMTLSIdentity = on } // session bearer is authenticated as the real user. The flag is the kill switch. func (s *Server) SetForwardedIdentity(on bool) { s.forwardedIdentity = on } +// SetTrustRotatedPeerCerts sets this node's RECOVERY switch for the peer +// certificate-serial pin. When true, a live host row whose recorded serial +// disagrees with the presented certificate no longer refuses it, provided the +// certificate is a CA-issued HOST certificate; the mismatch is logged instead. +// +// This exists because a fleet whose recorded serials have gone stale locks itself +// out completely — every daemon refuses every peer, and the correction cannot be +// replicated because replication is what is refused. Leave it false in steady +// state: with RegisterHost re-recording each node's own serial at startup, an +// ordinary rotation converges without it. +func (s *Server) SetTrustRotatedPeerCerts(on bool) { s.trustRotatedPeerCerts = on } + // SetRBACRealm sets this node's opt-in for realm-aware role-binding grammar. // The flag is the reversible kill switch; realm enforcement in GrantRole is // gated by this flag AND the RBACRealmV1 latch (see rbacRealmConfigured / @@ -315,9 +327,28 @@ func (s *Server) isTrustedHostCN(ctx context.Context, cn string) bool { // Real peer calls carry the leaf certificate. Bind an active name to the // exact CA-issued identity recorded at admission, so re-admitting a name // cannot reopen a lagging peer to that name's old certificate. - if cert := peerLeafCert(ctx); cert != nil && cert.SerialNumber != nil && - rows[0].String("cert_serial") != "" { - return strings.EqualFold(rows[0].String("cert_serial"), cert.SerialNumber.Text(16)) + // + // The pin only holds while the recorded serial keeps up with reality. + // RegisterHost re-records each node's own serial at startup, so an ordinary + // rotation converges by replication; trustRotatedPeerCerts is the recovery + // switch for a fleet that ALREADY stopped replicating, where the correction + // cannot travel because the stale serial is what refuses it. In that mode + // a mismatch falls back to the host-vs-client discriminator and is logged + // — it never relaxes the tombstone above, so a removed host stays removed. + recorded := rows[0].String("cert_serial") + if cert := peerLeafCert(ctx); cert != nil && cert.SerialNumber != nil && recorded != "" { + presented := cert.SerialNumber.Text(16) + if strings.EqualFold(recorded, presented) { + return true + } + if !s.trustRotatedPeerCerts { + return false + } + slog.Warn("peer presented a certificate other than the one recorded at admission; "+ + "admitting it because trust_rotated_peer_certs recovery mode is on — turn it back off "+ + "once the fleet has replicated its re-recorded serials", + "cn", cn, "recorded_serial", recorded, "presented_serial", presented) + return callerCertHasServerAuth(ctx) } return true } diff --git a/internal/grpcapi/peer_bootstrap_test.go b/internal/grpcapi/peer_bootstrap_test.go index 87f97f1..486e130 100644 --- a/internal/grpcapi/peer_bootstrap_test.go +++ b/internal/grpcapi/peer_bootstrap_test.go @@ -188,3 +188,55 @@ func TestPeerTrust_AnUnreadableHostRowRefusesThePeer(t *testing.T) { "falling through here re-admits a decommissioned node whenever the read fails") } } + +// TestPeerTrust_RecoveryFlagUnblocksARotatedFleet. +// +// Self-recording (corrosion.RegisterHost) makes a rotation converge on a HEALTHY +// cluster, but it cannot rescue one that has already stopped replicating: the +// corrected row has to reach the PEER, and the stale serial is precisely what +// blocks the peer channel — in both directions, since a pull is refused by the +// same check. That deadlock previously had no in-product exit; it was resolved by +// hand-editing every node's database. +// +// So there is a deliberate, default-OFF recovery switch. Turned on fleet-wide it +// downgrades the pin to trust-and-log for CA-issued HOST certificates, long +// enough for the self-recorded serials to replicate; then it is turned back off +// and the pin is enforcing again against correct data. +func TestPeerTrust_RecoveryFlagUnblocksARotatedFleet(t *testing.T) { + ctx := context.Background() + s := trustFixture(t) + if err := corrosion.InsertHost(ctx, s.db, corrosion.HostRecord{ + Name: "node-rot", Address: "10.0.0.12", State: "active", CertSerial: "bb", + }); err != nil { + t.Fatal(err) + } + rotated := certSerialCtx("node-rot", big.NewInt(0xaa), + x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth) + + // DEFAULT: the pin is enforcing, so a serial it does not recognise is refused. + if s.isTrustedHostCN(rotated, "node-rot") { + t.Fatal("the serial pin is not enforcing by default") + } + + s.SetTrustRotatedPeerCerts(true) + if !s.isTrustedHostCN(rotated, "node-rot") { + t.Fatal("recovery mode did not admit a CA-issued HOST certificate whose serial had rotated —" + + " a fleet locked out by stale serials has no way back without editing databases by hand") + } + + // Recovery mode relaxes the SERIAL check only. The discriminator that keeps a + // distributable operator certificate out of the cluster is untouched. + clientCert := certSerialCtx("node-rot", big.NewInt(0xcc), x509.ExtKeyUsageClientAuth) + if s.isTrustedHostCN(clientCert, "node-rot") { + t.Fatal("recovery mode accepted a distributable CLIENT certificate as a peer") + } + + // And a REMOVED host stays removed: the tombstone outranks recovery mode, so + // this cannot be used to resurrect a decommissioned node. + if err := corrosion.DeleteHost(ctx, s.db, "node-rot"); err != nil { + t.Fatal(err) + } + if s.isTrustedHostCN(rotated, "node-rot") { + t.Fatal("recovery mode re-admitted a decommissioned host") + } +} diff --git a/internal/grpcapi/server.go b/internal/grpcapi/server.go index 36c1f4f..0b920ed 100644 --- a/internal/grpcapi/server.go +++ b/internal/grpcapi/server.go @@ -33,8 +33,8 @@ import ( type Server struct { pb.UnimplementedLiteVirtServer - hostName string - dataDir string + hostName string + dataDir string // containersRoot is where per-container state (and the owner-epoch marker) // lives — /containers in production, injected by the daemon so the // runtime-inventory collector can read markers. Empty disables marker reads @@ -82,6 +82,15 @@ type Server struct { // being active cluster-wide; the flag is also the kill switch. Default false. strictMTLSIdentity bool + // trustRotatedPeerCerts, when true, downgrades the peer certificate-serial pin + // to trust-and-log for CA-issued HOST certificates. It is the RECOVERY switch + // for a fleet already locked out by stale recorded serials: self-recording + // converges a rotation on a healthy cluster, but cannot rescue one that has + // stopped replicating, because the corrected row has to travel over the very + // channel the stale serial blocks. Default false — turn it on fleet-wide, let + // the self-recorded serials replicate, then turn it back off. + trustRotatedPeerCerts bool + // forwardedIdentity, when true, is this node's enforcement switch for owner- // side promotion of a forwarded user identity (x-litevirt-fwd-bearer). Gated // by this flag AND the ForwardedIdentityV1 capability active cluster-wide. @@ -274,7 +283,6 @@ type Server struct { vmLocksMu sync.Mutex vmLocks map[string]*sync.Mutex - // activeBackups tracks VMs this daemon is *currently* backing up. It's // in-memory, so it's empty after a restart — which is exactly what lets // the reconciler tell a genuinely-in-flight backup apart from a