diff --git a/.github/workflows/scripts/integration-test-ci b/.github/workflows/scripts/integration-test-ci index 9078084c58..fd4b3544e4 100755 --- a/.github/workflows/scripts/integration-test-ci +++ b/.github/workflows/scripts/integration-test-ci @@ -47,7 +47,7 @@ try --waitmsg "Waiting for CRDs" crds_exist echo "Shutting down core rancher" kubectl scale deploy rancher -n cattle-system --replicas=0 kubectl wait pods -l app=rancher --for=delete -n cattle-system -# Make sure the webhook recreates configurations on startup +# Delete the existing webhook configurations so the helm upgrade recreates them from the chart templates. kubectl delete validatingwebhookconfiguration rancher.cattle.io kubectl delete mutatingwebhookconfiguration rancher.cattle.io @@ -83,6 +83,25 @@ upgrade_rancher_webhook() { kubectl rollout status deployment/rancher-webhook -n cattle-system try --max 90 --waitmsg "Waiting for webhooks to be registered" --failmsg "Webhooks not registered" check_webhooks + + # The chart now owns the WebhookConfigurations with caBundle: "" (filled by needacert at runtime). + # needacert runs inside the rancher pod, which is scaled to 0 in this test. Patch caBundle + # directly from the TLS secret so the kube-apiserver can verify the webhook's serving cert. + populate_ca_bundle() { + local ca_bundle + ca_bundle=$(kubectl get secret cattle-webhook-tls -n cattle-system \ + -o jsonpath='{.data.tls\.crt}' 2>/dev/null) + [[ -z "$ca_bundle" ]] && return 1 + for resource in validatingwebhookconfiguration mutatingwebhookconfiguration; do + local patch + patch=$(kubectl get "$resource" rancher.cattle.io -o json \ + | jq --arg ca "$ca_bundle" \ + '[range(.webhooks | length) | {"op":"replace","path":"/webhooks/\(.)/clientConfig/caBundle","value":$ca}]') + kubectl patch "$resource" rancher.cattle.io --type=json -p="$patch" + done + } + try --max 10 --delay 3 --waitmsg "Populating caBundle from TLS secret" \ + --failmsg "Failed to populate caBundle" populate_ca_bundle } try --max 3 --delay 2 --waitmsg "Upgrading Webhook" --failmsg "Failed to upgrade webhook" upgrade_rancher_webhook diff --git a/charts/rancher-webhook/templates/deployment.yaml b/charts/rancher-webhook/templates/deployment.yaml index 5678863487..2813a881a1 100644 --- a/charts/rancher-webhook/templates/deployment.yaml +++ b/charts/rancher-webhook/templates/deployment.yaml @@ -13,8 +13,12 @@ spec: labels: app: rancher-webhook spec: - {{- if $auth.clientCA }} volumes: + - name: tls + secret: + secretName: cattle-webhook-tls + optional: false + {{- if $auth.clientCA }} - name: client-ca secret: secretName: client-ca @@ -70,8 +74,11 @@ spec: port: "https" scheme: "HTTPS" periodSeconds: 5 - {{- if $auth.clientCA }} volumeMounts: + - name: tls + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + {{- if $auth.clientCA }} - name: client-ca mountPath: /tmp/k8s-webhook-server/client-ca readOnly: true @@ -79,11 +86,11 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - {{- if .Values.capNetBindService }} + {{- if .Values.capNetBindService }} capabilities: add: - NET_BIND_SERVICE - {{- end }} + {{- end }} serviceAccountName: rancher-webhook {{- if .Values.priorityClassName }} priorityClassName: "{{.Values.priorityClassName}}" diff --git a/charts/rancher-webhook/templates/rbac.yaml b/charts/rancher-webhook/templates/rbac.yaml index f4364995c0..e57e502dd5 100644 --- a/charts/rancher-webhook/templates/rbac.yaml +++ b/charts/rancher-webhook/templates/rbac.yaml @@ -1,12 +1,104 @@ apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: ClusterRole metadata: name: rancher-webhook +rules: +# --- Read-only: informer caches for validators/mutators --- +- apiGroups: ["management.cattle.io"] + resources: + - authconfigs + - clusters + - clusterproxyconfigs + - clusterroletemplatebindings + - features + - globalroles + - globalrolebindings + - nodes + - podsecurityadmissionconfigurationtemplates + - projects + - projectroletemplatebindings + - roletemplates + - settings + - tokens + - userattributes + - users + verbs: [get, list, watch] +- apiGroups: ["provisioning.cattle.io"] + resources: [clusters] + verbs: [get, list, watch, patch] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [roles, rolebindings, clusterroles, clusterrolebindings] + verbs: [get, list, watch] +- apiGroups: [""] + resources: [namespaces, secrets] + verbs: [get, list, watch] +- apiGroups: ["rke.cattle.io"] + resources: [etcdsnapshots] + verbs: [get, list, watch] +- apiGroups: ["apiregistration.k8s.io"] + resources: [apiservices] + verbs: [get, list, watch] +- apiGroups: ["apiextensions.k8s.io"] + resources: [customresourcedefinitions] + verbs: [get, list, watch] +- apiGroups: ["cluster.x-k8s.io"] + resources: [machinedeployments, clusters] + verbs: [get, list, watch] +- apiGroups: ["rke-machine-config.cattle.io"] + resources: ["*"] + verbs: [get, list, watch] +- apiGroups: ["rke-machine.cattle.io"] + resources: + - amazonec2machines + - azuremachines + - digitaloceanmachines + - harvestermachines + - linodemachines + - vmwarevspheremachines + verbs: [get, list, watch] +- apiGroups: ["catalog.cattle.io"] + resources: [clusterrepos] + verbs: [get, list, watch] +- apiGroups: ["auditlog.cattle.io"] + resources: [auditpolicies] + verbs: [get, list, watch] +- apiGroups: ["cluster.cattle.io"] + resources: [clusterauthtokens] + verbs: [get, list, watch] +# --- Write operations --- +- apiGroups: ["authorization.k8s.io"] + resources: [subjectaccessreviews] + verbs: [create] +- apiGroups: [""] + resources: [namespaces] + verbs: [create] +- apiGroups: [""] + resources: [secrets] + verbs: [create, update, delete] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [roles] + verbs: [update] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [rolebindings, clusterrolebindings] + verbs: [create] +# The fleetworkspace mutator creates a per-workspace ClusterRole and binds it (plus the +# fleet-admin ClusterRole) to the workspace creator. Those grants include permissions the +# webhook ServiceAccount does not itself hold (fleet.cattle.io/*), so Kubernetes RBAC +# escalation prevention requires escalate (to create the ClusterRole) and bind (to reference +# it from a [Cluster]RoleBinding). +- apiGroups: ["rbac.authorization.k8s.io"] + resources: [clusterroles] + verbs: [create, escalate, bind] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rancher-webhook-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: cluster-admin + name: rancher-webhook subjects: - kind: ServiceAccount name: rancher-webhook - namespace: {{.Release.Namespace}} \ No newline at end of file + namespace: {{.Release.Namespace}} diff --git a/charts/rancher-webhook/templates/service.yaml b/charts/rancher-webhook/templates/service.yaml index 220afebeae..e0848b8bce 100644 --- a/charts/rancher-webhook/templates/service.yaml +++ b/charts/rancher-webhook/templates/service.yaml @@ -3,6 +3,8 @@ apiVersion: v1 metadata: name: rancher-webhook namespace: cattle-system + annotations: + need-a-cert.cattle.io/secret-name: cattle-webhook-tls spec: ports: - port: 443 diff --git a/charts/rancher-webhook/templates/webhook.yaml b/charts/rancher-webhook/templates/webhook.yaml index 53a0687b6f..fe73a1d084 100644 --- a/charts/rancher-webhook/templates/webhook.yaml +++ b/charts/rancher-webhook/templates/webhook.yaml @@ -2,8 +2,1167 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: rancher.cattle.io +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/features.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.features.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + resources: + - features + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusters.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - clusters + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/rke-machine-config.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.rke-machine-config.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - rke-machine-config.cattle.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - '*' + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces.delete-namespace + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - fleet-local + - local + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - DELETE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - UPDATE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces.create-non-kubesystem + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - CREATE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/namespaces + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.namespaces.create-kubesystem-only + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - kube-system + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - CREATE + resources: + - namespaces + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterrepos.catalog.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterrepos.catalog.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - catalog.cattle.io + apiVersions: + - v1 + operations: + - UPDATE + - CREATE + resources: + - clusterrepos + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/auditpolicies.auditlog.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.auditpolicies.auditlog.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - auditlog.cattle.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - auditpolicies + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +{{- if .Values.mcm.enabled }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusters.provisioning.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.provisioning.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - provisioning.cattle.io + apiVersions: + - v1 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - clusters + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterproxyconfigs.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterproxyconfigs.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - clusterproxyconfigs + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/podsecurityadmissionconfigurationtemplates.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.podsecurityadmissionconfigurationtemplates.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - podsecurityadmissionconfigurationtemplates + scope: '*' + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/globalroles.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.globalroles.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - globalroles + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/globalrolebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.globalrolebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - globalrolebindings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/projectroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projectroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - projectroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - clusterroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/roletemplates.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.roletemplates.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + - DELETE + resources: + - roletemplates + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/secrets + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.secrets + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - DELETE + resources: + - secrets + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/nodedrivers.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.nodedrivers.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - DELETE + resources: + - nodedrivers + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/projects.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projects.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - projects + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/roles.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.roles.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/gr-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - roles + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/rolebindings.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.rolebindings.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/grb-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - rolebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/settings.management.cattle.io + port: 443 + failurePolicy: Ignore + matchPolicy: Equivalent + name: rancher.cattle.io.settings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - settings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/tokens.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.tokens.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - tokens + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/userattributes.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.userattributes.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - userattributes + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterroles.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterroles.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/gr-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - clusterroles + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/clusterrolebindings.rbac.authorization.k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterrolebindings.rbac.authorization.k8s.io + namespaceSelector: {} + objectSelector: + matchExpressions: + - key: authz.management.cattle.io/grb-owner + operator: Exists + rules: + - apiGroups: + - rbac.authorization.k8s.io + apiVersions: + - v1 + operations: + - UPDATE + resources: + - clusterrolebindings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/authconfigs.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.authconfigs.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - UPDATE + - CREATE + resources: + - authconfigs + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/users.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.users.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - users + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/machinedeployments-scale.cluster.x-k8s.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.machinedeployments-scale.cluster.x-k8s.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + resources: + - machinedeployments/scale + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/validation/proxyendpoints.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.proxyendpoints.management.cattle.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - proxyendpoints + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +{{- end }} --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: rancher.cattle.io +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/clusters.provisioning.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.provisioning.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - provisioning.cattle.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - clusters + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/clusters.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusters.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - clusters + scope: Cluster + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/fleetworkspaces.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.fleetworkspaces.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - fleetworkspaces + scope: Cluster + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/rke-machine-config.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.rke-machine-config.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - rke-machine-config.cattle.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - '*' + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 10 +{{- if .Values.mcm.enabled }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/secrets + port: 443 + failurePolicy: Fail + matchConditions: + - expression: request.operation == 'DELETE' || (object != null && object.type + == "provisioning.cattle.io/cloud-credential" && request.operation == 'CREATE') + || (object != null && object.metadata.namespace == "cattle-local-user-passwords") + name: filter-by-secret-type-cloud-credential + matchPolicy: Equivalent + name: rancher.cattle.io.secrets + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - '' + apiVersions: + - v1 + operations: + - CREATE + - DELETE + - UPDATE + resources: + - secrets + scope: Namespaced + sideEffects: NoneOnDryRun + timeoutSeconds: 15 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/projects.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projects.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + - UPDATE + resources: + - projects + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/globalrolebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.globalrolebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - globalrolebindings + scope: Cluster + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/projectroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.projectroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - projectroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: {{ .Values.caBundle | default "" }} + service: + name: rancher-webhook + namespace: cattle-system + path: /v1/webhook/mutation/clusterroletemplatebindings.management.cattle.io + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: rancher.cattle.io.clusterroletemplatebindings.management.cattle.io + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - management.cattle.io + apiVersions: + - v3 + operations: + - CREATE + resources: + - clusterroletemplatebindings + scope: Namespaced + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/charts/rancher-webhook/values.yaml b/charts/rancher-webhook/values.yaml index da96f2d1b8..34b5d40ed2 100644 --- a/charts/rancher-webhook/values.yaml +++ b/charts/rancher-webhook/values.yaml @@ -12,6 +12,11 @@ global: mcm: enabled: true +# caBundle is populated at runtime by needacert (wrangler/pkg/needacert) +# once the cattle-webhook-tls secret has been generated. Override only +# when running this chart standalone without needacert. +caBundle: "" + # tolerations for the webhook deployment. See https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ for more info tolerations: [] nodeSelector: {} diff --git a/pkg/server/server.go b/pkg/server/server.go index c41b3d09c9..427fe0e8de 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -12,50 +12,30 @@ import ( "path/filepath" "strconv" "strings" - "sync/atomic" + "sync" "time" - "github.com/google/uuid" - "github.com/rancher/dynamiclistener" - "github.com/rancher/dynamiclistener/server" "github.com/rancher/webhook/pkg/admission" "github.com/rancher/webhook/pkg/clients" "github.com/rancher/webhook/pkg/health" - admissionregistration "github.com/rancher/wrangler/v3/pkg/generated/controllers/admissionregistration.k8s.io/v1" "github.com/sirupsen/logrus" - v1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" ) const ( - serviceName = "rancher-webhook" - namespace = "cattle-system" - tlsName = "rancher-webhook.cattle-system.svc" - certName = "cattle-webhook-tls" - caName = "cattle-webhook-ca" validationPath = "/v1/webhook/validation" mutationPath = "/v1/webhook/mutation" - clientPort = int32(443) webhookHTTPPort = 0 // value of 0 indicates we do not want to use http. defaultWebhookHTTPSPort = 9443 webhookPortEnvKey = "CATTLE_PORT" - webhookURLEnvKey = "CATTLE_WEBHOOK_URL" + webhookCertDirEnvKey = "CATTLE_WEBHOOK_CERT_DIR" + defaultCertDir = "/tmp/k8s-webhook-server/serving-certs" allowedCNsEnv = "ALLOWED_CNS" ) var caFile = filepath.Join(os.TempDir(), "k8s-webhook-server", "client-ca", "ca.crt") -// leaderFlag indicates whether this process is the elected leader. -// Gate config mutation work on this to avoid concurrent writers. -var leaderFlag atomic.Bool - -// tlsOpt option function applied to all webhook servers. +// tlsOpt configures the TLS settings shared by the serving listener. var tlsOpt = func(config *tls.Config) { config.MinVersion = tls.VersionTLS12 config.CipherSuites = []uint16{ @@ -76,13 +56,6 @@ func ListenAndServe(ctx context.Context, cfg *rest.Config, mcmEnabled bool) erro return fmt.Errorf("failed to create a new client: %w", err) } - if err = setCertificateExpirationDays(); err != nil { - // If this error occurs, certificate creation will still work. However, our override will likely not have worked. - // This will not affect functionality of the webhook, but users may have to perform the workaround: - // https://github.com/rancher/docs/issues/3637 - logrus.Infof("[ListenAndServe] could not set certificate expiration days via environment variable: %v", err) - } - validators, err := Validation(clients) if err != nil { return err @@ -93,76 +66,13 @@ func ListenAndServe(ctx context.Context, cfg *rest.Config, mcmEnabled bool) erro return err } - k8sClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - return fmt.Errorf("failed to create kubernetes clientset: %w", err) - } - id := uuid.New().String() - - lock := &resourcelock.LeaseLock{ - LeaseMeta: metav1.ObjectMeta{ - Name: "rancher-webhook-leader", - Namespace: namespace, - }, - Client: k8sClient.CoordinationV1(), - LockConfig: resourcelock.ResourceLockConfig{ - Identity: id, - }, - } - - go leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ - Lock: lock, - LeaseDuration: 15 * time.Second, - RenewDeadline: 10 * time.Second, - RetryPeriod: 2 * time.Second, - ReleaseOnCancel: true, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: func(_ context.Context) { - leaderFlag.Store(true) - clients.Core.Secret().Enqueue(namespace, caName) - logrus.Infof("[%s] elected leader: will manage webhook configurations", id) - }, - OnStoppedLeading: func() { - leaderFlag.Store(false) - logrus.Infof("[%s] lost leadership: will stop managing webhook configurations", id) - }, - OnNewLeader: func(identity string) { - if identity == id { - logrus.Infof("[%s] I am the new leader", id) - } else { - logrus.Infof("[%s] observed new leader: %s", id, identity) - } - }, - }, - }) - - if err = listenAndServe(ctx, clients, validators, mutators); err != nil { - return err - } - - if err = clients.Start(ctx); err != nil { - return fmt.Errorf("failed to start client: %w", err) - } - - return nil -} - -// By default, dynamiclistener sets newly signed certificates to expire after 365 days. Since the -// self-signed certificate for webhook does not need to be rotated, we increase expiration time -// beyond relevance. In this case, that's 3650 days (10 years). -func setCertificateExpirationDays() error { - certExpirationDaysKey := "CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS" - if os.Getenv(certExpirationDaysKey) == "" { - return os.Setenv(certExpirationDaysKey, "3650") - } - return nil + return listenAndServe(ctx, clients, validators, mutators) } func listenAndServe(ctx context.Context, clients *clients.Clients, validators []admission.ValidatingAdmissionHandler, mutators []admission.MutatingAdmissionHandler) (rErr error) { router := http.NewServeMux() errChecker := health.NewErrorChecker("config-applied") health.RegisterHealthCheckers(router, errChecker) - errChecker.Store(errors.New("webhook configuration not yet applied")) logrus.Debug("Creating Webhook routes") for _, webhook := range validators { @@ -178,24 +88,6 @@ func listenAndServe(ctx context.Context, clients *clients.Clients, validators [] routerHandler := certAuth()(router) - handler := &secretHandler{ - validators: validators, - mutators: mutators, - errChecker: errChecker, - validatingController: clients.Admission.ValidatingWebhookConfiguration(), - mutatingController: clients.Admission.MutatingWebhookConfiguration(), - } - clients.Core.Secret().OnChange(ctx, "secrets", handler.sync) - - defer func() { - if rErr != nil { - return - } - rErr = clients.Start(ctx) - }() - - tlsConfig := &tls.Config{} - tlsOpt(tlsConfig) webhookHTTPSPort := defaultWebhookHTTPSPort if portStr := os.Getenv(webhookPortEnvKey); portStr != "" { var err error @@ -204,142 +96,131 @@ func listenAndServe(ctx context.Context, clients *clients.Clients, validators [] return fmt.Errorf("failed to decode webhook port value '%s': %w", portStr, err) } } - return server.ListenAndServe(ctx, webhookHTTPSPort, webhookHTTPPort, routerHandler, &server.ListenOpts{ - Secrets: clients.Core.Secret(), - CertNamespace: namespace, - CertName: certName, - CAName: caName, - TLSListenerConfig: dynamiclistener.Config{ - SANs: []string{ - tlsName, - }, - FilterCN: dynamiclistener.OnlyAllow(tlsName), - TLSConfig: tlsConfig, - }, - DisplayServerLogs: true, - IgnoreTLSHandshakeError: true, - }) -} - -type secretHandler struct { - validators []admission.ValidatingAdmissionHandler - mutators []admission.MutatingAdmissionHandler - errChecker *health.ErrorChecker - validatingController admissionregistration.ValidatingWebhookConfigurationClient - mutatingController admissionregistration.MutatingWebhookConfigurationClient -} -// sync updates the validating admission configuration whenever the TLS cert changes. -// Only the elected leader performs the updates, followers are a no-op. -func (s *secretHandler) sync(_ string, secret *corev1.Secret) (*corev1.Secret, error) { - // The leader is responsible for applying the webhook configuration. - // Follower pods are only responsible for serving traffic and can be marked as healthy once the certificates are generated. - if !leaderFlag.Load() { - s.errChecker.Store(nil) - return nil, nil + certDir := defaultCertDir + if dir := os.Getenv(webhookCertDirEnvKey); dir != "" { + certDir = dir } + certPath := filepath.Join(certDir, "tls.crt") + keyPath := filepath.Join(certDir, "tls.key") - if secret == nil || secret.Name != caName || secret.Namespace != namespace || len(secret.Data[corev1.TLSCertKey]) == 0 { - return nil, nil + reloader, err := newCertReloader(certPath, keyPath) + if err != nil { + return fmt.Errorf("failed to load serving cert from %s: %w", certDir, err) } - logrus.Info("Applying webhook config") - - validationClientConfig := v1.WebhookClientConfig{ - Service: &v1.ServiceReference{ - Namespace: namespace, - Name: serviceName, - Path: admission.Ptr(validationPath), - Port: admission.Ptr(clientPort), - }, - CABundle: secret.Data[corev1.TLSCertKey], + tlsConfig := &tls.Config{ + GetCertificate: reloader.getCertificate, } + tlsOpt(tlsConfig) - mutationClientConfig := v1.WebhookClientConfig{ - Service: &v1.ServiceReference{ - Namespace: namespace, - Name: serviceName, - Path: admission.Ptr(mutationPath), - Port: admission.Ptr(clientPort), - }, - CABundle: secret.Data[corev1.TLSCertKey], + server := &http.Server{ + Addr: fmt.Sprintf(":%d", webhookHTTPSPort), + Handler: routerHandler, + TLSConfig: tlsConfig, } - if devURL, ok := os.LookupEnv(webhookURLEnvKey); ok { - validationURL := devURL + validationPath - mutationURL := devURL + mutationPath - validationClientConfig = v1.WebhookClientConfig{ - URL: &validationURL, + + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + logrus.Warnf("webhook server shutdown returned error: %v", err) } - mutationClientConfig = v1.WebhookClientConfig{ - URL: &mutationURL, + }() + + logrus.Infof("listening on :%d serving certs from %s", webhookHTTPSPort, certDir) + errChecker.Store(nil) + + go func() { + if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { + logrus.Fatalf("webhook server stopped: %v", err) } + }() + + if err := clients.Start(ctx); err != nil { + return fmt.Errorf("failed to start clients: %w", err) } - validatingWebhooks := make([]v1.ValidatingWebhook, 0, len(s.validators)) - for _, webhook := range s.validators { - validatingWebhooks = append(validatingWebhooks, webhook.ValidatingWebhook(validationClientConfig)...) - } - mutatingWebhooks := make([]v1.MutatingWebhook, 0, len(s.mutators)) - for _, webhook := range s.mutators { - mutatingWebhooks = append(mutatingWebhooks, webhook.MutatingWebhook(mutationClientConfig)...) + + <-ctx.Done() + return nil +} + +// certReloader caches the parsed serving certificate and only re-parses it when +// the underlying files change, so that needacert can rotate the secret (kubelet +// refreshes the projected files on its own cycle) without a webhook pod restart +// or a full LoadX509KeyPair on every handshake. If a reload fails - e.g. it races +// a secret update and reads a mismatched cert/key pair - the last-known-good +// certificate keeps serving instead of failing the handshake. +type certReloader struct { + certPath, keyPath string + + mu sync.RWMutex + cert *tls.Certificate + certModTime time.Time + keyModTime time.Time +} + +func newCertReloader(certPath, keyPath string) (*certReloader, error) { + r := &certReloader{certPath: certPath, keyPath: keyPath} + if err := r.reload(); err != nil { + return nil, err } - validatingConfig := &v1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rancher.cattle.io", - }, - Webhooks: validatingWebhooks, + return r, nil +} + +// reload re-parses the cert/key files and, on success, updates the cache. +func (r *certReloader) reload() error { + certInfo, err := os.Stat(r.certPath) + if err != nil { + return err } - mutatingConfig := &v1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "rancher.cattle.io", - }, - Webhooks: mutatingWebhooks, + keyInfo, err := os.Stat(r.keyPath) + if err != nil { + return err } - err := s.ensureWebhookConfiguration(validatingConfig, mutatingConfig) + cert, err := tls.LoadX509KeyPair(r.certPath, r.keyPath) if err != nil { - logrus.Errorf("Failed to ensure configuration: %s", err.Error()) + return err } - s.errChecker.Store(err) - return secret, err - + r.mu.Lock() + defer r.mu.Unlock() + r.cert = &cert + r.certModTime = certInfo.ModTime() + r.keyModTime = keyInfo.ModTime() + return nil } -// ensureWebhookConfiguration creates or updates the current validating and mutating webhook configuration to have the desired webhook. -func (s *secretHandler) ensureWebhookConfiguration(validatingConfig *v1.ValidatingWebhookConfiguration, mutatingConfig *v1.MutatingWebhookConfiguration) error { - - currValidating, err := s.validatingController.Get(validatingConfig.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = s.validatingController.Create(validatingConfig) - if err != nil { - return fmt.Errorf("failed to create validating configuration: %w", err) - } - } else if err != nil { - return fmt.Errorf("failed to get validating configuration: %w", err) - } else { - currValidating.Webhooks = validatingConfig.Webhooks - _, err = s.validatingController.Update(currValidating) - if err != nil { - return fmt.Errorf("failed to update validating configuration: %w", err) - } +// getCertificate is used as tls.Config.GetCertificate. It only re-parses the +// cert/key when their mtimes have changed, and falls back to the cached +// certificate if a reload fails. +func (r *certReloader) getCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + certInfo, certErr := os.Stat(r.certPath) + keyInfo, keyErr := os.Stat(r.keyPath) + + r.mu.RLock() + cachedCert := r.cert + changed := certErr != nil || keyErr != nil || + !certInfo.ModTime().Equal(r.certModTime) || + !keyInfo.ModTime().Equal(r.keyModTime) + r.mu.RUnlock() + + if !changed { + return cachedCert, nil } - currMutation, err := s.mutatingController.Get(mutatingConfig.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err = s.mutatingController.Create(mutatingConfig) - if err != nil { - return fmt.Errorf("failed to create mutating configuration: %w", err) - } - } else if err != nil { - return fmt.Errorf("failed to get mutating configuration: %w", err) - } else { - currMutation.Webhooks = mutatingConfig.Webhooks - _, err = s.mutatingController.Update(currMutation) - if err != nil { - return fmt.Errorf("failed to update mutating configuration: %w", err) + if err := r.reload(); err != nil { + if cachedCert != nil { + logrus.Warnf("failed to reload serving cert from %s, falling back to cached cert: %v", r.certPath, err) + return cachedCert, nil } + return nil, err } - return nil + r.mu.RLock() + defer r.mu.RUnlock() + return r.cert, nil } // certAuth returns a middleware for cert-based authentication. diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index cfb2e73ce8..ab9c271260 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -1,188 +1,111 @@ package server import ( - "errors" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" "testing" + "time" - "github.com/rancher/webhook/pkg/health" - "github.com/rancher/wrangler/v3/pkg/generic/fake" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - v1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" ) -func TestSecretHandlerEnsureWebhookConfigurationCreate(t *testing.T) { - configName := "rancher.cattle.io" - - var ( - storedValidatingConfig *v1.ValidatingWebhookConfiguration - storedMutatingConfig *v1.MutatingWebhookConfiguration - ) - - ctrl := gomock.NewController(t) - validatingController := fake.NewMockNonNamespacedClientInterface[*v1.ValidatingWebhookConfiguration, *v1.ValidatingWebhookConfigurationList](ctrl) - validatingController.EXPECT().Get(configName, gomock.Any()).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: v1.GroupName, Resource: "validatingwebhookconfiguration"}, configName)).Times(1) - validatingController.EXPECT().Create(gomock.Any()).DoAndReturn(func(obj *v1.ValidatingWebhookConfiguration) (*v1.ValidatingWebhookConfiguration, error) { - storedValidatingConfig = obj.DeepCopy() - return obj, nil - }).Times(1) - - mutatingController := fake.NewMockNonNamespacedClientInterface[*v1.MutatingWebhookConfiguration, *v1.MutatingWebhookConfigurationList](ctrl) - mutatingController.EXPECT().Get(configName, gomock.Any()).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: v1.GroupName, Resource: "mutatingwebhookconfiguration"}, configName)).Times(1) - mutatingController.EXPECT().Create(gomock.Any()).DoAndReturn(func(obj *v1.MutatingWebhookConfiguration) (*v1.MutatingWebhookConfiguration, error) { - storedMutatingConfig = obj.DeepCopy() - return obj, nil - }).Times(1) - - handler := &secretHandler{ - validatingController: validatingController, - mutatingController: mutatingController, - } +// writeCert generates a fresh self-signed cert/key pair for cn and writes it to +// certPath/keyPath, so tests can simulate a secret rotation by calling it again. +func writeCert(t *testing.T, certPath, keyPath, cn string) { + t.Helper() - validatingConfig := &v1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: configName, - }, - Webhooks: []v1.ValidatingWebhook{ - { - Name: "rancher.cattle.io.features.management.cattle.io", - }, - }, - } - mutatingConfig := &v1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: configName, - }, - Webhooks: []v1.MutatingWebhook{ - { - Name: "rancher.cattle.io.clusters.provisioning.cattle.io", - }, - }, + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Unix(0, 0), + NotAfter: time.Unix(0, 0).Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, } - err := handler.ensureWebhookConfiguration(validatingConfig, mutatingConfig) + der, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv) require.NoError(t, err) - require.NotNil(t, storedValidatingConfig) - require.Len(t, storedValidatingConfig.Webhooks, 1) - assert.Equal(t, validatingConfig.Webhooks[0].Name, storedValidatingConfig.Webhooks[0].Name) + certOut, err := os.Create(certPath) + require.NoError(t, err) + require.NoError(t, pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: der})) + require.NoError(t, certOut.Close()) - require.NotNil(t, storedMutatingConfig) - require.Len(t, storedMutatingConfig.Webhooks, 1) - assert.Equal(t, mutatingConfig.Webhooks[0].Name, storedMutatingConfig.Webhooks[0].Name) + keyBytes, err := x509.MarshalECPrivateKey(priv) + require.NoError(t, err) + keyOut, err := os.Create(keyPath) + require.NoError(t, err) + require.NoError(t, pem.Encode(keyOut, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes})) + require.NoError(t, keyOut.Close()) } -// TestSyncLeaderLogic tests the logic within the sync function related to leader election. -func TestSyncLeaderLogic(t *testing.T) { - t.Parallel() - configName := "rancher.cattle.io" - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: caName, - Namespace: namespace, - }, - Data: map[string][]byte{ - corev1.TLSCertKey: []byte("cert-data"), - }, - } - testErr := errors.New("test error") - - tests := []struct { - name string - isLeader bool - getMutatingErr error - getValidatingErr error - createMutatingErr error - createValidatingErr error - expectedErr error - expectControllersRun bool - }{ - { - name: "follower becomes healthy", - isLeader: false, - expectedErr: nil, - }, - { - name: "leader becomes healthy on create", - isLeader: true, - getMutatingErr: apierrors.NewNotFound(schema.GroupResource{}, ""), - getValidatingErr: apierrors.NewNotFound(schema.GroupResource{}, ""), - expectedErr: nil, - expectControllersRun: true, - }, - { - name: "leader becomes unhealthy on create error", - isLeader: true, - getValidatingErr: apierrors.NewNotFound(schema.GroupResource{}, ""), - createValidatingErr: testErr, - expectedErr: testErr, - expectControllersRun: true, - }, - { - name: "leader becomes healthy on update", - isLeader: true, - expectedErr: nil, - expectControllersRun: true, - }, - } +func TestCertReloaderCachesUntilFilesChange(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "tls.crt") + keyPath := filepath.Join(dir, "tls.key") + writeCert(t, certPath, keyPath, "v1") - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - validatingController := fake.NewMockNonNamespacedClientInterface[*v1.ValidatingWebhookConfiguration, *v1.ValidatingWebhookConfigurationList](ctrl) - mutatingController := fake.NewMockNonNamespacedClientInterface[*v1.MutatingWebhookConfiguration, *v1.MutatingWebhookConfigurationList](ctrl) - errChecker := health.NewErrorChecker("test") - errChecker.Store(errors.New("initial error")) - - handler := &secretHandler{ - validatingController: validatingController, - mutatingController: mutatingController, - errChecker: errChecker, - } - - leaderFlag.Store(tt.isLeader) - - if tt.expectControllersRun { - validatingController.EXPECT().Get(configName, gomock.Any()).Return(&v1.ValidatingWebhookConfiguration{}, tt.getValidatingErr).Times(1) - if apierrors.IsNotFound(tt.getValidatingErr) { - validatingController.EXPECT().Create(gomock.Any()).Return(&v1.ValidatingWebhookConfiguration{}, tt.createValidatingErr).Times(1) - } else if tt.getValidatingErr == nil { - validatingController.EXPECT().Update(gomock.Any()).Return(&v1.ValidatingWebhookConfiguration{}, nil).Times(1) - } - - // Only expect calls to the mutating controller if the validating part is expected to succeed. - if (tt.getValidatingErr == nil || apierrors.IsNotFound(tt.getValidatingErr)) && tt.createValidatingErr == nil { - mutatingController.EXPECT().Get(configName, gomock.Any()).Return(&v1.MutatingWebhookConfiguration{}, tt.getMutatingErr).Times(1) - if apierrors.IsNotFound(tt.getMutatingErr) { - mutatingController.EXPECT().Create(gomock.Any()).Return(&v1.MutatingWebhookConfiguration{}, tt.createMutatingErr).Times(1) - } else if tt.getMutatingErr == nil { - mutatingController.EXPECT().Update(gomock.Any()).Return(&v1.MutatingWebhookConfiguration{}, nil).Times(1) - } - } - } - - _, err := handler.sync("test-sync", secret) - - // The only error we might get is a transient one from ensureWebhookConfiguration. - if tt.expectedErr != nil { - assert.ErrorContains(t, err, tt.expectedErr.Error(), "expected an error when ensuring webhook config") - } else { - require.NoError(t, err) - } - - healthErr := errChecker.Check(nil) - if tt.expectedErr == nil { - assert.NoError(t, healthErr, "expected pod to be healthy") - } else { - assert.Error(t, healthErr, "expected pod to be unhealthy") - } - }) - } + r, err := newCertReloader(certPath, keyPath) + require.NoError(t, err) + + first, err := r.getCertificate(nil) + require.NoError(t, err) + require.NotNil(t, first) + + second, err := r.getCertificate(nil) + require.NoError(t, err) + assert.Same(t, first, second, "expected cached cert to be reused when files are unchanged") + + // Bump mtimes so the reloader notices, but keep identical content: a + // no-op rotation should still return an equivalent (re-parsed) cert. + future := time.Now().Add(time.Minute) + require.NoError(t, os.Chtimes(certPath, future, future)) + require.NoError(t, os.Chtimes(keyPath, future, future)) + + third, err := r.getCertificate(nil) + require.NoError(t, err) + require.NotNil(t, third) + assert.NotSame(t, first, third, "expected reload after mtime change") +} + +func TestCertReloaderFallsBackToCachedCertOnReloadFailure(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "tls.crt") + keyPath := filepath.Join(dir, "tls.key") + writeCert(t, certPath, keyPath, "v1") + + r, err := newCertReloader(certPath, keyPath) + require.NoError(t, err) + + good, err := r.getCertificate(nil) + require.NoError(t, err) + + // Simulate a rotation race: tls.crt updated, tls.key not yet - or + // corrupted - so LoadX509KeyPair fails on the new pair. + future := time.Now().Add(time.Minute) + require.NoError(t, os.WriteFile(certPath, []byte("not a cert"), 0o600)) + require.NoError(t, os.Chtimes(certPath, future, future)) + + fallback, err := r.getCertificate(nil) + require.NoError(t, err, "reload failure should fall back to cached cert, not error the handshake") + assert.Same(t, good, fallback) +} + +func TestNewCertReloaderFailsWithNoCachedCert(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "tls.crt") + keyPath := filepath.Join(dir, "tls.key") + + _, err := newCertReloader(certPath, keyPath) + assert.Error(t, err, "startup with no cert files and no cache should fail, not serve nothing") }