diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml index 99bf4c5..4f46187 100644 --- a/.gitea/workflows/build_push.yml +++ b/.gitea/workflows/build_push.yml @@ -123,50 +123,11 @@ jobs: - name: Verify the cluster is reachable run: | - # Fail fast, and with a useful message, if the kubeconfig cannot - # talk to the API server. - # - # The certificate-authority-data inside the kubeconfig is the k3s - # server CA. If the cluster's CA has been rotated since the secret - # was created, kubectl fails with: - # tls: failed to verify certificate: x509: certificate signed by - # unknown authority - # which is a confusing way to learn that a secret is stale. - if kubectl version --request-timeout=15s >/dev/null 2>&1; then - echo "cluster reachable" - exit 0 - fi - - echo "the kubeconfig could not verify the API server - refreshing the CA" - - # The API server publishes its own CA at /cacerts. Fetch it over an - # insecure connection - we cannot verify it yet, that is the problem - # being solved - and pin it explicitly. This makes the deploy - # self-healing after a CA rotation, instead of needing the secret - # to be regenerated by hand. - SERVER=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.server}') - echo "server: $SERVER" - curl -sk "$SERVER/cacerts" -o $HOME/.kube/ca.crt - - if [ ! -s $HOME/.kube/ca.crt ]; then - echo "ERROR: could not fetch the cluster CA from $SERVER/cacerts" - exit 1 - fi - - kubectl config set-cluster default \ - --embed-certs=true \ - --certificate-authority=$HOME/.kube/ca.crt >/dev/null - rm -f $HOME/.kube/ca.crt - - if kubectl version --request-timeout=15s >/dev/null 2>&1; then - echo "cluster reachable after refreshing the CA" - exit 0 - fi - - echo "ERROR: cannot reach the cluster API server even with a refreshed CA." - echo " The kubeconfig in the KUBEC_CONFIG_BUILDX_NEW_2 secret may be" - echo " stale or invalid, and needs regenerating." - exit 1 + # The kubeconfig embeds the k3s server CA. If the cluster's CA has + # been rotated since the secret was created, kubectl cannot verify + # the server. The script refreshes the CA, but only if it matches + # the fingerprint pinned in the secret - see the script for why. + bash scripts/kube-ca-refresh.sh "${{ secrets.KUBE_SERVER_CA_FINGERPRINT }}" - name: Install kubectl run: | diff --git a/scripts/kube-ca-refresh.sh b/scripts/kube-ca-refresh.sh new file mode 100755 index 0000000..f513919 --- /dev/null +++ b/scripts/kube-ca-refresh.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Make sure kubectl can talk to the cluster API server. +# +# The kubeconfig embeds the k3s server CA. If the cluster's CA has been +# rotated since the secret was created, kubectl fails with: +# +# tls: failed to verify certificate: x509: certificate signed by unknown authority +# +# which reads like a network problem rather than a stale secret. This script +# refreshes the CA in place. +# +# The refresh is PINNED: the server's CA is fetched over an unverified +# connection, but it is only used if its SHA-256 fingerprint matches the value +# in the KUBE_SERVER_CA_FINGERPRINT secret. An attacker able to intercept +# the fetch cannot use it - their CA will not hash to the pinned fingerprint. +# The runner has cluster-admin credentials, so trusting an unverified CA here +# would be a privilege-escalation hole. +# +# Usage: +# kube-ca-refresh.sh +# +# Exit codes: +# 0 the cluster is reachable +# 1 the cluster is not reachable and could not be fixed safely + +set -euo pipefail + +EXPECTED="${1:-}" + +if kubectl version --request-timeout=15s >/dev/null 2>&1; then + echo "cluster reachable" + exit 0 +fi + +if [ -z "$EXPECTED" ]; then + echo "ERROR: the kubeconfig cannot verify the API server, and no expected CA" >&2 + echo " fingerprint was supplied, so the CA cannot be refreshed safely." >&2 + echo " Set KUBE_SERVER_CA_FINGERPRINT to the SHA-256 fingerprint of the" >&2 + echo " cluster's CA:" >&2 + echo " openssl x509 -in server-ca.crt -noout -fingerprint -sha256" >&2 + exit 1 +fi + +SERVER=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.server}') +# The cluster entry's NAME - patching the CA into a *new* entry named +# "default" would leave the original entry (and its stale CA) in use. +CLUSTER_NAME=$(kubectl config view --raw -o jsonpath='{.clusters[0].name}') +echo "server: $SERVER cluster entry: $CLUSTER_NAME" + +CA_FILE=$(mktemp) +trap 'rm -f "$CA_FILE"' EXIT + +curl -sk "$SERVER/cacerts" -o "$CA_FILE" +if [ ! -s "$CA_FILE" ]; then + echo "ERROR: could not fetch the cluster CA from $SERVER/cacerts" >&2 + exit 1 +fi + +# Normalise both to lowercase hex without separators, so the comparison does +# not depend on the colon-separated format openssl prints. +normalise() { + tr -d ':' | tr 'A-Z' 'a-z' +} + +ACTUAL=$(openssl x509 -in "$CA_FILE" -noout -fingerprint -sha256 | cut -d= -f2 | normalise) +WANTED=$(printf '%s' "$EXPECTED" | normalise) + +echo "expected CA fingerprint: $WANTED" +echo "actual CA fingerprint: $ACTUAL" + +if [ "$ACTUAL" != "$WANTED" ]; then + echo "ERROR: the cluster CA does not match the pinned fingerprint." >&2 + echo " Refusing to use it. Either the CA has been rotated and the" >&2 + echo " fingerprint secret needs updating, or the connection was" >&2 + echo " intercepted." >&2 + exit 1 +fi + +# The fingerprint matches the pinned value, so this is the CA we expect. +kubectl config set-cluster "$CLUSTER_NAME" \ + --embed-certs=true \ + --certificate-authority="$CA_FILE" >/dev/null + +if kubectl version --request-timeout=15s >/dev/null 2>&1; then + echo "cluster reachable after refreshing the CA" + exit 0 +fi + +echo "ERROR: cannot reach the cluster API server even with a refreshed CA." >&2 +echo " The kubeconfig secret may be stale or invalid." >&2 +exit 1