fix(ci): self-heal a stale cluster CA in the deploy job (#9)

fix(ci): self-heal a stale cluster CA in the deploy job (#9)

.gitea/workflows/build_push.yml

@@ -28,6 +28,16 @@ jobs:
           . .venv/bin/activate
           python -m pytest tests/ -q
 
+      - name: Shellcheck the scripts
+        run: |
+          # The operational scripts are shell, so they get the same treatment
+          # as the Python code.
+          if ! command -v shellcheck >/dev/null 2>&1; then
+            apt-get update -qq
+            apt-get install -y -qq shellcheck
+          fi
+          shellcheck scripts/*.sh
+
   # NOTE: the build job deliberately does NOT have a kubeconfig.  The
   # `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build step
   # that has no need of the cluster should not be able to read it.  The deploy
@@ -118,9 +128,18 @@ jobs:
           # cluster-admin-equivalent, so it should not be readable by
           # anything else on the runner.
           install -m 600 /dev/null $HOME/.kube/config
-          echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config
+          echo "${{ secrets.KUBEC_CONFIG_BUILDX_NEW_2 }}" > $HOME/.kube/config
           chmod 600 $HOME/.kube/config
 
+      - name: Verify the cluster is reachable
+        run: |
+          # The kubeconfig embeds the k3s server CA.  If the cluster's CA has
+          # been rotated since the secret was created, kubectl cannot verify
+          # the server.  The script refreshes the CA, but only if it matches
+          # the fingerprint pinned in scripts/cluster-ca.sha256 - see the
+          # script for why.
+          bash scripts/kube-ca-refresh.sh
+
       - name: Install kubectl
         run: |
           # Skip the whole apt dance if the runner already has a kubectl.

scripts/cluster-ca.sha256

@@ -0,0 +1,17 @@
+# SHA-256 fingerprint of the cluster's API server CA (the k3s server-ca.crt).
+#
+# This is a PUBLIC value, not a secret - it can be read straight off the
+# certificate.  It lives in the repo rather than in a Gitea secret so that
+# changing it is a reviewable commit, and so nobody with secret-write access
+# can silently repoint the CI runner at a different CA.
+#
+# The deploy job fetches the cluster's CA over an unverified connection and
+# compares its fingerprint against this value.  A CA that does not match is
+# refused, so an attacker able to intercept the fetch cannot substitute
+# their own.
+#
+# To refresh it after a legitimate CA rotation:
+#   openssl x509 -in /var/lib/rancher/k3s/server/tls/server-ca.crt #     -noout -fingerprint -sha256
+# and update the line below (colons are stripped before comparison, so either
+# format works).
+d23a17bb9c5f7c66d15e15635d6bb928e3a34f38142705e4e6516075a5f35907

scripts/kube-ca-refresh.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+#
+# Make sure kubectl can talk to the cluster API server.
+#
+# The kubeconfig embeds the k3s server CA.  If the cluster's CA has been
+# rotated since the secret was created, kubectl fails with:
+#
+#   tls: failed to verify certificate: x509: certificate signed by unknown authority
+#
+# which reads like a network problem rather than a stale secret.  This script
+# refreshes the CA in place.
+#
+# The refresh is PINNED: the server's CA is fetched over an unverified
+# connection, but it is only used if its SHA-256 fingerprint matches the value
+# pinned in scripts/cluster-ca.sha256.  An attacker able to intercept the
+# fetch cannot use it - their CA will not hash to the pinned fingerprint.
+# The runner has cluster-admin credentials, so trusting an unverified CA here
+# would be a privilege-escalation hole.
+#
+# The fingerprint is a PUBLIC value - it can be read straight off the
+# certificate - so it lives in the repo rather than in a secret.  That way a
+# change to it is a reviewable commit, and nobody with secret-write access
+# can silently repoint the CI runner at a different CA.
+#
+# The expected fingerprint is read from scripts/cluster-ca.sha256 next to
+# this script.  It can be overridden with the first argument.
+#
+# Usage:
+#   kube-ca-refresh.sh [expected-fingerprint]
+#
+# Exit codes:
+#   0  the cluster is reachable
+#   1  the cluster is not reachable and could not be fixed safely
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+EXPECTED="${1:-}"
+if [ -z "$EXPECTED" ] && [ -f "$SCRIPT_DIR/cluster-ca.sha256" ]; then
+    # The file may carry comments; take the last non-comment line.
+    EXPECTED=$(grep -vE '^[[:space:]]*(#|$)' "$SCRIPT_DIR/cluster-ca.sha256" | tail -1 | tr -d '[:space:]')
+fi
+
+if kubectl version --request-timeout=15s >/dev/null 2>&1; then
+    echo "cluster reachable"
+    exit 0
+fi
+
+if [ -z "$EXPECTED" ]; then
+    echo "ERROR: the kubeconfig cannot verify the API server, and no expected CA" >&2
+    echo "  fingerprint was supplied, so the CA cannot be refreshed safely." >&2
+    echo "  and no pinned CA fingerprint could be found." >&2
+    echo "  Set one in scripts/cluster-ca.sha256:" >&2
+    echo "    openssl x509 -in server-ca.crt -noout -fingerprint -sha256" >&2
+    exit 1
+fi
+
+# Guard against a kubeconfig with no clusters, or an empty one.
+CLUSTER_COUNT=$(kubectl config view --raw -o jsonpath='{.clusters}' 2>/dev/null \
+    | grep -c '"name"' || true)
+if [ "${CLUSTER_COUNT:-0}" -eq 0 ]; then
+    echo "ERROR: the kubeconfig has no cluster entries." >&2
+    echo "  The KUBEC_CONFIG_BUILDX_NEW_2 secret looks wrong or empty." >&2
+    exit 1
+fi
+
+SERVER=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.server}')
+# The cluster entry's NAME - patching the CA into a *new* entry named
+# "default" would leave the original entry (and its stale CA) in use.
+CLUSTER_NAME=$(kubectl config view --raw -o jsonpath='{.clusters[0].name}')
+
+if [ -z "$SERVER" ] || [ -z "$CLUSTER_NAME" ]; then
+    echo "ERROR: could not read the server or cluster name from the kubeconfig." >&2
+    exit 1
+fi
+
+echo "server: $SERVER  cluster entry: $CLUSTER_NAME"
+
+CA_FILE=$(mktemp)
+trap 'rm -f "$CA_FILE"' EXIT
+
+# Timeouts so an unreachable API server fails the job rather than hanging
+# the runner indefinitely.
+curl -sk --connect-timeout 10 --max-time 30 "$SERVER/cacerts" -o "$CA_FILE"
+if [ ! -s "$CA_FILE" ]; then
+    echo "ERROR: could not fetch the cluster CA from $SERVER/cacerts" >&2
+    exit 1
+fi
+
+# Normalise both to lowercase hex without separators, so the comparison does
+# not depend on the colon-separated format openssl prints.
+normalise() {
+    tr -d ':' | tr '[:upper:]' '[:lower:]'
+}
+
+ACTUAL=$(openssl x509 -in "$CA_FILE" -noout -fingerprint -sha256 | cut -d= -f2 | normalise)
+WANTED=$(printf '%s' "$EXPECTED" | normalise)
+
+echo "expected CA fingerprint: $WANTED"
+echo "actual   CA fingerprint: $ACTUAL"
+
+if [ "$ACTUAL" != "$WANTED" ]; then
+    echo "ERROR: the cluster CA does not match the pinned fingerprint." >&2
+    echo "  expected: $WANTED" >&2
+    echo "  actual:   $ACTUAL" >&2
+    echo "  Refusing to use it.  Either the CA has been rotated and the" >&2
+    echo "  pinned fingerprint needs updating, or the connection was" >&2
+    echo "  intercepted." >&2
+    exit 1
+fi
+
+# The fingerprint matches the pinned value, so this is the CA we expect.
+kubectl config set-cluster "$CLUSTER_NAME" \
+    --embed-certs=true \
+    --certificate-authority="$CA_FILE" >/dev/null
+
+if kubectl version --request-timeout=15s >/dev/null 2>&1; then
+    echo "cluster reachable after refreshing the CA"
+    exit 0
+fi
+
+echo "ERROR: cannot reach the cluster API server even with a refreshed CA." >&2
+echo "  The kubeconfig secret may be stale or invalid." >&2
+exit 1