diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml index e1de732..8adc926 100644 --- a/.gitea/workflows/build_push.yml +++ b/.gitea/workflows/build_push.yml @@ -28,6 +28,16 @@ jobs: . .venv/bin/activate python -m pytest tests/ -q + - name: Shellcheck the scripts + run: | + # The operational scripts are shell, so they get the same treatment + # as the Python code. + if ! command -v shellcheck >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq shellcheck + fi + shellcheck scripts/*.sh + # NOTE: the build job deliberately does NOT have a kubeconfig. The # `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build step # that has no need of the cluster should not be able to read it. The deploy @@ -118,9 +128,18 @@ jobs: # cluster-admin-equivalent, so it should not be readable by # anything else on the runner. install -m 600 /dev/null $HOME/.kube/config - echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config + echo "${{ secrets.KUBEC_CONFIG_BUILDX_NEW_2 }}" > $HOME/.kube/config chmod 600 $HOME/.kube/config + - name: Verify the cluster is reachable + run: | + # The kubeconfig embeds the k3s server CA. If the cluster's CA has + # been rotated since the secret was created, kubectl cannot verify + # the server. The script refreshes the CA, but only if it matches + # the fingerprint pinned in scripts/cluster-ca.sha256 - see the + # script for why. + bash scripts/kube-ca-refresh.sh + - name: Install kubectl run: | # Skip the whole apt dance if the runner already has a kubectl. diff --git a/scripts/cluster-ca.sha256 b/scripts/cluster-ca.sha256 new file mode 100644 index 0000000..3f6664b --- /dev/null +++ b/scripts/cluster-ca.sha256 @@ -0,0 +1,17 @@ +# SHA-256 fingerprint of the cluster's API server CA (the k3s server-ca.crt). +# +# This is a PUBLIC value, not a secret - it can be read straight off the +# certificate. It lives in the repo rather than in a Gitea secret so that +# changing it is a reviewable commit, and so nobody with secret-write access +# can silently repoint the CI runner at a different CA. +# +# The deploy job fetches the cluster's CA over an unverified connection and +# compares its fingerprint against this value. A CA that does not match is +# refused, so an attacker able to intercept the fetch cannot substitute +# their own. +# +# To refresh it after a legitimate CA rotation: +# openssl x509 -in /var/lib/rancher/k3s/server/tls/server-ca.crt # -noout -fingerprint -sha256 +# and update the line below (colons are stripped before comparison, so either +# format works). +d23a17bb9c5f7c66d15e15635d6bb928e3a34f38142705e4e6516075a5f35907 diff --git a/scripts/kube-ca-refresh.sh b/scripts/kube-ca-refresh.sh new file mode 100755 index 0000000..f80117b --- /dev/null +++ b/scripts/kube-ca-refresh.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# +# Make sure kubectl can talk to the cluster API server. +# +# The kubeconfig embeds the k3s server CA. If the cluster's CA has been +# rotated since the secret was created, kubectl fails with: +# +# tls: failed to verify certificate: x509: certificate signed by unknown authority +# +# which reads like a network problem rather than a stale secret. This script +# refreshes the CA in place. +# +# The refresh is PINNED: the server's CA is fetched over an unverified +# connection, but it is only used if its SHA-256 fingerprint matches the value +# pinned in scripts/cluster-ca.sha256. An attacker able to intercept the +# fetch cannot use it - their CA will not hash to the pinned fingerprint. +# The runner has cluster-admin credentials, so trusting an unverified CA here +# would be a privilege-escalation hole. +# +# The fingerprint is a PUBLIC value - it can be read straight off the +# certificate - so it lives in the repo rather than in a secret. That way a +# change to it is a reviewable commit, and nobody with secret-write access +# can silently repoint the CI runner at a different CA. +# +# The expected fingerprint is read from scripts/cluster-ca.sha256 next to +# this script. It can be overridden with the first argument. +# +# Usage: +# kube-ca-refresh.sh [expected-fingerprint] +# +# Exit codes: +# 0 the cluster is reachable +# 1 the cluster is not reachable and could not be fixed safely + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +EXPECTED="${1:-}" +if [ -z "$EXPECTED" ] && [ -f "$SCRIPT_DIR/cluster-ca.sha256" ]; then + # The file may carry comments; take the last non-comment line. + EXPECTED=$(grep -vE '^[[:space:]]*(#|$)' "$SCRIPT_DIR/cluster-ca.sha256" | tail -1 | tr -d '[:space:]') +fi + +if kubectl version --request-timeout=15s >/dev/null 2>&1; then + echo "cluster reachable" + exit 0 +fi + +if [ -z "$EXPECTED" ]; then + echo "ERROR: the kubeconfig cannot verify the API server, and no expected CA" >&2 + echo " fingerprint was supplied, so the CA cannot be refreshed safely." >&2 + echo " and no pinned CA fingerprint could be found." >&2 + echo " Set one in scripts/cluster-ca.sha256:" >&2 + echo " openssl x509 -in server-ca.crt -noout -fingerprint -sha256" >&2 + exit 1 +fi + +# Guard against a kubeconfig with no clusters, or an empty one. +CLUSTER_COUNT=$(kubectl config view --raw -o jsonpath='{.clusters}' 2>/dev/null \ + | grep -c '"name"' || true) +if [ "${CLUSTER_COUNT:-0}" -eq 0 ]; then + echo "ERROR: the kubeconfig has no cluster entries." >&2 + echo " The KUBEC_CONFIG_BUILDX_NEW_2 secret looks wrong or empty." >&2 + exit 1 +fi + +SERVER=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.server}') +# The cluster entry's NAME - patching the CA into a *new* entry named +# "default" would leave the original entry (and its stale CA) in use. +CLUSTER_NAME=$(kubectl config view --raw -o jsonpath='{.clusters[0].name}') + +if [ -z "$SERVER" ] || [ -z "$CLUSTER_NAME" ]; then + echo "ERROR: could not read the server or cluster name from the kubeconfig." >&2 + exit 1 +fi + +echo "server: $SERVER cluster entry: $CLUSTER_NAME" + +CA_FILE=$(mktemp) +trap 'rm -f "$CA_FILE"' EXIT + +# Timeouts so an unreachable API server fails the job rather than hanging +# the runner indefinitely. +curl -sk --connect-timeout 10 --max-time 30 "$SERVER/cacerts" -o "$CA_FILE" +if [ ! -s "$CA_FILE" ]; then + echo "ERROR: could not fetch the cluster CA from $SERVER/cacerts" >&2 + exit 1 +fi + +# Normalise both to lowercase hex without separators, so the comparison does +# not depend on the colon-separated format openssl prints. +normalise() { + tr -d ':' | tr '[:upper:]' '[:lower:]' +} + +ACTUAL=$(openssl x509 -in "$CA_FILE" -noout -fingerprint -sha256 | cut -d= -f2 | normalise) +WANTED=$(printf '%s' "$EXPECTED" | normalise) + +echo "expected CA fingerprint: $WANTED" +echo "actual CA fingerprint: $ACTUAL" + +if [ "$ACTUAL" != "$WANTED" ]; then + echo "ERROR: the cluster CA does not match the pinned fingerprint." >&2 + echo " expected: $WANTED" >&2 + echo " actual: $ACTUAL" >&2 + echo " Refusing to use it. Either the CA has been rotated and the" >&2 + echo " pinned fingerprint needs updating, or the connection was" >&2 + echo " intercepted." >&2 + exit 1 +fi + +# The fingerprint matches the pinned value, so this is the CA we expect. +kubectl config set-cluster "$CLUSTER_NAME" \ + --embed-certs=true \ + --certificate-authority="$CA_FILE" >/dev/null + +if kubectl version --request-timeout=15s >/dev/null 2>&1; then + echo "cluster reachable after refreshing the CA" + exit 0 +fi + +echo "ERROR: cannot reach the cluster API server even with a refreshed CA." >&2 +echo " The kubeconfig secret may be stale or invalid." >&2 +exit 1