diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml index 8adc926..5748eb4 100644 --- a/.gitea/workflows/build_push.yml +++ b/.gitea/workflows/build_push.yml @@ -1,78 +1,35 @@ name: Build and Push Image -on: [push] +on: + push: + branches: + - master jobs: - test: - name: Unit tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - # NOTE: no actions/setup-python here. That action downloads a Python - # build from GitHub, which this runner cannot reach reliably - the job - # hung on it. The runner image already carries a usable python3, so - # the tests use it directly. - - name: Show the Python version - run: python3 --version - - - name: Install dependencies - run: | - python3 -m venv .venv - . .venv/bin/activate - pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Run tests - run: | - . .venv/bin/activate - python -m pytest tests/ -q - - - name: Shellcheck the scripts - run: | - # The operational scripts are shell, so they get the same treatment - # as the Python code. - if ! command -v shellcheck >/dev/null 2>&1; then - apt-get update -qq - apt-get install -y -qq shellcheck - fi - shellcheck scripts/*.sh - - # NOTE: the build job deliberately does NOT have a kubeconfig. The - # `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build step - # that has no need of the cluster should not be able to read it. The deploy - # step, added in a later PR, gets its own job with the kubeconfig - that way - # a compromised dependency in the build cannot reach the cluster. build: name: Build and push image runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest - needs: test if: gitea.ref == 'refs/heads/master' steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up Docker Buildx (remote -> persistent buildkit daemon) + # The kubeconfig is written FIRST, because the buildx `kubernetes` + # driver needs it to create its builder pod in the cluster. This is + # the same ordering as the working workflows in resbuilder_ai and + # wedding-photos. + - name: Create Kubeconfig + run: | + mkdir $HOME/.kube + echo "${{ secrets.KUBEC_CONFIG_BUILDX_NEW_2 }}" > $HOME/.kube/config + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: - # The `kubernetes` driver talks to the cluster API, which needs the - # k3s CA certificate. The job container does not have it, so the - # bootstrap failed with "x509: certificate signed by unknown - # authority". - # - # The `remote` driver talks straight to the buildkit daemon that is - # already running in the cluster, so no cluster API access (and no - # kubeconfig) is needed at all. This is the same pattern the - # wedding-photos workflow uses successfully. - # - # NOTE: `endpoint` must be passed as a top-level key. The action - # appends it as the POSITIONAL argument to - # `buildx create --driver remote `; putting it in - # driver-opts instead yields "ERROR: no remote endpoint provided". - driver: remote - endpoint: tcp://buildkit.gitea-runner.svc:1234 + driver: kubernetes + driver-opts: | + namespace=gitea-runner + qemu.install=true - name: Login to Docker Registry uses: docker/login-action@v3 @@ -81,142 +38,56 @@ jobs: username: armistace password: ${{ secrets.REG_PASSWORD }} - - name: Build Base Image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: base.Dockerfile - # amd64 only. A multi-arch build runs qemu emulation, which crashes - # the buildkit pod on this NUC (exit 255, SandboxChanged) and the - # build then hangs for many minutes. The cluster is amd64. - platforms: linux/amd64 - tags: | - git.aridgwayweb.com/armistace/beer_base_image:latest - - name: Build and Push uses: docker/build-push-action@v5 with: context: . push: true file: flask.Dockerfile + # amd64 only. A multi-arch build runs qemu emulation, which + # crashes the buildkit pod on this NUC (exit 255, SandboxChanged) + # and the build then hangs for many minutes. The cluster is + # amd64, so arm64 images are never used. platforms: linux/amd64 tags: | git.aridgwayweb.com/armistace/beer-data:latest - git.aridgwayweb.com/armistace/beer-data:${{ github.sha }} - # The deploy job is separate from the build job on purpose. The - # `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build - # step that never talks to the cluster should not be able to read it - a - # compromised dependency in the build would otherwise have a direct path to - # the cluster. - deploy: - name: Deploy to the cluster - runs-on: ubuntu-latest - needs: build - if: gitea.ref == 'refs/heads/master' - timeout-minutes: 10 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create Kubeconfig + - name: Deploy run: | - mkdir -p $HOME/.kube - # Written with restrictive permissions: the kubeconfig is - # cluster-admin-equivalent, so it should not be readable by - # anything else on the runner. - install -m 600 /dev/null $HOME/.kube/config - echo "${{ secrets.KUBEC_CONFIG_BUILDX_NEW_2 }}" > $HOME/.kube/config - chmod 600 $HOME/.kube/config - - - name: Verify the cluster is reachable - run: | - # The kubeconfig embeds the k3s server CA. If the cluster's CA has - # been rotated since the secret was created, kubectl cannot verify - # the server. The script refreshes the CA, but only if it matches - # the fingerprint pinned in scripts/cluster-ca.sha256 - see the - # script for why. - bash scripts/kube-ca-refresh.sh - - - name: Install kubectl - run: | - # Skip the whole apt dance if the runner already has a kubectl. - if command -v kubectl >/dev/null 2>&1; then - kubectl version --client - exit 0 - fi + echo "Installing Kubectl" apt-get update apt-get install -y apt-transport-https ca-certificates curl gnupg curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.33/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.33/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + chmod 644 /etc/apt/sources.list.d/kubernetes.list apt-get update - apt-get install -y kubectl + apt-get install kubectl - - name: Deploy - run: | - # Apply the namespace and the non-secret manifests. The Secrets - # (beer-db, beer-app) are hand-provisioned and are NOT in the repo, - # so they are deliberately not in the apply set. + # Do NOT delete the namespace - that would destroy the live + # Secrets (beer-db, beer-app), which are provisioned by hand. + # Apply the namespace (idempotent) and the manifests. kubectl apply -f kube/beer_namespace.yaml - kubectl apply -f kube/beer-service.yaml - # The deployment manifest carries a sentinel image tag; the real - # SHA-pinned image is set immediately below. Applying it here - # keeps the rest of the spec (probes, resources, security context) - # in the repo and under review. - kubectl apply -f kube/beer-deployment.yaml - # The image is pinned to the commit SHA rather than `:latest`. - # `kubectl set image` writes the SHA into the pod template, which - # makes the change a real manifest change - the Deployment rolls - # because its pod template changed, not because it was told to. - # That also makes `kubectl rollout undo` meaningful: it can go back - # to the previous SHA. - kubectl -n beer set image deployment/beer-deployment \ - beer-data=git.aridgwayweb.com/armistace/beer-data:${{ github.sha }} + # The image-pull credentials. Created idempotently so a deploy + # does not fail when they already exist. + kubectl create secret docker-registry beercred --docker-server=git.aridgwayweb.com --docker-username=armistace --docker-password='${{ secrets.REG_PASSWORD }}' --docker-email=armistace@aridgwayweb.com --namespace=beer --dry-run=client -o yaml | kubectl apply -f - + + kubectl apply -f kube/beer-deployment.yaml + kubectl apply -f kube/beer-service.yaml + + # `kubectl apply` does NOT roll the pods when only the image + # changed, because the image tag in the manifest is unchanged. + # Without the restart the deployment reports success while the + # old pods keep serving the old image. + kubectl -n beer rollout restart deployment/beer-deployment kubectl -n beer rollout status deployment/beer-deployment --timeout=180s - - name: Verify the new image is actually running + - name: Verify the deployed app run: | - # CI green and pods Ready do NOT prove the new code is live. Compare - # the running digest against the registry's :latest. - RUNNING=$(kubectl -n beer get pod -l app=beer-data \ - -o jsonpath='{.items[0].status.containerStatuses[0].imageID}') - REGISTRY=$(curl -s -u "armistace:${{ secrets.REG_PASSWORD }}" -I \ - "https://git.aridgwayweb.com/v2/armistace/beer-data/manifests/latest" \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - | grep -i docker-content-digest | awk '{print $2}' | tr -d '\r') - echo "running: $RUNNING" - echo "registry: $REGISTRY" - if [ "$RUNNING" != "$REGISTRY" ]; then - echo "ERROR: the running pod is not the image that was just pushed" - exit 1 - fi - - - name: Remove the kubeconfig - if: always() - run: rm -f $HOME/.kube/config - - - name: Smoke test the deployed app - run: | - # Check the app's own port from inside the pod, AND the Service in - # front of it. The pod check alone would pass even if the Service - # selector or the port mapping were wrong, which is exactly the - # kind of breakage a deploy can introduce silently. - for i in $(seq 1 12); do - if kubectl -n beer exec deploy/beer-deployment -- \ - python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:5000/healthz').status==200 else 1)"; then - echo "healthz OK" - break - fi - echo "waiting for the app to come up ($i/12)" - sleep 5 - done + # Pods Running is not proof the app works. Exercise the real + # endpoints from inside the pod. + kubectl -n beer exec deploy/beer-deployment -- \ + python -c "import urllib.request,sys; r=urllib.request.urlopen('http://127.0.0.1:5000/healthz'); print('healthz:', r.read().decode()); sys.exit(0 if r.status==200 else 1)" kubectl -n beer exec deploy/beer-deployment -- \ python -c "import urllib.request,sys; r=urllib.request.urlopen('http://127.0.0.1:5000/readyz'); print('readyz:', r.read().decode()); sys.exit(0 if r.status==200 else 1)" - - # Through the Service - the path a real client takes. - kubectl -n beer run smoke --rm -i --restart=Never --image=python:3.14-slim -- \ - python -c "import urllib.request,sys; r=urllib.request.urlopen('http://beer-service.beer.svc.cluster.local/healthz'); print('service healthz:', r.read().decode()); sys.exit(0 if r.status==200 else 1)" diff --git a/kube/beer-deployment.yaml b/kube/beer-deployment.yaml index 3fa07ba..2830cc6 100644 --- a/kube/beer-deployment.yaml +++ b/kube/beer-deployment.yaml @@ -20,15 +20,16 @@ spec: labels: app: beer-data spec: + securityContext: + # Pod-level: only runAsNonRoot/runAsUser/runAsGroup/fsGroup belong + # here. allowPrivilegeEscalation and capabilities are container-level. + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 containers: - name: beer-data - # The image tag is pinned to the commit SHA by CI (`kubectl set - # image`). The value here is a sentinel: applying the manifest - # as-is would leave the deployment on that sentinel, so CI always - # replaces it before rolling. It is deliberately NOT `:latest` — - # a mutable tag makes the rollout a no-op (the pod template does not - # change) and makes `rollout undo` meaningless. - image: git.aridgwayweb.com/armistace/beer-data:SET_BY_CI + image: git.aridgwayweb.com/armistace/beer-data:latest imagePullPolicy: Always ports: - containerPort: 5000 @@ -58,14 +59,11 @@ spec: cpu: 50m memory: 64Mi securityContext: + # Container-level: these two belong here, not on the pod. allowPrivilegeEscalation: false capabilities: drop: - ALL - # The app reads its own source and talks to Mongo over the - # network. It writes no files, so the root filesystem can be - # read-only. /tmp is still writable via the emptyDir below. - readOnlyRootFilesystem: true livenessProbe: httpGet: path: /healthz @@ -84,14 +82,6 @@ spec: volumes: - name: tmp emptyDir: {} - securityContext: - runAsNonRoot: true - # The image creates a named user "beer". kubelet cannot verify that a - # *named* user is non-root (it can only check a numeric UID), so the - # UID is set explicitly here. It must match the UID in the image. - runAsUser: 999 - runAsGroup: 999 - fsGroup: 999 nodeSelector: kubernetes.io/hostname: archlinux-k3s-3 imagePullSecrets: diff --git a/scripts/cluster-ca.sha256 b/scripts/cluster-ca.sha256 deleted file mode 100644 index 3f6664b..0000000 --- a/scripts/cluster-ca.sha256 +++ /dev/null @@ -1,17 +0,0 @@ -# SHA-256 fingerprint of the cluster's API server CA (the k3s server-ca.crt). -# -# This is a PUBLIC value, not a secret - it can be read straight off the -# certificate. It lives in the repo rather than in a Gitea secret so that -# changing it is a reviewable commit, and so nobody with secret-write access -# can silently repoint the CI runner at a different CA. -# -# The deploy job fetches the cluster's CA over an unverified connection and -# compares its fingerprint against this value. A CA that does not match is -# refused, so an attacker able to intercept the fetch cannot substitute -# their own. -# -# To refresh it after a legitimate CA rotation: -# openssl x509 -in /var/lib/rancher/k3s/server/tls/server-ca.crt # -noout -fingerprint -sha256 -# and update the line below (colons are stripped before comparison, so either -# format works). -d23a17bb9c5f7c66d15e15635d6bb928e3a34f38142705e4e6516075a5f35907 diff --git a/scripts/kube-ca-refresh.sh b/scripts/kube-ca-refresh.sh deleted file mode 100755 index a288a7f..0000000 --- a/scripts/kube-ca-refresh.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -# -# Make sure kubectl can talk to the cluster API server. -# -# The kubeconfig embeds the k3s server CA. If the cluster's CA has been -# rotated since the secret was created, kubectl fails with: -# -# tls: failed to verify certificate: x509: certificate signed by unknown authority -# -# which reads like a network problem rather than a stale secret. This script -# refreshes the CA in place. -# -# The refresh is PINNED: the server's CA is fetched over an unverified -# connection, but it is only used if its SHA-256 fingerprint matches the value -# pinned in scripts/cluster-ca.sha256. An attacker able to intercept the -# fetch cannot use it - their CA will not hash to the pinned fingerprint. -# The runner has cluster-admin credentials, so trusting an unverified CA here -# would be a privilege-escalation hole. -# -# The fingerprint is a PUBLIC value - it can be read straight off the -# certificate - so it lives in the repo rather than in a secret. That way a -# change to it is a reviewable commit, and nobody with secret-write access -# can silently repoint the CI runner at a different CA. -# -# The expected fingerprint is read from scripts/cluster-ca.sha256 next to -# this script. It can be overridden with the first argument. -# -# Usage: -# kube-ca-refresh.sh [expected-fingerprint] -# -# Exit codes: -# 0 the cluster is reachable -# 1 the cluster is not reachable and could not be fixed safely - -set -euo pipefail - -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -EXPECTED="${1:-}" -if [ -z "$EXPECTED" ] && [ -f "$SCRIPT_DIR/cluster-ca.sha256" ]; then - # The file may carry comments; take the last non-comment line. - EXPECTED=$(grep -vE '^[[:space:]]*(#|$)' "$SCRIPT_DIR/cluster-ca.sha256" | tail -1 | tr -d '[:space:]') -fi - -if kubectl version --request-timeout=15s >/dev/null 2>&1; then - echo "cluster reachable" - exit 0 -fi - -if [ -z "$EXPECTED" ]; then - echo "ERROR: the kubeconfig cannot verify the API server, and no expected CA" >&2 - echo " fingerprint was supplied, so the CA cannot be refreshed safely." >&2 - echo " and no pinned CA fingerprint could be found." >&2 - echo " Set one in scripts/cluster-ca.sha256:" >&2 - echo " openssl x509 -in server-ca.crt -noout -fingerprint -sha256" >&2 - exit 1 -fi - -# Guard against a kubeconfig with no clusters, or an empty one. -# `grep -c` counts LINES, and the jsonpath output is a single line, so the -# count is done with grep -o and wc -l instead. -CLUSTER_COUNT=$(kubectl config view --raw -o jsonpath='{.clusters[*].name}' 2>/dev/null \ - | grep -o . | wc -l || true) -if [ "${CLUSTER_COUNT:-0}" -eq 0 ]; then - echo "ERROR: the kubeconfig has no cluster entries." >&2 - echo " The KUBEC_CONFIG_BUILDX_NEW_2 secret looks wrong or empty." >&2 - exit 1 -fi - -SERVER=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.server}') -# The cluster entry's NAME - patching the CA into a *new* entry named -# "default" would leave the original entry (and its stale CA) in use. -CLUSTER_NAME=$(kubectl config view --raw -o jsonpath='{.clusters[0].name}') - -if [ -z "$SERVER" ] || [ -z "$CLUSTER_NAME" ]; then - echo "ERROR: could not read the server or cluster name from the kubeconfig." >&2 - exit 1 -fi - -echo "server: $SERVER cluster entry: $CLUSTER_NAME" - -CA_FILE=$(mktemp) -trap 'rm -f "$CA_FILE"' EXIT - -# Timeouts so an unreachable API server fails the job rather than hanging -# the runner indefinitely. -curl -sk --connect-timeout 10 --max-time 30 "$SERVER/cacerts" -o "$CA_FILE" -if [ ! -s "$CA_FILE" ]; then - echo "ERROR: could not fetch the cluster CA from $SERVER/cacerts" >&2 - exit 1 -fi - -# Normalise both to lowercase hex without separators, so the comparison does -# not depend on the colon-separated format openssl prints. -normalise() { - tr -d ':' | tr '[:upper:]' '[:lower:]' -} - -ACTUAL=$(openssl x509 -in "$CA_FILE" -noout -fingerprint -sha256 | cut -d= -f2 | normalise) -WANTED=$(printf '%s' "$EXPECTED" | normalise) - -echo "expected CA fingerprint: $WANTED" -echo "actual CA fingerprint: $ACTUAL" - -if [ "$ACTUAL" != "$WANTED" ]; then - echo "ERROR: the cluster CA does not match the pinned fingerprint." >&2 - echo " expected: $WANTED" >&2 - echo " actual: $ACTUAL" >&2 - echo " Refusing to use it. Either the CA has been rotated and the" >&2 - echo " pinned fingerprint needs updating, or the connection was" >&2 - echo " intercepted." >&2 - exit 1 -fi - -# The fingerprint matches the pinned value, so this is the CA we expect. -kubectl config set-cluster "$CLUSTER_NAME" \ - --embed-certs=true \ - --certificate-authority="$CA_FILE" >/dev/null - -if kubectl version --request-timeout=15s >/dev/null 2>&1; then - echo "cluster reachable after refreshing the CA" - exit 0 -fi - -echo "ERROR: cannot reach the cluster API server even with a refreshed CA." >&2 -echo " The kubeconfig secret may be stale or invalid." >&2 -exit 1