From 55c5de9d6147f303d29223e7dc4eefdbcf597016 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 9 Nov 2026 11:45:41 +1000 Subject: [PATCH 1/4] feat(deploy): add the kubernetes manifests so the deployment is in the repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cluster objects for the beer namespace were created by hand. This puts them in the repo so they can be applied and reviewed like everything else. - `beer-deployment.yaml` — the Deployment, with the security context, the resource limits, and both probes wired up (`/healthz` liveness, `/readyz` readiness). - `beer-service.yaml` — the NodePort service. - `beer_namespace.yaml` — the namespace. The credentials are NOT in these files. `MONGO_PASS` and `SECRET_KEY` come from the live `beer-db` and `beer-app` Secrets, so applying the manifests cannot clobber a hand-provisioned credential. Notes on the pod security context --------------------------------- `runAsNonRoot: true` cannot be set on the container when the image's user is a *named* user - kubelet can only verify a numeric UID. The pod-level security context therefore sets `runAsUser: 999`, which matches the `beer` user the image creates (verified with `id` in the built image). The container also drops all capabilities and refuses privilege escalation. --- kube/beer-deployment.yaml | 91 +++++++++++++++++++++++++++++++++++++++ kube/beer-service.yaml | 17 ++++++++ kube/beer_namespace.yaml | 6 +++ 3 files changed, 114 insertions(+) create mode 100644 kube/beer-deployment.yaml create mode 100644 kube/beer-service.yaml create mode 100644 kube/beer_namespace.yaml diff --git a/kube/beer-deployment.yaml b/kube/beer-deployment.yaml new file mode 100644 index 0000000..dc67fb2 --- /dev/null +++ b/kube/beer-deployment.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beer-deployment + namespace: beer + labels: + app: beer-data +spec: + replicas: 3 + selector: + matchLabels: + app: beer-data + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: beer-data + spec: + containers: + - name: beer-data + image: git.aridgwayweb.com/armistace/beer-data:latest + imagePullPolicy: Always + ports: + - containerPort: 5000 + protocol: TCP + env: + - name: MONGO_HOST + value: "192.168.178.229" + - name: MONGO_USER + value: "root" + # MONGO_PASS and SECRET_KEY come from the live Secret - they are + # deliberately NOT in this file. See RUNBOOK.md. + - name: MONGO_PASS + valueFrom: + secretKeyRef: + name: beer-db + key: mongo_pass + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: beer-app + key: secret_key + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + livenessProbe: + httpGet: + path: /healthz + port: 5000 + initialDelaySeconds: 10 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 5000 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} + securityContext: + runAsNonRoot: true + # The image creates a named user "beer". kubelet cannot verify that a + # *named* user is non-root (it can only check a numeric UID), so the + # UID is set explicitly here. It must match the UID in the image. + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + nodeSelector: + kubernetes.io/hostname: archlinux-k3s-3 + imagePullSecrets: + - name: beercred + terminationGracePeriodSeconds: 30 + restartPolicy: Always diff --git a/kube/beer-service.yaml b/kube/beer-service.yaml new file mode 100644 index 0000000..e562b70 --- /dev/null +++ b/kube/beer-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: beer-service + namespace: beer + labels: + app: beer-data +spec: + type: NodePort + selector: + app: beer-data + ports: + - name: http + port: 80 + targetPort: 5000 + nodePort: 30015 + protocol: TCP diff --git a/kube/beer_namespace.yaml b/kube/beer_namespace.yaml new file mode 100644 index 0000000..1f0f5b5 --- /dev/null +++ b/kube/beer_namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: beer + labels: + app: beer-data -- 2.39.5 From b73df93f20c9883c8c17a06ed2d7fcafb95afbce Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 9 Nov 2026 11:46:06 +1000 Subject: [PATCH 2/4] feat(ci): add the deploy job so the app deploys from the repo The workflow built and pushed the image but never deployed it - the cluster objects had been created by hand. This adds a `deploy` job that applies the manifests in `kube/` and rolls the deployment. The deploy job is separate from the build job on purpose. The `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build step that never talks to the cluster should not be able to read it. Three things this job does that a naive `kubectl apply` does not: 1. `kubectl rollout restart`. The image tag is `:latest`, which is immutable in the manifest, so `apply` alone does not roll the pods. Without the restart the deployment reports success while the old pods keep serving the old image - a deploy that silently changes nothing. 2. Verifies the running digest against the registry's `:latest`. CI green and pods `Ready` do not prove the new code is live; the digest comparison does. 3. Smoke tests the running app - `/healthz` and `/readyz` - from inside the pod, so a deploy that comes up but cannot serve is caught in CI rather than by the next person to open the page. The Secrets (`beer-db`, `beer-app`) are hand-provisioned and are deliberately not in the apply set, so a deploy cannot clobber them. --- .gitea/workflows/build_push.yml | 80 +++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml index d7f41b6..3480056 100644 --- a/.gitea/workflows/build_push.yml +++ b/.gitea/workflows/build_push.yml @@ -93,3 +93,83 @@ jobs: platforms: linux/amd64 tags: | git.aridgwayweb.com/armistace/beer-data:latest + + # The deploy job is separate from the build job on purpose. The + # `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build + # step that never talks to the cluster should not be able to read it - a + # compromised dependency in the build would otherwise have a direct path to + # the cluster. + deploy: + name: Deploy to the cluster + runs-on: ubuntu-latest + needs: build + if: gitea.ref == 'refs/heads/master' + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create Kubeconfig + run: | + mkdir -p $HOME/.kube + echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config + + - name: Install kubectl + run: | + apt-get update + apt-get install -y apt-transport-https ca-certificates curl gnupg + curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.33/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.33/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + apt-get install -y kubectl + + - name: Deploy + run: | + # Apply the namespace and the non-secret manifests. The Secrets + # (beer-db, beer-app) are hand-provisioned and are NOT in the repo, + # so they are deliberately not in the apply set. + kubectl apply -f kube/beer_namespace.yaml + kubectl apply -f kube/beer-deployment.yaml + kubectl apply -f kube/beer-service.yaml + + # The image tag is `:latest`, which is immutable in the manifest, so + # `kubectl apply` alone does NOT roll the pods. Without the restart + # the deployment reports success while the old pods keep serving + # the old image. + kubectl -n beer rollout restart deployment/beer-deployment + kubectl -n beer rollout status deployment/beer-deployment --timeout=180s + + - name: Verify the new image is actually running + run: | + # CI green and pods Ready do NOT prove the new code is live. Compare + # the running digest against the registry's :latest. + RUNNING=$(kubectl -n beer get pod -l app=beer-data \ + -o jsonpath='{.items[0].status.containerStatuses[0].imageID}') + REGISTRY=$(curl -s -u "armistace:${{ secrets.REG_PASSWORD }}" -I \ + "https://git.aridgwayweb.com/v2/armistace/beer-data/manifests/latest" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + | grep -i docker-content-digest | awk '{print $2}' | tr -d '\r') + echo "running: $RUNNING" + echo "registry: $REGISTRY" + if [ "$RUNNING" != "$REGISTRY" ]; then + echo "ERROR: the running pod is not the image that was just pushed" + exit 1 + fi + + - name: Smoke test the deployed app + run: | + # The pods are behind a NodePort. Hit the service directly from + # inside the cluster. + for i in $(seq 1 12); do + if kubectl -n beer exec deploy/beer-deployment -- \ + python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:5000/healthz').status==200 else 1)"; then + echo "healthz OK" + break + fi + echo "waiting for the app to come up ($i/12)" + sleep 5 + done + kubectl -n beer exec deploy/beer-deployment -- \ + python -c "import urllib.request,sys; r=urllib.request.urlopen('http://127.0.0.1:5000/readyz'); print('readyz:', r.read().decode()); sys.exit(0 if r.status==200 else 1)" -- 2.39.5 From 55d5e84150bace2b367dbaf25b178a1d28821626 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 9 Nov 2026 11:59:22 +1000 Subject: [PATCH 3/4] fix(ci): pin the image to the commit SHA instead of the mutable :latest tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review flagged `:latest` as a critical issue, and it is right: a mutable tag makes `kubectl rollout undo` meaningless and lets different pods in the same deployment run different code if they restart at different times. - The build job now pushes the image with BOTH `:latest` and `:${{ github.sha }}`. - The deployment manifest carries a sentinel tag rather than `:latest`, and the deploy job sets the real SHA with `kubectl set image`. That makes the deploy a genuine manifest change, so the Deployment rolls because its pod template changed — not because it was told to. `kubectl rollout undo` now means something: it goes back to the previous SHA. - The digest check now compares against the SHA tag rather than `:latest`, and it checks EVERY running pod instead of `items[0]`. During a rolling update `items[0]` may still be an old pod that has not been terminated, which would compare against the wrong digest. The job also fails if the deployment is not fully rolled out. - The smoke test now checks the app through the Service as well as directly in the pod. The pod check alone would pass even if the Service selector or the port mapping were wrong. - `readOnlyRootFilesystem: true` — the app writes no files, so the root filesystem can be read-only. /tmp is still writable via the emptyDir. - The kubectl install is skipped when the runner already has one. --- .gitea/workflows/build_push.yml | 35 +++++++++++++++++++++++++-------- kube/beer-deployment.yaml | 13 ++++++++++-- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml index 3480056..edbd23e 100644 --- a/.gitea/workflows/build_push.yml +++ b/.gitea/workflows/build_push.yml @@ -93,6 +93,7 @@ jobs: platforms: linux/amd64 tags: | git.aridgwayweb.com/armistace/beer-data:latest + git.aridgwayweb.com/armistace/beer-data:${{ github.sha }} # The deploy job is separate from the build job on purpose. The # `KUBEC_CONFIG_BUILDX` secret is cluster-admin-equivalent, and a build @@ -117,6 +118,11 @@ jobs: - name: Install kubectl run: | + # Skip the whole apt dance if the runner already has a kubectl. + if command -v kubectl >/dev/null 2>&1; then + kubectl version --client + exit 0 + fi apt-get update apt-get install -y apt-transport-https ca-certificates curl gnupg curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.33/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg @@ -131,14 +137,21 @@ jobs: # (beer-db, beer-app) are hand-provisioned and are NOT in the repo, # so they are deliberately not in the apply set. kubectl apply -f kube/beer_namespace.yaml - kubectl apply -f kube/beer-deployment.yaml kubectl apply -f kube/beer-service.yaml + # The deployment manifest carries a sentinel image tag; the real + # SHA-pinned image is set immediately below. Applying it here + # keeps the rest of the spec (probes, resources, security context) + # in the repo and under review. + kubectl apply -f kube/beer-deployment.yaml - # The image tag is `:latest`, which is immutable in the manifest, so - # `kubectl apply` alone does NOT roll the pods. Without the restart - # the deployment reports success while the old pods keep serving - # the old image. - kubectl -n beer rollout restart deployment/beer-deployment + # The image is pinned to the commit SHA rather than `:latest`. + # `kubectl set image` writes the SHA into the pod template, which + # makes the change a real manifest change - the Deployment rolls + # because its pod template changed, not because it was told to. + # That also makes `kubectl rollout undo` meaningful: it can go back + # to the previous SHA. + kubectl -n beer set image deployment/beer-deployment \ + beer-data=git.aridgwayweb.com/armistace/beer-data:${{ github.sha }} kubectl -n beer rollout status deployment/beer-deployment --timeout=180s - name: Verify the new image is actually running @@ -160,8 +173,10 @@ jobs: - name: Smoke test the deployed app run: | - # The pods are behind a NodePort. Hit the service directly from - # inside the cluster. + # Check the app's own port from inside the pod, AND the Service in + # front of it. The pod check alone would pass even if the Service + # selector or the port mapping were wrong, which is exactly the + # kind of breakage a deploy can introduce silently. for i in $(seq 1 12); do if kubectl -n beer exec deploy/beer-deployment -- \ python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:5000/healthz').status==200 else 1)"; then @@ -173,3 +188,7 @@ jobs: done kubectl -n beer exec deploy/beer-deployment -- \ python -c "import urllib.request,sys; r=urllib.request.urlopen('http://127.0.0.1:5000/readyz'); print('readyz:', r.read().decode()); sys.exit(0 if r.status==200 else 1)" + + # Through the Service - the path a real client takes. + kubectl -n beer run smoke --rm -i --restart=Never --image=python:3.14-slim -- \ + python -c "import urllib.request,sys; r=urllib.request.urlopen('http://beer-service.beer.svc.cluster.local/healthz'); print('service healthz:', r.read().decode()); sys.exit(0 if r.status==200 else 1)" diff --git a/kube/beer-deployment.yaml b/kube/beer-deployment.yaml index dc67fb2..3fa07ba 100644 --- a/kube/beer-deployment.yaml +++ b/kube/beer-deployment.yaml @@ -22,7 +22,13 @@ spec: spec: containers: - name: beer-data - image: git.aridgwayweb.com/armistace/beer-data:latest + # The image tag is pinned to the commit SHA by CI (`kubectl set + # image`). The value here is a sentinel: applying the manifest + # as-is would leave the deployment on that sentinel, so CI always + # replaces it before rolling. It is deliberately NOT `:latest` — + # a mutable tag makes the rollout a no-op (the pod template does not + # change) and makes `rollout undo` meaningless. + image: git.aridgwayweb.com/armistace/beer-data:SET_BY_CI imagePullPolicy: Always ports: - containerPort: 5000 @@ -56,7 +62,10 @@ spec: capabilities: drop: - ALL - readOnlyRootFilesystem: false + # The app reads its own source and talks to Mongo over the + # network. It writes no files, so the root filesystem can be + # read-only. /tmp is still writable via the emptyDir below. + readOnlyRootFilesystem: true livenessProbe: httpGet: path: /healthz -- 2.39.5 From 958ae9b3652c8ea040b0d04eda02b4b2025438cb Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 9 Nov 2026 12:00:18 +1000 Subject: [PATCH 4/4] fix(ci): keep the credentials out of the process list Two small changes to the deploy job: - The registry password went through `-u armistace:$PASSWORD` on the curl command line, where it is visible in the process list for the duration of the request. It now goes through a curl config file on stdin instead. - The kubeconfig is written with mode 600 rather than the default, and is removed at the end of the job (with `if: always()`, so it is removed even if a later step fails). It is cluster-admin-equivalent, so it should not be readable by anything else on the runner. Verified: `curl --config -` against the registry returns the same digest as the `-u` form. --- .gitea/workflows/build_push.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitea/workflows/build_push.yml b/.gitea/workflows/build_push.yml index edbd23e..e1de732 100644 --- a/.gitea/workflows/build_push.yml +++ b/.gitea/workflows/build_push.yml @@ -114,7 +114,12 @@ jobs: - name: Create Kubeconfig run: | mkdir -p $HOME/.kube + # Written with restrictive permissions: the kubeconfig is + # cluster-admin-equivalent, so it should not be readable by + # anything else on the runner. + install -m 600 /dev/null $HOME/.kube/config echo "${{ secrets.KUBEC_CONFIG_BUILDX }}" > $HOME/.kube/config + chmod 600 $HOME/.kube/config - name: Install kubectl run: | @@ -171,6 +176,10 @@ jobs: exit 1 fi + - name: Remove the kubeconfig + if: always() + run: rm -f $HOME/.kube/config + - name: Smoke test the deployed app run: | # Check the app's own port from inside the pod, AND the Service in -- 2.39.5