fix(deploy): fix silent success on failed deploys (CON-654)

- Fix boolean type coercion: use !inputs.use-env-tag instead of == false
- Fail loudly (exit 1) when deploy produces no changes instead of exit 0
- Add debug logging: image name, tag, deploy paths, current/updated images
- Validate yq actually matched and updated container images before committing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitea/workflows/build-push.yaml

@@ -29,6 +29,11 @@ on:
         required: false
         type: string
         default: ""
+      version:
+        description: "Optional version for pinned dev tags (e.g. '123' produces 'dev-v123'). Only applies on dev branch."
+        required: false
+        type: string
+        default: ""
       runner:
         description: "Runner label to use (e.g. ubuntu-latest, self-hosted-arm64). ARM64 builds use QEMU emulation on amd64 runners by default — set this to a native ARM64 runner for faster Rust/heavy builds."
         required: false
@@ -43,6 +48,9 @@ on:
       image-tag:
         description: "The sha-based image tag that was pushed"
         value: ${{ jobs.build.outputs.image-tag }}
+      env-tag:
+        description: "The environment tag that was pushed (prod, staging, dev, or dev-v{N})"
+        value: ${{ jobs.build.outputs.env-tag }}
 
 jobs:
   build:
@@ -53,16 +61,40 @@ jobs:
       options: --privileged
     outputs:
       image-tag: ${{ steps.tag.outputs.tag }}
+      env-tag: ${{ steps.tag.outputs.env-tag }}
     steps:
       - uses: actions/checkout@v4
 
       - name: Compute image tag
         id: tag
+        shell: bash
         run: |
           SHORT_SHA="${{ gitea.sha }}"
           SHORT_SHA="${SHORT_SHA:0:7}"
           echo "tag=sha-${SHORT_SHA}" >> "$GITEA_OUTPUT"
 
+          # Determine environment tag from branch
+          REF="${{ gitea.ref }}"
+          VERSION="${{ inputs.version }}"
+          case "$REF" in
+            refs/heads/main|refs/heads/master)
+              echo "env-tag=prod" >> "$GITEA_OUTPUT"
+              ;;
+            refs/heads/staging)
+              echo "env-tag=staging" >> "$GITEA_OUTPUT"
+              ;;
+            refs/heads/dev)
+              if [ -n "$VERSION" ]; then
+                echo "env-tag=dev-v${VERSION}" >> "$GITEA_OUTPUT"
+              else
+                echo "env-tag=dev" >> "$GITEA_OUTPUT"
+              fi
+              ;;
+            *)
+              echo "env-tag=" >> "$GITEA_OUTPUT"
+              ;;
+          esac
+
       - name: Log in to Gitea registry
         uses: docker/login-action@v3
         with:
@@ -78,6 +110,7 @@ jobs:
           tags: |
             type=sha,prefix=sha-
             type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=${{ steps.tag.outputs.env-tag }},enable=${{ steps.tag.outputs.env-tag != '' }}
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 

.gitea/workflows/deploy-k8s.yaml

@@ -22,6 +22,11 @@ on:
         required: false
         type: string
         default: "wectrl-net/wectrl-k8s-cluster"
+      use-env-tag:
+        description: "If true, skip SHA-based manifest update (service uses mutable env tags with imagePullPolicy: Always)"
+        required: false
+        type: boolean
+        default: false
     secrets:
       GIT_USER:
         required: true
@@ -32,15 +37,23 @@ jobs:
   deploy:
     name: Update k8s Cluster Repo
     runs-on: ubuntu-latest
-    if: gitea.ref == 'refs/heads/main'
+    if: gitea.ref == 'refs/heads/main' && !inputs.use-env-tag
     steps:
       - name: Install yq
+        shell: bash
         run: |
           set -euo pipefail
-          wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64
+          ARCH=$(uname -m)
+          case "$ARCH" in
+            aarch64|arm64) YQ_ARCH="arm64" ;;
+            x86_64|amd64)  YQ_ARCH="amd64" ;;
+            *)             YQ_ARCH="amd64" ;;
+          esac
+          wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_${YQ_ARCH}"
           chmod +x /usr/local/bin/yq
 
       - name: Update image tag in cluster repo
+        shell: bash
         run: |
           set -euo pipefail
 
@@ -49,6 +62,104 @@ jobs:
           IMAGE_TAG="sha-${SHORT_SHA}"
           RUN_URL="${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}"
 
+          echo "=== Deploy parameters ==="
+          echo "Image: ${{ inputs.image-name }}"
+          echo "Tag: ${IMAGE_TAG}"
+          echo "Deploy paths: ${{ inputs.deploy-paths }}"
+          echo "K8s repo: ${{ inputs.k8s-repo }}"
+          echo "========================="
+
+          git clone https://${{ secrets.GIT_USER }}:${{ secrets.GIT_TOKEN }}@git.wectrl.net/${{ inputs.k8s-repo }}.git
+          cd wectrl-k8s-cluster
+
+          git config user.email "ci@wectrl.net"
+          git config user.name "Gitea CI"
+
+          MATCHED=0
+          for DEPLOY_PATH in ${{ inputs.deploy-paths }}; do
+            echo "Processing: ${DEPLOY_PATH}"
+
+            # Show current image before update
+            CURRENT_IMAGE=$(yq '.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))' "${DEPLOY_PATH}" 2>/dev/null || true)
+            if [ -z "${CURRENT_IMAGE}" ]; then
+              echo "ERROR: No container image matching '${{ inputs.image-name }}:*' found in ${DEPLOY_PATH}"
+              echo "Images in file:"
+              yq '.spec.template.spec.containers[].image' "${DEPLOY_PATH}" || true
+              exit 1
+            fi
+            echo "  Current: ${CURRENT_IMAGE}"
+
+            yq -i '(.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))) = "${{ inputs.image-name }}:'"${IMAGE_TAG}"'"' \
+              "${DEPLOY_PATH}"
+
+            # Verify the update took effect
+            UPDATED_IMAGE=$(yq '.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))' "${DEPLOY_PATH}")
+            echo "  Updated: ${UPDATED_IMAGE}"
+            if [ "${UPDATED_IMAGE}" != "${{ inputs.image-name }}:${IMAGE_TAG}" ]; then
+              echo "ERROR: Image tag update failed for ${DEPLOY_PATH}"
+              exit 1
+            fi
+
+            MATCHED=$((MATCHED + 1))
+            git add "${DEPLOY_PATH}"
+          done
+
+          echo "Successfully updated ${MATCHED} manifest(s)"
+
+          if git diff --staged --quiet; then
+            echo "ERROR: No image tag changes to commit — manifests may already be at ${IMAGE_TAG}"
+            exit 1
+          fi
+
+          git commit -m "deploy: ${{ inputs.service-name }} ${IMAGE_TAG}"
+          git push origin main
+          echo "Cluster repo updated — ArgoCD will sync within ~3 min"
+
+  # When using mutable env tags, bump a deploy annotation so ArgoCD detects the change
+  notify-argocd:
+    name: Bump deploy annotation
+    runs-on: ubuntu-latest
+    if: inputs.use-env-tag
+    steps:
+      - name: Install yq
+        shell: bash
+        run: |
+          set -euo pipefail
+          ARCH=$(uname -m)
+          case "$ARCH" in
+            aarch64|arm64) YQ_ARCH="arm64" ;;
+            x86_64|amd64)  YQ_ARCH="amd64" ;;
+            *)             YQ_ARCH="amd64" ;;
+          esac
+          wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_${YQ_ARCH}"
+          chmod +x /usr/local/bin/yq
+
+      - name: Bump annotation in cluster repo
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          SHA="${{ gitea.sha }}"
+          SHORT_SHA="${SHA:0:7}"
+          TIMESTAMP="$(date -u +%Y%m%d%H%M%S)"
+          RUN_URL="${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}"
+
+          # Determine env from branch
+          REF="${{ gitea.ref }}"
+          case "$REF" in
+            refs/heads/main|refs/heads/master) ENV_TAG="prod" ;;
+            refs/heads/staging) ENV_TAG="staging" ;;
+            refs/heads/dev) ENV_TAG="dev" ;;
+            *) echo "Branch not mapped to environment, skipping"; exit 0 ;;
+          esac
+
+          echo "=== Deploy parameters ==="
+          echo "Image: ${{ inputs.image-name }}"
+          echo "SHA: sha-${SHORT_SHA}"
+          echo "Env: ${ENV_TAG}"
+          echo "Deploy paths: ${{ inputs.deploy-paths }}"
+          echo "========================="
+
           git clone https://${{ secrets.GIT_USER }}:${{ secrets.GIT_TOKEN }}@git.wectrl.net/${{ inputs.k8s-repo }}.git
           cd wectrl-k8s-cluster
 
@@ -56,18 +167,17 @@ jobs:
           git config user.name "Gitea CI"
 
           for DEPLOY_PATH in ${{ inputs.deploy-paths }}; do
-            yq -i '(.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))) = "${{ inputs.image-name }}:'"${IMAGE_TAG}"'"' \
+            echo "Processing: ${DEPLOY_PATH}"
+            yq -i '.spec.template.metadata.annotations."deploy.wectrl.net/restart" = "'"${TIMESTAMP}-sha-${SHORT_SHA}"'"' \
               "${DEPLOY_PATH}"
             git add "${DEPLOY_PATH}"
           done
 
           if git diff --staged --quiet; then
-            echo "No image tag changes to commit"
+            echo "ERROR: No annotation changes to commit — this should not happen"
-          else
+            exit 1
-            git commit -m "deploy: ${{ inputs.service-name }} ${IMAGE_TAG}
-
-Source: ${SHA}
-Run: ${RUN_URL}"
-            git push origin main
-            echo "Cluster repo updated — ArgoCD will sync within ~3 min"
           fi
+
+          git commit -m "deploy: ${{ inputs.service-name }} ${ENV_TAG} (sha-${SHORT_SHA})"
+          git push origin main
+          echo "Annotation bumped — ArgoCD will roll pods within ~3 min"