fix(deploy): fix silent success on failed deploys (CON-654)

- Fix boolean type coercion: use !inputs.use-env-tag instead of == false
- Fail loudly (exit 1) when deploy produces no changes instead of exit 0
- Add debug logging: image name, tag, deploy paths, current/updated images
- Validate yq actually matched and updated container images before committing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitea/workflows/build-push.yaml

@@ -29,8 +29,8 @@ on:
         required: false
         type: string
         default: ""
-      target:
-        description: "Docker build target stage (for multi-stage builds)"
+      version:
+        description: "Optional version for pinned dev tags (e.g. '123' produces 'dev-v123'). Only applies on dev branch."
         required: false
         type: string
         default: ""
@@ -39,16 +39,6 @@ on:
         required: false
         type: string
         default: "ubuntu-latest"
-      environment-tag:
-        description: "Mutable environment tag to push (e.g. prod, staging, dev). If empty, auto-detected from branch: main/master→prod, staging→staging, dev→dev."
-        required: false
-        type: string
-        default: ""
-      version-tag:
-        description: "Pinned version tag to push (e.g. dev-v123). Used for node-specific testing."
-        required: false
-        type: string
-        default: ""
     secrets:
       REGISTRY_USER:
         required: true
@@ -58,6 +48,9 @@ on:
       image-tag:
         description: "The sha-based image tag that was pushed"
         value: ${{ jobs.build.outputs.image-tag }}
+      env-tag:
+        description: "The environment tag that was pushed (prod, staging, dev, or dev-v{N})"
+        value: ${{ jobs.build.outputs.env-tag }}
 
 jobs:
   build:
@@ -68,16 +61,40 @@ jobs:
       options: --privileged
     outputs:
       image-tag: ${{ steps.tag.outputs.tag }}
+      env-tag: ${{ steps.tag.outputs.env-tag }}
     steps:
       - uses: actions/checkout@v4
 
       - name: Compute image tag
         id: tag
+        shell: bash
         run: |
           SHORT_SHA="${{ gitea.sha }}"
           SHORT_SHA="${SHORT_SHA:0:7}"
           echo "tag=sha-${SHORT_SHA}" >> "$GITEA_OUTPUT"
 
+          # Determine environment tag from branch
+          REF="${{ gitea.ref }}"
+          VERSION="${{ inputs.version }}"
+          case "$REF" in
+            refs/heads/main|refs/heads/master)
+              echo "env-tag=prod" >> "$GITEA_OUTPUT"
+              ;;
+            refs/heads/staging)
+              echo "env-tag=staging" >> "$GITEA_OUTPUT"
+              ;;
+            refs/heads/dev)
+              if [ -n "$VERSION" ]; then
+                echo "env-tag=dev-v${VERSION}" >> "$GITEA_OUTPUT"
+              else
+                echo "env-tag=dev" >> "$GITEA_OUTPUT"
+              fi
+              ;;
+            *)
+              echo "env-tag=" >> "$GITEA_OUTPUT"
+              ;;
+          esac
+
       - name: Log in to Gitea registry
         uses: docker/login-action@v3
         with:
@@ -92,11 +109,8 @@ jobs:
           images: ${{ inputs.image-name }}
           tags: |
             type=sha,prefix=sha-
-            type=raw,value=prod,enable=${{ gitea.ref == 'refs/heads/main' || gitea.ref == 'refs/heads/master' }}
-            type=raw,value=staging,enable=${{ gitea.ref == 'refs/heads/staging' }}
-            type=raw,value=dev,enable=${{ gitea.ref == 'refs/heads/dev' }}
-            type=raw,value=${{ inputs.environment-tag }},enable=${{ inputs.environment-tag != '' }}
-            type=raw,value=${{ inputs.version-tag }},enable=${{ inputs.version-tag != '' }}
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=${{ steps.tag.outputs.env-tag }},enable=${{ steps.tag.outputs.env-tag != '' }}
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
@@ -108,7 +122,6 @@ jobs:
         with:
           context: ${{ inputs.context }}
           file: ${{ inputs.context }}/${{ inputs.dockerfile }}
-          target: ${{ inputs.target || '' }}
           platforms: ${{ inputs.platforms }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}

.gitea/workflows/deploy-k8s.yaml

@@ -1,17 +1,12 @@
-# Reusable workflow: Bump deploy-timestamp in wectrl-k8s-cluster repo (GitOps deploy)
+# Reusable workflow: Update image tag in wectrl-k8s-cluster repo (GitOps deploy)
 # Usage: uses: wectrl-net/ci-templates/.gitea/workflows/deploy-k8s.yaml@main
-#
-# With environment-based mutable tags (:prod, :staging, :dev), the image tag in
-# k8s manifests stays constant. To trigger a rollout after pushing a new image,
-# this workflow bumps the deploy-timestamp pod annotation, which causes ArgoCD to
-# detect a diff and sync the deployment.
 name: Deploy to K8s (GitOps)
 
 on:
   workflow_call:
     inputs:
       image-name:
-        description: "Full image name (e.g. git.wectrl.net/wectrl-net/my-service) — used in commit message for traceability"
+        description: "Full image name (e.g. git.wectrl.net/wectrl-net/my-service)"
         required: true
         type: string
       deploy-paths:
@@ -27,6 +22,11 @@ on:
         required: false
         type: string
         default: "wectrl-net/wectrl-k8s-cluster"
+      use-env-tag:
+        description: "If true, skip SHA-based manifest update (service uses mutable env tags with imagePullPolicy: Always)"
+        required: false
+        type: boolean
+        default: false
     secrets:
       GIT_USER:
         required: true
@@ -37,23 +37,129 @@ jobs:
   deploy:
     name: Update k8s Cluster Repo
     runs-on: ubuntu-latest
-    if: gitea.ref == 'refs/heads/main'
+    if: gitea.ref == 'refs/heads/main' && !inputs.use-env-tag
     steps:
       - name: Install yq
+        shell: bash
         run: |
           set -euo pipefail
-          wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64
+          ARCH=$(uname -m)
+          case "$ARCH" in
+            aarch64|arm64) YQ_ARCH="arm64" ;;
+            x86_64|amd64)  YQ_ARCH="amd64" ;;
+            *)             YQ_ARCH="amd64" ;;
+          esac
+          wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_${YQ_ARCH}"
           chmod +x /usr/local/bin/yq
 
-      - name: Bump deploy-timestamp in cluster repo
+      - name: Update image tag in cluster repo
+        shell: bash
         run: |
           set -euo pipefail
 
           SHA="${{ gitea.sha }}"
           SHORT_SHA="${SHA:0:7}"
-          TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+          IMAGE_TAG="sha-${SHORT_SHA}"
           RUN_URL="${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}"
 
+          echo "=== Deploy parameters ==="
+          echo "Image: ${{ inputs.image-name }}"
+          echo "Tag: ${IMAGE_TAG}"
+          echo "Deploy paths: ${{ inputs.deploy-paths }}"
+          echo "K8s repo: ${{ inputs.k8s-repo }}"
+          echo "========================="
+
+          git clone https://${{ secrets.GIT_USER }}:${{ secrets.GIT_TOKEN }}@git.wectrl.net/${{ inputs.k8s-repo }}.git
+          cd wectrl-k8s-cluster
+
+          git config user.email "ci@wectrl.net"
+          git config user.name "Gitea CI"
+
+          MATCHED=0
+          for DEPLOY_PATH in ${{ inputs.deploy-paths }}; do
+            echo "Processing: ${DEPLOY_PATH}"
+
+            # Show current image before update
+            CURRENT_IMAGE=$(yq '.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))' "${DEPLOY_PATH}" 2>/dev/null || true)
+            if [ -z "${CURRENT_IMAGE}" ]; then
+              echo "ERROR: No container image matching '${{ inputs.image-name }}:*' found in ${DEPLOY_PATH}"
+              echo "Images in file:"
+              yq '.spec.template.spec.containers[].image' "${DEPLOY_PATH}" || true
+              exit 1
+            fi
+            echo "  Current: ${CURRENT_IMAGE}"
+
+            yq -i '(.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))) = "${{ inputs.image-name }}:'"${IMAGE_TAG}"'"' \
+              "${DEPLOY_PATH}"
+
+            # Verify the update took effect
+            UPDATED_IMAGE=$(yq '.spec.template.spec.containers[].image | select(test("^${{ inputs.image-name }}:"))' "${DEPLOY_PATH}")
+            echo "  Updated: ${UPDATED_IMAGE}"
+            if [ "${UPDATED_IMAGE}" != "${{ inputs.image-name }}:${IMAGE_TAG}" ]; then
+              echo "ERROR: Image tag update failed for ${DEPLOY_PATH}"
+              exit 1
+            fi
+
+            MATCHED=$((MATCHED + 1))
+            git add "${DEPLOY_PATH}"
+          done
+
+          echo "Successfully updated ${MATCHED} manifest(s)"
+
+          if git diff --staged --quiet; then
+            echo "ERROR: No image tag changes to commit — manifests may already be at ${IMAGE_TAG}"
+            exit 1
+          fi
+
+          git commit -m "deploy: ${{ inputs.service-name }} ${IMAGE_TAG}"
+          git push origin main
+          echo "Cluster repo updated — ArgoCD will sync within ~3 min"
+
+  # When using mutable env tags, bump a deploy annotation so ArgoCD detects the change
+  notify-argocd:
+    name: Bump deploy annotation
+    runs-on: ubuntu-latest
+    if: inputs.use-env-tag
+    steps:
+      - name: Install yq
+        shell: bash
+        run: |
+          set -euo pipefail
+          ARCH=$(uname -m)
+          case "$ARCH" in
+            aarch64|arm64) YQ_ARCH="arm64" ;;
+            x86_64|amd64)  YQ_ARCH="amd64" ;;
+            *)             YQ_ARCH="amd64" ;;
+          esac
+          wget -qO /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_${YQ_ARCH}"
+          chmod +x /usr/local/bin/yq
+
+      - name: Bump annotation in cluster repo
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          SHA="${{ gitea.sha }}"
+          SHORT_SHA="${SHA:0:7}"
+          TIMESTAMP="$(date -u +%Y%m%d%H%M%S)"
+          RUN_URL="${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}"
+
+          # Determine env from branch
+          REF="${{ gitea.ref }}"
+          case "$REF" in
+            refs/heads/main|refs/heads/master) ENV_TAG="prod" ;;
+            refs/heads/staging) ENV_TAG="staging" ;;
+            refs/heads/dev) ENV_TAG="dev" ;;
+            *) echo "Branch not mapped to environment, skipping"; exit 0 ;;
+          esac
+
+          echo "=== Deploy parameters ==="
+          echo "Image: ${{ inputs.image-name }}"
+          echo "SHA: sha-${SHORT_SHA}"
+          echo "Env: ${ENV_TAG}"
+          echo "Deploy paths: ${{ inputs.deploy-paths }}"
+          echo "========================="
+
           git clone https://${{ secrets.GIT_USER }}:${{ secrets.GIT_TOKEN }}@git.wectrl.net/${{ inputs.k8s-repo }}.git
           cd wectrl-k8s-cluster
 
@@ -61,19 +167,17 @@ jobs:
           git config user.name "Gitea CI"
 
           for DEPLOY_PATH in ${{ inputs.deploy-paths }}; do
-            yq -i '.spec.template.metadata.annotations.deploy-timestamp = "'"${TIMESTAMP}"'"' \
+            echo "Processing: ${DEPLOY_PATH}"
+            yq -i '.spec.template.metadata.annotations."deploy.wectrl.net/restart" = "'"${TIMESTAMP}-sha-${SHORT_SHA}"'"' \
               "${DEPLOY_PATH}"
             git add "${DEPLOY_PATH}"
           done
 
           if git diff --staged --quiet; then
-            echo "No changes to commit"
-          else
-            git commit -m "deploy: ${{ inputs.service-name }} sha-${SHORT_SHA}
-
-Timestamp: ${TIMESTAMP}
-Source: ${SHA}
-Run: ${RUN_URL}"
-            git push origin main
-            echo "Cluster repo updated — ArgoCD will sync within ~3 min"
+            echo "ERROR: No annotation changes to commit — this should not happen"
+            exit 1
           fi
+
+          git commit -m "deploy: ${{ inputs.service-name }} ${ENV_TAG} (sha-${SHORT_SHA})"
+          git push origin main
+          echo "Annotation bumped — ArgoCD will roll pods within ~3 min"