personal-lifecycle-tracker / .github / workflows / ci.yml
ci.yml
Raw
name: CI/CD Pipeline

on:
  push:
    branches: [main, dev, develop]
  pull_request:
    branches: [main, dev]
  workflow_dispatch:
    inputs:
      deploy_to_render:
        description: 'Deploy to Render after test passage'
        required: false
        type: boolean
        default: true
      target_environment:
        description: 'Target Deployment Environment'
        required: false
        type: choice
        options:
          - production
          - dev
        default: production
      render_health_url:
        description: 'Custom Health Endpoint URL to validate'
        required: false
        type: string
        default: ''

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  # ===========================================================================
  # 1. PARALLEL TESTS: Backend Unit & Contract Tests
  # ===========================================================================
  test-backend:
    name: Backend Tests (FastAPI & Pytest)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Ensure Docker Network Exists
        run: |
                    docker network inspect personal-lifecycle-observability-net >/dev/null 2>&1 || docker network create personal-lifecycle-observability-net

      - name: Build Backend Image
        run: |
                    docker compose build backend

      - name: Run Backend Unit Tests (Pytest)
        run: |
                    docker compose run --rm -T --no-deps backend pytest tests -m "not integration" -v

  # ===========================================================================
  # 1. PARALLEL TESTS: Frontend Lint & Vitest Suite
  # ===========================================================================
  test-frontend:
    name: Frontend Tests (React & Vitest)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Ensure Docker Network Exists
        run: |
                    docker network inspect personal-lifecycle-observability-net >/dev/null 2>&1 || docker network create personal-lifecycle-observability-net

      - name: Build Frontend Image
        run: |
                    docker compose build frontend

      - name: Run Frontend Lint & Vitest Unit Tests
        run: |
                    docker compose run --rm -T --no-deps frontend sh -c "npm run lint && npm test"

  # ===========================================================================
  # 1. PARALLEL TESTS: Deterministic Security Audit (Semgrep SAST & npm audit)
  # ===========================================================================
  security-audit:
    name: Security Audit (Semgrep SAST & Audit)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Ensure Docker Network Exists
        run: |
                    docker network inspect personal-lifecycle-observability-net >/dev/null 2>&1 || docker network create personal-lifecycle-observability-net

      - name: Build Frontend Image
        run: |
                    docker compose build frontend

      - name: Run Semgrep OSS SAST Analysis
        run: |
                    docker run --rm -v ${{ github.workspace }}:/src returntocorp/semgrep semgrep --config=p/python --config=p/secrets --config=p/jwt --error /src/backend

      - name: Run Frontend Security Audit
        run: |
                    docker compose run --rm -T --no-deps frontend npm audit --audit-level=high

  # ===========================================================================
  # 2. DOCKER COMPOSE STACK: Integration & End-to-End Tests
  # ===========================================================================
  integration-and-e2e:
    name: Compose Stack (Integration & E2E Tests)
    needs: [test-backend, test-frontend, security-audit]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Ensure Docker Network Exists
        run: |
                    docker network inspect personal-lifecycle-observability-net >/dev/null 2>&1 || docker network create personal-lifecycle-observability-net

      - name: Build Docker Compose Services (including test profile)
        run: |
                    docker compose --profile test build

      - name: Start Compose Services (Database, Backend, Frontend)
        run: |
                    docker compose up -d db backend frontend

      - name: Wait for Stack Healthiness
        run: |
          echo "Waiting for backend health endpoint (http://127.0.0.1:8003/health)..."
          timeout 90s bash -c 'until curl -fsS http://127.0.0.1:8003/health > /dev/null 2>&1; do sleep 2; done'
          echo "Backend is healthy!"
          curl -s http://127.0.0.1:8003/health

          echo "Waiting for frontend endpoint (http://127.0.0.1:3003)..."
          timeout 90s bash -c 'until curl -fsS http://127.0.0.1:3003 > /dev/null 2>&1; do sleep 2; done'
          echo "Frontend is healthy!"          

      - name: Run PostgreSQL Integration Tests
        run: |
                    docker compose run --rm -T -e INTEGRATION_BACKEND_URL=http://personal-lifecycle-tracker-backend:8000 backend pytest tests/integration -v

      - name: Run Playwright End-to-End Tests
        env:
          BASE_URL: http://127.0.0.1:3003
          BACKEND_URL: http://127.0.0.1:8003
        run: |
                    docker compose up --exit-code-from e2e e2e

      - name: Dump Compose Logs on Failure
        if: failure()
        run: |
          docker compose logs --tail 100 backend
          docker compose logs --tail 100 frontend
          docker compose logs --tail 100 db
          docker compose logs --tail 100 e2e          

      - name: Tear Down Compose Stack
        if: always()
        run: |
                    docker compose down -v

  # ===========================================================================
  # 3. BUILD STAGE: Build Images & Push to Container Registry (ECR)
  # ===========================================================================
  build-and-push-ecr:
    name: Build & Push to Registry (ECR)
    needs: [integration-and-e2e]
    if: |
      ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/develop') && github.event_name != 'pull_request') ||
      github.event_name == 'workflow_dispatch'      
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write # Required for GitHub OIDC role assumption
    outputs:
      image_tag: ${{ steps.generate-tag.outputs.image_tag }}
      short_sha: ${{ steps.generate-tag.outputs.short_sha }}
      timestamp: ${{ steps.generate-tag.outputs.timestamp }}
      backend_image: ${{ steps.build-and-push.outputs.backend_image }}
      frontend_image: ${{ steps.build-and-push.outputs.frontend_image }}
      target_env: ${{ steps.resolve-env.outputs.target_env }}
      ecr_registry: ${{ steps.build-and-push.outputs.ecr_registry }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Resolve Target Environment
        id: resolve-env
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.target_environment }}" ]; then
            TARGET="${{ inputs.target_environment }}"
          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
            TARGET="production"
          else
            TARGET="dev"
          fi
          echo "target_env=$TARGET" >> $GITHUB_OUTPUT
          echo "TARGET_ENV=$TARGET" >> $GITHUB_ENV
          echo "=== Target Environment: $TARGET ==="          

      # Generate unique image tag using pattern YYYYMMDD-HHMMSS-shortsha
      - name: Generate Image Tag (YYYYMMDD-HHMMSS-shortsha)
        id: generate-tag
        run: |
          SHORT_SHA=$(git rev-parse --short=7 HEAD)
          TIMESTAMP=$(date -u +'%Y%m%d-%H%M%S')
          IMAGE_TAG="${TIMESTAMP}-${SHORT_SHA}"
          echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
          echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
          echo "timestamp=$TIMESTAMP" >> $GITHUB_OUTPUT
          echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV
          echo "================================================================"
          echo "Generated Image Tag: $IMAGE_TAG"
          echo "Format: YYYYMMDD-HHMMSS-shortsha (${TIMESTAMP}-${SHORT_SHA})"
          echo "================================================================"          

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      # Assume IAM Role using GitHub OIDC (Keyless Cloud Authentication)
      - name: Authenticate via GitHub OIDC Role
        id: oidc-auth
        continue-on-error: true
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME || secrets.RENDER_OIDC_ROLE_ARN }}
          aws-region: ${{ secrets.AWS_REGION || 'us-east-1' }}
          role-session-name: GitHubActionsECRBuild-${{ github.run_id }}
        env:
          RENDER_OIDC_ROLE_ARN: ${{ secrets.RENDER_OIDC_ROLE_ARN }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}

      # Log in to Amazon ECR
      - name: Log in to Amazon ECR
        id: login-ecr
        if: steps.oidc-auth.outcome == 'success'
        continue-on-error: true
        uses: aws-actions/amazon-ecr-login@v2

      # Build and Push Container Images to ECR
      - name: Build & Push Images to ECR
        id: build-and-push
        run: |
          TAG="${{ steps.generate-tag.outputs.image_tag }}"
          TARGET="${{ steps.resolve-env.outputs.target_env }}"
          REGISTRY="${{ steps.login-ecr.outputs.registry }}"
          BACKEND_REPO="${{ secrets.ECR_BACKEND_REPOSITORY || 'personal-lifecycle-tracker-backend' }}"
          FRONTEND_REPO="${{ secrets.ECR_FRONTEND_REPOSITORY || 'personal-lifecycle-tracker-frontend' }}"

          if [ -n "$REGISTRY" ]; then
            echo "Amazon ECR Registry authenticated: $REGISTRY"
            # Auto-create repositories if they don't already exist
            aws ecr describe-repositories --repository-names "$BACKEND_REPO" --region "${{ secrets.AWS_REGION || 'us-east-1' }}" >/dev/null 2>&1 || \
              aws ecr create-repository --repository-name "$BACKEND_REPO" --region "${{ secrets.AWS_REGION || 'us-east-1' }}" >/dev/null 2>&1 || true

            aws ecr describe-repositories --repository-names "$FRONTEND_REPO" --region "${{ secrets.AWS_REGION || 'us-east-1' }}" >/dev/null 2>&1 || \
              aws ecr create-repository --repository-name "$FRONTEND_REPO" --region "${{ secrets.AWS_REGION || 'us-east-1' }}" >/dev/null 2>&1 || true

            BACKEND_IMAGE="$REGISTRY/$BACKEND_REPO:$TAG"
            FRONTEND_IMAGE="$REGISTRY/$FRONTEND_REPO:$TAG"
          else
            echo "Notice: Operating without remote ECR registry; building and tagging images locally."
            REGISTRY="local-registry"
            BACKEND_IMAGE="personal-lifecycle-tracker-backend:$TAG"
            FRONTEND_IMAGE="personal-lifecycle-tracker-frontend:$TAG"
          fi

          echo "----------------------------------------------------------------"
          echo "Building Backend Image: $BACKEND_IMAGE"
          echo "----------------------------------------------------------------"
          docker build \
            --target base \
            -t "$BACKEND_IMAGE" \
            -t "$REGISTRY/$BACKEND_REPO:${TARGET}-latest" \
            -t "$REGISTRY/$BACKEND_REPO:latest" \
            ./backend

          echo "----------------------------------------------------------------"
          echo "Building Frontend Image: $FRONTEND_IMAGE"
          echo "----------------------------------------------------------------"
          docker build \
            --target prod \
            -t "$FRONTEND_IMAGE" \
            -t "$REGISTRY/$FRONTEND_REPO:${TARGET}-latest" \
            -t "$REGISTRY/$FRONTEND_REPO:latest" \
            ./frontend

          if [ -n "${{ steps.login-ecr.outputs.registry }}" ]; then
            echo "----------------------------------------------------------------"
            echo "Pushing Backend Images to Amazon ECR..."
            echo "----------------------------------------------------------------"
            docker push "$BACKEND_IMAGE"
            docker push "$REGISTRY/$BACKEND_REPO:${TARGET}-latest"
            docker push "$REGISTRY/$BACKEND_REPO:latest"

            echo "----------------------------------------------------------------"
            echo "Pushing Frontend Images to Amazon ECR..."
            echo "----------------------------------------------------------------"
            docker push "$FRONTEND_IMAGE"
            docker push "$REGISTRY/$FRONTEND_REPO:${TARGET}-latest"
            docker push "$REGISTRY/$FRONTEND_REPO:latest"

            # Record deployed image metadata in AWS SSM Parameter Store
            echo "Recording deployed image tag in AWS SSM Parameter Store..."
            aws ssm put-parameter --name "/personal-lifecycle/${TARGET}/image_tag" --value "$TAG" --type String --overwrite || true
            aws ssm put-parameter --name "/personal-lifecycle/${TARGET}/backend_image" --value "$BACKEND_IMAGE" --type String --overwrite || true
            aws ssm put-parameter --name "/personal-lifecycle/${TARGET}/frontend_image" --value "$FRONTEND_IMAGE" --type String --overwrite || true
          fi

          echo "backend_image=$BACKEND_IMAGE" >> $GITHUB_OUTPUT
          echo "frontend_image=$FRONTEND_IMAGE" >> $GITHUB_OUTPUT
          echo "ecr_registry=$REGISTRY" >> $GITHUB_OUTPUT          

      - name: Summarize Build Stage
        run: |
          TAG="${{ steps.generate-tag.outputs.image_tag }}"
          TARGET="${{ steps.resolve-env.outputs.target_env }}"
          echo "### ๐Ÿ“ฆ Build Stage Summary (ECR)" >> $GITHUB_STEP_SUMMARY
          echo "- **Target Environment**: \`$TARGET\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Image Tag Pattern**: \`YYYYMMDD-HHMMSS-shortsha\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Generated Tag**: \`$TAG\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Backend Image**: \`${{ steps.build-and-push.outputs.backend_image }}\`" >> $GITHUB_STEP_SUMMARY
          echo "- **Frontend Image**: \`${{ steps.build-and-push.outputs.frontend_image }}\`" >> $GITHUB_STEP_SUMMARY
          if [ -n "${{ steps.login-ecr.outputs.registry }}" ]; then
            echo "- **Registry Status**: โœ… Pushed to Amazon ECR (\`${{ steps.login-ecr.outputs.registry }}\`)" >> $GITHUB_STEP_SUMMARY
          else
            echo "- **Registry Status**: โš ๏ธ Built locally (ECR credentials not supplied)" >> $GITHUB_STEP_SUMMARY
          fi          

  # ===========================================================================
  # 4. DEPLOY STAGE: Pull Images from Registry & Serve
  # ===========================================================================
  deploy:
    name: Deploy & Serve from Registry
    needs: [build-and-push-ecr]
    if: ${{ !failure() && !cancelled() }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write # Required for GitHub OIDC role assumption

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Authenticate via GitHub OIDC Role
        id: oidc-auth
        continue-on-error: true
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME || secrets.RENDER_OIDC_ROLE_ARN }}
          aws-region: ${{ secrets.AWS_REGION || 'us-east-1' }}
          role-session-name: GitHubActionsDeploy-${{ github.run_id }}
        env:
          RENDER_OIDC_ROLE_ARN: ${{ secrets.RENDER_OIDC_ROLE_ARN }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}

      - name: Log in to Amazon ECR
        id: login-ecr
        if: steps.oidc-auth.outcome == 'success'
        continue-on-error: true
        uses: aws-actions/amazon-ecr-login@v2

      - name: Ensure Docker Network Exists
        run: |
                    docker network inspect personal-lifecycle-observability-net >/dev/null 2>&1 || docker network create personal-lifecycle-observability-net

      # Pull images from registry and serve them
      - name: Pull Image from Registry & Serve (Zero Host Build)
        run: |
          BACKEND_IMAGE="${{ needs.build-and-push-ecr.outputs.backend_image }}"
          FRONTEND_IMAGE="${{ needs.build-and-push-ecr.outputs.frontend_image }}"
          TARGET_ENV="${{ needs.build-and-push-ecr.outputs.target_env }}"
          IMAGE_TAG="${{ needs.build-and-push-ecr.outputs.image_tag }}"

          echo "================================================================"
          echo "DEPLOY STAGE: Pulling image from registry and serving it"
          echo "Target Environment: $TARGET_ENV"
          echo "Image Tag: $IMAGE_TAG"
          echo "Backend Image: $BACKEND_IMAGE"
          echo "Frontend Image: $FRONTEND_IMAGE"
          echo "================================================================"

          # Pull pre-built images from registry (Zero Host Build)
          if [ -n "${{ steps.login-ecr.outputs.registry }}" ]; then
            echo "Pulling backend image from ECR..."
            docker pull "$BACKEND_IMAGE" || true
            echo "Pulling frontend image from ECR..."
            docker pull "$FRONTEND_IMAGE" || true
          else
            echo "Notice: Operating without remote ECR registry; building images on runner..."
            BACKEND_IMAGE="$BACKEND_IMAGE" FRONTEND_IMAGE="$FRONTEND_IMAGE" \
              docker compose -f docker-compose.prod.yml build
          fi

          # Launch container stack without building
          echo "Serving application containers using pre-built images..."
          BACKEND_IMAGE="$BACKEND_IMAGE" FRONTEND_IMAGE="$FRONTEND_IMAGE" \
            docker compose -f docker-compose.prod.yml up -d --no-build

          # Verify containers are running
          docker compose -f docker-compose.prod.yml ps          

      # Retrieve Render credentials from AWS Secrets Manager / SSM (if configured)
      - name: Retrieve Render Credentials via OIDC Session
        if: env.RENDER_OIDC_ROLE_ARN != '' || env.AWS_ROLE_TO_ASSUME != ''
        run: |
          TARGET_ENV="${{ needs.build-and-push-ecr.outputs.target_env }}"
          echo "Checking for Render credentials in AWS Secrets Manager for environment: $TARGET_ENV..."
          SECRET_NAME="personal-lifecycle/render-${TARGET_ENV}"
          if ! aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then
            SECRET_NAME="personal-lifecycle/render"
          fi

          if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then
            SECRET_VAL=$(aws secretsmanager get-secret-value --secret-id "$SECRET_NAME" --query SecretString --output text)
            HOOK_URL=$(echo "$SECRET_VAL" | grep -o '"deploy_hook_url": *"[^"]*"' | cut -d'"' -f4 || true)
            API_KEY=$(echo "$SECRET_VAL" | grep -o '"api_key": *"[^"]*"' | cut -d'"' -f4 || true)
            SRV_ID=$(echo "$SECRET_VAL" | grep -o '"service_id": *"[^"]*"' | cut -d'"' -f4 || true)
            if [ -n "$HOOK_URL" ]; then echo "RENDER_DEPLOY_HOOK_URL=$HOOK_URL" >> $GITHUB_ENV; fi
            if [ -n "$API_KEY" ]; then echo "RENDER_API_KEY=$API_KEY" >> $GITHUB_ENV; fi
            if [ -n "$SRV_ID" ]; then echo "RENDER_SERVICE_ID=$SRV_ID" >> $GITHUB_ENV; fi
          elif aws ssm get-parameter --name "/personal-lifecycle/${TARGET_ENV}/deploy_hook_url" >/dev/null 2>&1; then
            HOOK_URL=$(aws ssm get-parameter --name "/personal-lifecycle/${TARGET_ENV}/deploy_hook_url" --with-decryption --query Parameter.Value --output text)
            echo "RENDER_DEPLOY_HOOK_URL=$HOOK_URL" >> $GITHUB_ENV
          elif aws ssm get-parameter --name "/personal-lifecycle/render/deploy_hook_url" >/dev/null 2>&1; then
            HOOK_URL=$(aws ssm get-parameter --name "/personal-lifecycle/render/deploy_hook_url" --with-decryption --query Parameter.Value --output text)
            echo "RENDER_DEPLOY_HOOK_URL=$HOOK_URL" >> $GITHUB_ENV
          fi          
        env:
          RENDER_OIDC_ROLE_ARN: ${{ secrets.RENDER_OIDC_ROLE_ARN }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}

      # Deploy to Render if configured
      - name: Trigger Render Deployment
        id: deploy-render
        env:
          DEPLOY_HOOK: ${{ env.RENDER_DEPLOY_HOOK_URL || secrets.RENDER_DEPLOY_HOOK_URL }}
          API_KEY: ${{ env.RENDER_API_KEY || secrets.RENDER_API_KEY }}
          SERVICE_ID: ${{ env.RENDER_SERVICE_ID || secrets.RENDER_SERVICE_ID }}
        run: |
          if [ -n "$DEPLOY_HOOK" ]; then
            echo "Triggering deployment via Render Deploy Hook..."
            RESPONSE=$(curl -sS -w "\nHTTP_STATUS:%{http_code}" -X POST "$DEPLOY_HOOK")
            HTTP_CODE=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d':' -f2)
            echo "Render Deploy Hook response code: $HTTP_CODE"
          elif [ -n "$API_KEY" ] && [ -n "$SERVICE_ID" ]; then
            echo "Triggering deployment via Render REST API for service: $SERVICE_ID..."
            RESPONSE=$(curl -sS -w "\nHTTP_STATUS:%{http_code}" -X POST \
              "https://api.render.com/v1/services/${SERVICE_ID}/deploys" \
              -H "Authorization: Bearer ${API_KEY}" \
              -H "Accept: application/json" \
              -H "Content-Type: application/json" \
              -d '{"clearCache": "do_not_clear"}')
            HTTP_CODE=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d':' -f2)
            echo "Render API response code: $HTTP_CODE"
          else
            echo "Render deploy hook/API not configured; local container stack served."
          fi          

      # Validate Deployment Health Endpoint
      - name: Validate Deployment Health Endpoint
        env:
          MAX_ATTEMPTS: 30
          POLL_INTERVAL: 10
        run: |
          TARGET_ENV="${{ needs.build-and-push-ecr.outputs.target_env }}"
          if [ -n "${{ inputs.render_health_url }}" ]; then
            HEALTH_URL="${{ inputs.render_health_url }}"
          elif [ -n "${{ secrets.RENDER_HEALTH_URL }}" ]; then
            HEALTH_URL="${{ secrets.RENDER_HEALTH_URL }}"
          elif [ -n "$RENDER_DEPLOY_HOOK_URL" ] || [ -n "$RENDER_API_KEY" ]; then
            echo ">>> No explicit RENDER_HEALTH_URL provided in inputs/secrets."
            echo ">>> Skipping external domain healthcheck poll. Deployment trigger succeeded."
            exit 0
          else
            HEALTH_URL="http://127.0.0.1:8088/health"
          fi

          echo "================================================================"
          echo "Validating deployment health for: $TARGET_ENV"
          echo "Target Health URL: $HEALTH_URL"
          echo "Max Attempts: $MAX_ATTEMPTS (Checking every ${POLL_INTERVAL}s)"
          echo "================================================================"

          sleep 5

          HEALTH_SUCCESS=false
          for i in $(seq 1 $MAX_ATTEMPTS); do
            echo "Attempt $i/$MAX_ATTEMPTS: Polling $HEALTH_URL..."
            RESPONSE=$(curl -sS -w "\nHTTP_STATUS:%{http_code}" --connect-timeout 10 --max-time 15 "$HEALTH_URL" 2>&1 || true)
            HTTP_CODE=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d':' -f2 || true)
            PAYLOAD=$(echo "$RESPONSE" | grep -v "HTTP_STATUS:" || true)

            if [ "$HTTP_CODE" = "200" ]; then
              echo ">>> Healthcheck returned HTTP 200 OK!"
              echo "Response payload: $PAYLOAD"
              if echo "$PAYLOAD" | grep -q '"status"[[:space:]]*:[[:space:]]*"healthy"'; then
                echo ">>> Validated payload contains 'status: healthy'!"
                HEALTH_SUCCESS=true
                break
              else
                echo "HTTP 200 received; waiting for healthy status in payload..."
              fi
            else
              echo "Received status: ${HTTP_CODE:-connection failure}. Retrying in ${POLL_INTERVAL}s..."
            fi

            sleep $POLL_INTERVAL
          done

          if [ "$HEALTH_SUCCESS" = "true" ]; then
            echo "================================================================"
            echo "SUCCESS: Deployment is live and healthy!"
            echo "================================================================"
            echo "### ๐Ÿš€ Deploy Stage Summary" >> $GITHUB_STEP_SUMMARY
            echo "- **Status**: โœ… **Live & Healthy (HTTP 200)**" >> $GITHUB_STEP_SUMMARY
            echo "- **Health Endpoint**: \`$HEALTH_URL\`" >> $GITHUB_STEP_SUMMARY
            echo "- **Served Image Tag**: \`${{ needs.build-and-push-ecr.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
            exit 0
          else
            echo "================================================================"
            echo "ERROR: Health check validation timed out after $MAX_ATTEMPTS attempts."
            echo "Target URL: $HEALTH_URL"
            echo "================================================================"
            if [ "${{ secrets.STRICT_HEALTH_CHECK }}" = "false" ]; then
              echo "STRICT_HEALTH_CHECK=false configured; continuing without failing build."
              exit 0
            fi
            exit 1
          fi          

  # ===========================================================================
  # 5. ON-CALL AUTONOMOUS INCIDENT DISPATCH (Fires automatically on ANY failure)
  # ===========================================================================
  notify-on-call:
    name: Notify On-Call Responder on Failure
    if: failure()
    needs: [test-backend, test-frontend, security-audit, integration-and-e2e, build-and-push-ecr, deploy]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Dispatch Incident Payload to On-Call Responder
        env:
          WORKFLOW_NAME: ${{ github.workflow }}
          RUN_ID: ${{ github.run_id }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          HEAD_BRANCH: ${{ github.ref_name }}
          HEAD_SHA: ${{ github.sha }}
        run: |
          echo "================================================================"
          echo "  DISPATCHING AUTOMATED ON-CALL INCIDENT EVENT FOR CI FAILURE   "
          echo "================================================================"
          python3 on-call-engineer/scripts/receive-alert <<EOF
          {
            "workflow_run": {
              "id": $RUN_ID,
              "name": "$WORKFLOW_NAME",
              "conclusion": "failure",
              "head_branch": "$HEAD_BRANCH",
              "head_sha": "$HEAD_SHA",
              "html_url": "$RUN_URL",
              "updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
            }
          }
          EOF