Blue/Green Deployment (4): Operations and Optimization

# SLO example
slos:
  availability:
    target: 99.9%  # allows roughly 43.2 minutes of downtime per month
    window: 30d
    indicator: |
      sum(rate(http_requests_total{status!~"5.."}[5m])) /
      sum(rate(http_requests_total[5m]))
 
  latency:
    target: 95%    # 95% of requests under 200ms
    window: 30d
    indicator: |
      histogram_quantile(0.95,
        sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
      )
 
  error_budget:
    monthly: 43.2m  # 99.9% SLO = 43.2 minutes/month
    alert_threshold: 50%  # alert when half the budget is spent

# prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: blue-green-alerts
spec:
  groups:
  - name: deployment
    rules:
    # 5xx error rate spike
    - alert: HighErrorRate
      expr: |
        sum(rate(http_requests_total{status=~"5.."}[5m])) /
        sum(rate(http_requests_total[5m])) > 0.01
      for: 2m
      labels:
        severity: critical
        action: rollback
      annotations:
        summary: "High error rate detected after deployment"
        description: "Error rate is {{ $value | humanizePercentage }} (threshold: 1%)"
        runbook_url: "https://wiki.example.com/runbooks/high-error-rate"
 
    # latency spike
    - alert: HighLatency
      expr: |
        histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
        > 0.5
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "P99 latency exceeded 500ms"
        description: "Current P99: {{ $value | humanizeDuration }}"
 
    # pod restart detection
    - alert: PodRestartLoop
      expr: |
        increase(kube_pod_container_status_restarts_total{
          pod=~"my-app-.*"
        }[15m]) > 3
      labels:
        severity: critical
        action: rollback
      annotations:
        summary: "Pod restart loop detected"
 
    # post-deployment rollback detection
    - alert: DeploymentRollback
      expr: |
        kube_deployment_status_observed_generation{deployment="my-app"}
        < kube_deployment_metadata_generation{deployment="my-app"}
      for: 1m
      labels:
        severity: info
      annotations:
        summary: "Deployment rollback in progress"

{
  "dashboard": {
    "title": "Blue/Green Deployment Dashboard",
    "panels": [
      {
        "title": "Traffic Distribution",
        "type": "piechart",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{color=\"blue\"}[5m]))",
            "legendFormat": "Blue"
          },
          {
            "expr": "sum(rate(http_requests_total{color=\"green\"}[5m]))",
            "legendFormat": "Green"
          }
        ]
      },
      {
        "title": "Error Rate by Version",
        "type": "timeseries",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{status=~\"5..\",color=\"blue\"}[5m])) / sum(rate(http_requests_total{color=\"blue\"}[5m]))",
            "legendFormat": "Blue 5xx"
          },
          {
            "expr": "sum(rate(http_requests_total{status=~\"5..\",color=\"green\"}[5m])) / sum(rate(http_requests_total{color=\"green\"}[5m]))",
            "legendFormat": "Green 5xx"
          }
        ]
      },
      {
        "title": "P99 Latency Comparison",
        "type": "timeseries",
        "targets": [
          {
            "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{color=\"blue\"}[5m])) by (le))",
            "legendFormat": "Blue P99"
          },
          {
            "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{color=\"green\"}[5m])) by (le))",
            "legendFormat": "Green P99"
          }
        ]
      },
      {
        "title": "Deployment Events",
        "type": "annotations",
        "datasource": "-- Grafana --",
        "query": "tags=deployment"
      }
    ]
  }
}

# deploy_marker.py - record deployment events in Grafana
import os
import requests
from datetime import datetime
 
def mark_deployment(version: str, environment: str, color: str):
    """Create a deployment event marker in Grafana"""
    grafana_url = "https://grafana.example.com"
    api_key = os.environ["GRAFANA_API_KEY"]
 
    annotation = {
        "time": int(datetime.now().timestamp() * 1000),
        "tags": ["deployment", environment, color],
        "text": f"Deployed {version} to {color} ({environment})"
    }
 
    response = requests.post(
        f"{grafana_url}/api/annotations",
        json=annotation,
        headers={"Authorization": f"Bearer {api_key}"}
    )
    return response.status_code == 200

# Use spot instances for the Green environment (AWS EKS)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app-green
spec:
  template:
    spec:
      nodeSelector:
        node.kubernetes.io/lifecycle: spot
      tolerations:
      - key: "node.kubernetes.io/lifecycle"
        operator: "Equal"
        value: "spot"
        effect: "NoSchedule"
      # graceful shutdown for spot interruptions
      terminationGracePeriodSeconds: 30
      containers:
      - name: app
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 10"]

# green-hpa.yaml - keep the Green environment minimal
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: my-app-green
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-app-green
  minReplicas: 1  # minimum footprint in steady state
  maxReplicas: 10 # expand during deployment
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 0  # scale up immediately
      policies:
      - type: Percent
        value: 100
        periodSeconds: 15
    scaleDown:
      stabilizationWindowSeconds: 300  # wait 5 minutes before scaling down

#!/bin/bash
# scale-green.sh - shrink the Green environment outside business hours
 
HOUR=$(date +%H)
DAY=$(date +%u)
 
# scale down outside business hours (Mon-Fri 09:00-18:00)
if [ $DAY -gt 5 ] || [ $HOUR -lt 9 ] || [ $HOUR -gt 18 ]; then
  echo "Off-hours: Scaling down Green environment"
  kubectl scale deployment my-app-green --replicas=0
else
  echo "Business hours: Maintaining Green environment"
  kubectl scale deployment my-app-green --replicas=1
fi

# automated with CronJobs
apiVersion: batch/v1
kind: CronJob
metadata:
  name: scale-green-down
spec:
  schedule: "0 19 * * 1-5"  # weekdays at 19:00
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: deployment-scaler
          containers:
          - name: kubectl
            image: bitnami/kubectl
            command:
            - kubectl
            - scale
            - deployment/my-app-green
            - --replicas=0
          restartPolicy: OnFailure
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: scale-green-up
spec:
  schedule: "0 8 * * 1-5"  # weekdays at 08:00
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: deployment-scaler
          containers:
          - name: kubectl
            image: bitnami/kubectl
            command:
            - kubectl
            - scale
            - deployment/my-app-green
            - --replicas=1
          restartPolicy: OnFailure

# Route53 TTL example
aws route53 change-resource-record-sets \
  --hosted-zone-id Z123456 \
  --change-batch '{
    "Changes": [{
      "Action": "UPSERT",
      "ResourceRecordSet": {
        "Name": "api.example.com",
        "Type": "A",
        "TTL": 60,
        "ResourceRecords": [{"Value": "1.2.3.4"}]
      }
    }]
  }'

# CloudFront cache invalidation
aws cloudfront create-invalidation \
  --distribution-id E123456789 \
  --paths "/*"
 
# Fastly cache purge
curl -X POST "https://api.fastly.com/service/{service_id}/purge_all" \
  -H "Fastly-Key: $FASTLY_API_KEY"

# Kubernetes Service - connection draining settings
apiVersion: v1
kind: Service
metadata:
  name: my-app
  annotations:
    # AWS ALB
    service.beta.kubernetes.io/aws-load-balancer-connection-draining-enabled: "true"
    service.beta.kubernetes.io/aws-load-balancer-connection-draining-timeout: "60"
spec:
  # ...

# Pod - graceful shutdown
apiVersion: apps/v1
kind: Deployment
spec:
  template:
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: app
        lifecycle:
          preStop:
            exec:
              # stop new connections, wait for existing ones to finish
              command:
              - /bin/sh
              - -c
              - |
                # make the health check fail
                touch /tmp/shutdown
                # wait for existing connections to drain
                sleep 30

# Externalizing sessions through Redis (Spring Boot)
# application.yml
spring:
  session:
    store-type: redis
    redis:
      namespace: spring:session
  redis:
    host: redis.example.com
    port: 6379
 
---
# When using session affinity in Kubernetes
apiVersion: v1
kind: Service
metadata:
  name: my-app
spec:
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 3600

# diagnose
kubectl get pods -l color=green
kubectl describe pod my-app-green-xxx
 
# common causes
# 1. image pull failure
kubectl get events --field-selector reason=Failed
 
# 2. readiness probe failure
kubectl logs my-app-green-xxx
 
# 3. insufficient resources
kubectl describe node | grep -A5 "Allocated resources"

# roll back immediately
kubectl patch svc my-app -p '{"spec":{"selector":{"color":"blue"}}}'
 
# root cause analysis
# 1. check logs
kubectl logs -l color=green --tail=100
 
# 2. check dependency connectivity
kubectl exec -it my-app-green-xxx -- curl -v http://db-service:5432
 
# 3. compare environment variables
kubectl get deployment my-app-blue -o json | jq '.spec.template.spec.containers[0].env'
kubectl get deployment my-app-green -o json | jq '.spec.template.spec.containers[0].env'

# check Redis sessions
redis-cli KEYS "spring:session:*" | wc -l
 
# fix: verify session migration
# 1. check session store connectivity
kubectl exec my-app-green-xxx -- nc -zv redis.example.com 6379
 
# 2. check session key format (cross-version compatibility)
redis-cli GET "spring:session:sessions:xxx"