09. Deployment & Scaling

flowchart TD
    subgraph DEV["Development"]
        CODE["Code + Prompts"]
        DOCKER["Docker Image"]
    end
    subgraph CI["CI Pipeline"]
        BUILD["Build Image"]
        TEST["Test + Eval"]
    end
    subgraph CD["CD Pipeline"]
        DEV_ENV["Deploy Dev"]
        STAGING["Deploy Staging"]
        PROD["Deploy Production\nCanary → 100%"]
    end
    subgraph PROD_ENV["Production"]
        K8S["Kubernetes Cluster"]
        HPA["Autoscaling"]
        MON["Monitoring"]
    end

    DEV --> CI
    CI --> CD
    CD --> PROD_ENV

    style DEV fill:#3b82f6,color:#fff
    style CI fill:#8b5cf6,color:#fff
    style CD fill:#f59e0b,color:#fff
    style PROD_ENV fill:#22c55e,color:#fff

sequenceDiagram
    participant Dev as Developer
    participant CI as CI/CD
    participant Prod as Production
    participant Monitor as Monitoring

    Dev->>CI: Push new prompt v5
    CI->>CI: Run tests (all pass)
    CI->>Prod: Deploy to all users

    Prod->>Prod: Quality drops 15%
    Monitor->>Monitor: Detects regression
    Monitor->>Dev: Alert: quality regression detected

    Dev->>Prod: Rollback to v4
    Note over Dev,Monitor: If they had used canary<br/>only 5% of users would<br/>have seen the bad prompt

FROM node:20-slim

WORKDIR /app

# Install dependencies
COPY package*.json ./
RUN npm ci --only=production

# Copy application code
COPY . .

# Copy prompt templates (versioned)
COPY prompts/ ./prompts/

# Set environment
ENV NODE_ENV=production
ENV PROMPT_VERSION=v4

EXPOSE 3000

CMD ["node", "server.js"]

# Build stage
FROM node:20 AS builder
WORKDIR /app
COPY package*.json ./
RUN npm ci
COPY . .
RUN npm run build

# Runtime stage
FROM node:20-slim
WORKDIR /app
COPY --from=builder /app/dist ./dist
COPY --from=builder /app/node_modules ./node_modules
COPY prompts/ ./prompts/
CMD ["node", "dist/server.js"]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ai-router
  labels:
    app: ai-router
    version: v4
spec:
  replicas: 5
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: ai-router
  template:
    metadata:
      labels:
        app: ai-router
    spec:
      containers:
      - name: ai-router
        image: myregistry/ai-router:v4
        ports:
        - containerPort: 3000
        env:
        - name: PROMPT_VERSION
          value: "v4"
        - name: LLM_API_KEY
          valueFrom:
            secretKeyRef:
              name: llm-secrets
              key: openai-key
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 30
        readinessProbe:
          httpGet:
            path: /ready
            port: 3000
          initialDelaySeconds: 5

flowchart TD
    subgraph INGRESS["Ingress Layer"]
        LB["Load Balancer"]
        GW["API Gateway"]
    end
    subgraph SERVICES["AI Services"]
        ROUTER["Prompt Router\nDeployment: 3-10 pods"]
        RAG["RAG Service\nDeployment: 2-5 pods"]
        AGENT["Agent Service\nDeployment: 2-5 pods"]
        GUARD["Guardrail Service\nDeployment: 3-8 pods"]
    end
    subgraph INFRA["Infrastructure"]
        REDIS["Redis\nStatefulSet"]
        VECTOR["Vector DB\nExternal/Operator"]
        MON["Monitoring\nPrometheus + Grafana"]
    end

    LB --> GW
    GW --> ROUTER
    ROUTER --> RAG
    ROUTER --> AGENT
    RAG --> VECTOR
    ROUTER --> GUARD
    RAG --> REDIS
    SERVICES --> MON

    style INGRESS fill:#3b82f6,color:#fff
    style SERVICES fill:#8b5cf6,color:#fff
    style INFRA fill:#6366f1,color:#fff

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: ai-router-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: ai-router
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80

flowchart TD
    subgraph BLUE["Blue (Current)"]
        B1["App v3\nPrompt v4\nModel: GPT-4o"]
        B2["100% Traffic"]
    end
    subgraph GREEN["Green (New)"]
        G1["App v4\nPrompt v5\nModel: GPT-4o"]
        G2["0% Traffic"]
    end

    BLUE -->|"Step 1: Deploy Green"| GREEN
    GREEN -->|"Step 2: Route 100% to Green"| G2_ACTIVE["Green: 100% Traffic"]
    G2_ACTIVE -->|"Step 3: Verify"| VERIFY{"Quality OK?"}
    VERIFY -->|"Yes"| DONE["Blue decommissioned"]
    VERIFY -->|"No"| ROLLBACK["Route back to Blue"]

    style BLUE fill:#3b82f6,color:#fff
    style GREEN fill:#22c55e,color:#fff
    style ROLLBACK fill:#ef4444,color:#fff

flowchart TD
    START["Deploy v5"] --> CANARY_1["Canary: 1%\nMonitor for 10 min"]
    CANARY_1 --> CHECK_1{"Quality OK?\nLatency OK?\nCost OK?"}
    CHECK_1 -->|"No"| ROLLBACK["Rollback to v4"]
    CHECK_1 -->|"Yes"| CANARY_2["Canary: 5%\nMonitor for 30 min"]
    CANARY_2 --> CHECK_2{"All metrics stable?"}
    CHECK_2 -->|"No"| ROLLBACK
    CHECK_2 -->|"Yes"| CANARY_3["Canary: 25%\nMonitor for 2 hours"]
    CANARY_3 --> CHECK_3{"All metrics stable?"}
    CHECK_3 -->|"No"| ROLLBACK
    CHECK_3 -->|"Yes"| FULL["Full Rollout: 100%"]

    style START fill:#f59e0b,color:#fff
    style ROLLBACK fill:#ef4444,color:#fff
    style FULL fill:#22c55e,color:#fff

flowchart LR
    REQ["Request"] --> GW["API Gateway"]
    GW --> FN["Lambda / Cloud Function"]
    FN --> MODEL["LLM API"]
    MODEL --> FN
    FN --> GW
    GW --> RESP["Response"]

    style REQ fill:#f59e0b,color:#fff
    style FN fill:#3b82f6,color:#fff
    style MODEL fill:#22c55e,color:#fff

// AWS Lambda handler for AI routing
exports.handler = async (event) => {
  const { query, user_id } = JSON.parse(event.body);

  // Classify query complexity
  const complexity = await classifyQuery(query);

  // Route to appropriate model
  const model = complexity === 'simple' ? 'gpt-4o-mini' : 'gpt-4o';

  // Call LLM
  const response = await callLLM(query, model);

  return {
    statusCode: 200,
    body: JSON.stringify({ response, model, cost: response.cost }),
  };
};

flowchart TD
    subgraph EDGE["Edge Locations"]
        US_EAST["US East\n< 10ms latency"]
        US_WEST["US West\n< 10ms latency"]
        EU["EU (Frankfurt)\n< 10ms latency"]
        APAC["APAC (Tokyo)\n< 10ms latency"]
    end
    subgraph CENTRAL["Central Services"]
        LLM_GW["LLM Gateway\nCentral API keys"]
        MODEL_REG["Model Registry"]
        PROMPT_REG["Prompt Registry"]
    end

    US_EAST --> CENTRAL
    US_WEST --> CENTRAL
    EU --> CENTRAL
    APAC --> CENTRAL

    style EDGE fill:#22c55e,color:#fff
    style CENTRAL fill:#3b82f6,color:#fff

flowchart TD
    COMMIT["Git Commit\nCode + Prompts"] --> BUILD["CI Build\nDocker image\nRun tests\nRun evaluation"]
    BUILD --> REGISTRY["Registry\nDocker Hub / ECR"]
    REGISTRY --> DEV_DEPLOY["Deploy to Dev\nAutomated"]
    DEV_DEPLOY --> DEV_TEST["Dev Tests\nIntegration + E2E"]
    DEV_TEST -->|"Pass"| STAGE_DEPLOY["Deploy to Staging\nManual approval"]
    DEV_TEST -->|"Fail"| FIX["Fix and recommit"]
    STAGE_DEPLOY --> STAGE_TEST["Staging Evaluation\nGolden dataset\nCanary test"]
    STAGE_TEST -->|"Pass"| PROD_DEPLOY["Deploy to Production\nCanary → 100%"]
    STAGE_TEST -->|"Fail"| FIX

    style COMMIT fill:#3b82f6,color:#fff
    style PROD_DEPLOY fill:#22c55e,color:#fff
    style FIX fill:#ef4444,color:#fff

mindmap
  root((Scaling AI))
    Horizontal
      More API gateway instances
      More prompt router workers
      More RAG service instances
    Vertical
      Larger instances for LLM processing
      More memory for context handling
    Caching
      Reduce load on LLM APIs
      Reduce latency
    Queue-based
      Decouple request handling
      Smooth traffic spikes
    Multi-region
      Reduce global latency
      Regional failover