04. Context Window

flowchart LR
    PROMPT["Long Prompt\n(history + instructions + documents)"]
    PROMPT --> CW["Context Window\n(LLM's working memory)"]
    CW --> MODEL["Transformer\n(processes all tokens\nwithin window)"]
    MODEL --> RESPONSE["Generated Response"]

    PROMPT -.->|"❌ Beyond window\n(Lost — model cannot see)"| FORGOTTEN["Forgotten Content"]

    style PROMPT fill:#3b82f6,color:#fff
    style CW fill:#22c55e,color:#fff
    style MODEL fill:#8b5cf6,color:#fff
    style RESPONSE fill:#22c55e,color:#fff
    style FORGOTTEN fill:#ef4444,color:#fff

flowchart TD
    subgraph QUADRATIC["Quadratic Cost of Attention"]
        L1["2,000 tokens → 4M attention pairs\n(fast, cheap)"]
        L2["8,000 tokens → 64M attention pairs\n(moderate)"]
        L3["32,000 tokens → 1B attention pairs\n(expensive)"]
        L4["128,000 tokens → 16B attention pairs\n(very expensive)"]
        L5["1,000,000 tokens → 1T attention pairs\n(extremely expensive)"]
    end

    style L1 fill:#22c55e,color:#fff
    style L2 fill:#8b5cf6,color:#fff
    style L3 fill:#f59e0b,color:#fff
    style L4 fill:#ef4444,color:#fff
    style L5 fill:#dc2626,color:#fff

flowchart TD
    subgraph DESK["Desk Surface = Context Window"]
        A["Paper 1: Instructions"]
        B["Paper 2: Conversation so far"]
        C["Paper 3: Current question"]
        D["Paper 4: Reference document"]
    end

    subgraph FLOOR["Floor = Lost (outside context)"]
        E["Paper 5: Old conversation\n(Forgotten)"]
        F["Paper 6: Long document\n(Truncated)"]
        G["Paper 7: Additional context\n(Dropped)"]
    end

    style A fill:#22c55e,color:#fff
    style B fill:#22c55e,color:#fff
    style C fill:#22c55e,color:#fff
    style D fill:#22c55e,color:#fff
    style E fill:#ef4444,color:#fff
    style F fill:#ef4444,color:#fff
    style G fill:#ef4444,color:#fff
    style DESK fill:#3b82f6,color:#fff
    style FLOOR fill:#dc2626,color:#fff

flowchart LR
    GPT2["GPT-2\n1,024 tokens\n2019"] --> GPT35["GPT-3.5\n4,096 tokens\n2022"]
    GPT35 --> GPT4["GPT-4\n8,192 → 32K → 128K\n2023"]
    GPT4 --> GPT4O["GPT-4o\n128K tokens\n2024"]
    GPT4O --> CLAUDE3["Claude 3\n200K tokens\n2024"]
    CLAUDE3 --> GEMINI15["Gemini 1.5 Pro\n1M tokens\n2024"]
    GEMINI15 --> CLAUDE4["Claude 3.5 / 4\n200K tokens\n2024-"]

    style GPT2 fill:#ef4444,color:#fff
    style GPT35 fill:#f59e0b,color:#fff
    style GPT4 fill:#f59e0b,color:#fff
    style GPT4O fill:#8b5cf6,color:#fff
    style CLAUDE3 fill:#8b5cf6,color:#fff
    style GEMINI15 fill:#22c55e,color:#fff
    style CLAUDE4 fill:#22c55e,color:#fff

flowchart TD
    CW["Total Context Window\n(e.g., 128K tokens)"]
    CW --> SYS["System Prompt\n(instructions, personality, rules)"]
    CW --> HIST["Conversation History\n(previous messages in the chat)"]
    CW --> INPUT["Current Input\n(user's latest message + attachments)"]
    CW --> OUTPUT["Model Output\n(generated response)"]

    SYS --> USED1["~200-2,000 tokens"]
    HIST --> USED2["? (grows with each message)"]
    INPUT --> USED3["? (user-dependent)"]
    OUTPUT --> USED4["? (generated in this turn)"]

    style CW fill:#3b82f6,color:#fff
    style SYS fill:#8b5cf6,color:#fff
    style HIST fill:#f59e0b,color:#fff
    style INPUT fill:#ef4444,color:#fff
    style OUTPUT fill:#22c55e,color:#fff

flowchart TD
    START["User sends message"] --> CHECK{"Total tokens\nwithin context\nwindow?"}
    CHECK -->|"Yes"| FULL["Full context sent\nto model"]
    CHECK -->|"No"| STRATEGY{"What happens?"}

    STRATEGY --> TRUNC["Truncation\n(Oldest messages dropped)"]
    STRATEGY --> SUM["Summarization\n(Previous conversation\ncompressed)"]
    STRATEGY --> ERROR["Error\n(API rejects the request)"]

    TRUNC --> LOSE["Model loses context\nof early conversation"]
    SUM --> LOSE2["Summary may miss\ndetails and nuance"]
    ERROR --> RETRY["User must shorten\ntheir prompt"]

    style START fill:#3b82f6,color:#fff
    style CHECK fill:#f59e0b,color:#fff
    style FULL fill:#22c55e,color:#fff
    style TRUNC fill:#ef4444,color:#fff
    style SUM fill:#f59e0b,color:#fff
    style ERROR fill:#dc2626,color:#fff
    style LOSE fill:#ef4444,color:#fff
    style LOSE2 fill:#ef4444,color:#fff

sequenceDiagram
    participant User
    participant App as Chat Application
    participant LLM as Language Model

    User->>App: Message 1: "What is AI?"
    App->>LLM: [System Prompt] + "What is AI?"
    LLM-->>App: "AI stands for Artificial Intelligence..."
    App-->>User: Shows response

    User->>App: Message 2: "Explain it to a child"
    App->>LLM: [System Prompt] + "What is AI?" + "AI stands for..." + "Explain it to a child"
    LLM-->>App: "Imagine a computer that can learn like a child..."
    App-->>User: Shows response

    User->>App: Message 3: "Give me an example"
    App->>LLM: [System Prompt] + Q1 + A1 + Q2 + A2 + "Give me an example"
    Note over App,LLM: Context grows with each turn!
    LLM-->>App: "Think of a smart toy that recognizes your face..."
    App-->>User: Shows response

    User->>App: Message 10: (after a long conversation)
    App->>App: Context now exceeds window →
    App->>App: Must truncate old messages
    App->>LLM: [System Prompt] + (last N messages that fit)
    Note over App: Early parts of conversation lost!

flowchart TD
    BUDGET["Total Budget: 128,000 tokens"]

    BUDGET --> SYS["System Instructions: 2,000 tokens\n(1.6% of budget)"]
    BUDGET --> FILE["Attached Document: 50,000 tokens\n(39% of budget)"]
    BUDGET --> HIST["Conversation History: 60,000 tokens\n(46.9% of budget)"]
    BUDGET --> QUESTION["Current Question: 500 tokens\n(0.4% of budget)"]
    BUDGET --> OUT["Available for Response: 15,500 tokens\n(12.1% of budget)"]

    style SYS fill:#3b82f6,color:#fff
    style FILE fill:#f59e0b,color:#fff
    style HIST fill:#8b5cf6,color:#fff
    style QUESTION fill:#22c55e,color:#fff
    style OUT fill:#22c55e,color:#fff

Prompt includes: "Analyze this 300-page novel..."
Document tokens: 390,000 tokens
Context window: 200,000 tokens (Claude 3)
Result: 190,000 tokens of the novel are outside the window

Message 1-20: Normal chat (~8,000 tokens)
Message 21: User attaches a large code file (15,000 tokens)
Message 22: User asks a follow-up (still fine)
Message 23: User asks again (now old messages may be dropped)

Before message 23: System prompt + messages 1-22 = ~25,000 tokens
After message 23: System prompt + messages 2-22 + msg 23 + new attachment = too much
→ Messages 1-10 are dropped
→ Model no longer remembers setup details from early conversation

System: 2,000 tokens
Codebase context: 100,000 tokens (full repository)
User question: 500 tokens
Total: 102,500 tokens → exceeds most models' context

Solution: Use RAG to retrieve only relevant files instead of sending the full repo

flowchart LR
    ORIG["Original:\n12,000 tokens\n(full conversation history)"]
    ORIG --> COMP["Compressor:\n(summarization or\nembedding-based)"]
    COMP --> SMALL["Compressed:\n2,000 tokens\n(essential meaning preserved)"]
    SMALL --> MODEL["LLM"]

    style ORIG fill:#ef4444,color:#fff
    style COMP fill:#f59e0b,color:#fff
    style SMALL fill:#22c55e,color:#fff
    style MODEL fill:#8b5cf6,color:#fff

import tiktoken

encoding = tiktoken.get_encoding("cl100k_base")

def count_tokens(text: str) -> int:
    return len(encoding.encode(text))

def check_fits(text: str, model_context: int, model_name: str):
    tokens = count_tokens(text)
    fits = tokens <= model_context
    print(f"{model_name}: {tokens} token{'s' if tokens != 1 else ''}")
    print(f"  Context window: {model_context}")
    print(f"  Fits: {'✅ Yes' if fits else '❌ No — exceeds by ' + str(tokens - model_context) + ' tokens'}")
    print()

# Model context windows
MODELS = {
    "GPT-3.5-turbo": 16385,
    "GPT-4o": 128000,
    "Claude 3 Opus": 200000,
    "Gemini 1.5 Pro": 1000000,
    "LLaMA 2": 4096,
}

# Test content
short_text = "Hello world! This is a short message."
long_text = "This is a long document. " * 10000  # ~100,000 words

print("Short text (all models):")
for name, window in MODELS.items():
    check_fits(short_text, window, name)

print("\nLong text:")
for name, window in MODELS.items():
    check_fits(long_text, window, name)

// Simulating context window management in a chat application

const MAX_CONTEXT_TOKENS = 128000; // GPT-4o window
const SYSTEM_PROMPT_TOKENS = 500; // estimated

// Simplified token counter
function estimateTokens(text) {
  return Math.ceil(text.split(/\s+/).length * 1.3);
}

class ChatSession {
  constructor() {
    this.messages = [];
    this.totalTokens = SYSTEM_PROMPT_TOKENS;
  }

  addMessage(role, content) {
    const tokens = estimateTokens(content);
    const message = { role, content, tokens };

    // Check if adding this message would exceed the window
    if (this.totalTokens + tokens > MAX_CONTEXT_TOKENS) {
      // Remove oldest messages until we fit
      while (this.messages.length > 0 &&
             this.totalTokens + tokens > MAX_CONTEXT_TOKENS * 0.9) {
        const removed = this.messages.shift();
        this.totalTokens -= removed.tokens;
        console.log(`Dropped message: "${removed.content.slice(0, 40)}..."`);
      }
    }

    this.messages.push(message);
    this.totalTokens += tokens;
  }

  getContext() {
    return {
      messages: this.messages,
      totalTokens: this.totalTokens,
      availableTokens: MAX_CONTEXT_TOKENS - this.totalTokens
    };
  }
}

// Usage simulation
const chat = new ChatSession();

// Simulate a long conversation
for (let i = 1; i <= 100; i++) {
  chat.addMessage('user', `This is message number ${i} in our conversation about AI and machine learning.`);
  chat.addMessage('assistant', `This is response number ${i} acknowledging your question about AI and ML.`);
}

const context = chat.getContext();
console.log(`Total messages in context: ${context.messages.length}`);
console.log(`Total tokens used: ${context.totalTokens}`);
console.log(`Available tokens remaining: ${context.availableTokens}`);
// Output:
// Total messages in context: ~138 (system drops oldest)
// Total tokens used: ~115,000
// Available tokens remaining: ~12,500