Prompt Caching

Copy

• Copy for LLM

  Copy page as Markdown for LLMs
• View as Markdown

  Open this page as Markdown
• Open in ChatGPT

  Get insights from ChatGPT
• Open in Claude

  Get insights from Claude

Request 1: [System Prompt + Long Context] + User Query
           └─────────────────────────────┘
                 Processed & Cached
                 
Request 2: [Same System Prompt + Long Context] + Different User Query
           └────────────────────────────────┘
                   Retrieved from Cache
                   
Result: Faster response, lower cost

{
  "usage": {
    "inputNeurons": 1500,
    "cachedNeurons": 1200,
    "outputSynapses": 150,
    "totalNeurons": 1500,
    "totalSynapses": 150
  }
}

// ✅ Good - Static content cached
{
  "inputs": [
    {
      "role": "system",
      "texts": [{"text": "You are an expert Python developer..."}]  // Cached
    },
    {
      "role": "user",
      "texts": [{"text": "How do I fix this error: {{user_error}}"}]  // Dynamic
    }
  ]
}

// ❌ Bad - Dynamic content breaks cache
{
  "inputs": [
    {
      "role": "system",
      "texts": [{"text": "Current time: {{timestamp}}"}]  // Changes every request
    },
    {
      "role": "system",
      "texts": [{"text": "You are an expert Python developer..."}]  // Won't cache
    }
  ]
}

{
  "inputs": [
    {
      "role": "system",
      "texts": [
        {"text": "# API Documentation\n\n... (5000 tokens of docs) ..."}
      ]
    },
    {
      "role": "system",
      "texts": [
        {"text": "# Code Style Guide\n\n... (3000 tokens) ..."}
      ]
    },
    {
      "role": "user",
      "texts": [{"text": "Help me write a function"}]  // Only this part changes
    }
  ]
}

# First message in thread - builds cache
response1 = requests.post('/v1/model/response', json={
    "model": "gpt-4.1",
    "thread": "thread_user123",
    "inputs": [{"role": "user", "texts": [{"text": "Hello"}]}]
})

# Follow-up - conversation history cached
response2 = requests.post('/v1/model/response', json={
    "model": "gpt-4.1",
    "thread": "thread_user123",  # Previous messages cached
    "inputs": [{"role": "user", "texts": [{"text": "Tell me more"}]}]
})

{
  "model": "gpt-4.1",
  "prompt": {
    "id": "prompt_code_reviewer",  // Template content gets cached
    "variables": {
      "language": "Python",  // Only variables change
      "focus": "security"
    }
  },
  "inputs": [
    {"role": "user", "texts": [{"text": "Review this code: ..."}]}
  ]
}

# Embed large documentation in system prompt
DOCS = """
[5000 tokens of product documentation]
"""

def answer_question(question):
    return requests.post('/v1/model/response', json={
        "model": "gpt-4.1",
        "inputs": [
            {"role": "system", "texts": [{"text": DOCS}]},  # Cached after first call
            {"role": "user", "texts": [{"text": question}]}
        ]
    })

# First call: Full processing
answer_question("What is feature X?")  # cachedNeurons: 0

# Subsequent calls: Cached docs
answer_question("How do I use feature Y?")  # cachedNeurons: 5000
answer_question("What's the pricing?")  # cachedNeurons: 5000

# Provide codebase context once
CODEBASE_CONTEXT = """
[Large codebase structure and key files]
"""

def analyze_code(code_snippet):
    return requests.post('/v1/model/response', json={
        "model": "gpt-4.1",
        "inputs": [
            {"role": "system", "texts": [{"text": CODEBASE_CONTEXT}]},  # Cached
            {"role": "user", "texts": [{"text": f"Analyze: {code_snippet}"}]}
        ]
    })

// First message - establishes cache
await createResponse({
  model: 'gpt-4.1',
  thread: 'support_ticket_123',
  inputs: [
    {
      role: 'system',
      texts: [{ text: customerContext }]  // Customer history cached
    },
    {
      role: 'user',
      texts: [{ text: 'I need help' }]
    }
  ]
});

// Follow-ups automatically benefit from cache
await createResponse({
  model: 'gpt-4.1',
  thread: 'support_ticket_123',  // Previous context cached
  inputs: [
    { role: 'user', texts: [{ text: 'Can you clarify?' }] }
  ]
});

import requests

response = requests.post('/v1/model/response', json={
    "model": "gpt-4.1",
    "inputs": [...]
})

data = response.json()
usage = data['usage']

cache_hit_rate = usage['cachedNeurons'] / usage['inputNeurons'] * 100
cost_savings = (usage['cachedNeurons'] * 0.50)  # 50% discount

print(f"Cache hit rate: {cache_hit_rate}%")
print(f"Neurons saved: {usage['cachedNeurons']}")
print(f"Cost savings: ${cost_savings / 1_000_000:.4f}")

const logCacheMetrics = (response) => {
  const { usage } = response;
  const hitRate = (usage.cachedNeurons / usage.inputNeurons) * 100;
  
  console.log({
    timestamp: new Date(),
    model: response.model,
    inputNeurons: usage.inputNeurons,
    cachedNeurons: usage.cachedNeurons,
    cacheHitRate: `${hitRate.toFixed(1)}%`,
    estimatedSavings: (usage.cachedNeurons * 0.0000025 * 0.5).toFixed(4)  // GPT-4.1 pricing
  });
};