Rate limits & concurrency

import asyncio
from openai import AsyncOpenAI

client = AsyncOpenAI(api_key="sk-...", base_url="https://openp.ai/v1")
sem = asyncio.Semaphore(32)  # at most 32 concurrent requests

async def call(prompt: str):
    async with sem:
        return await client.chat.completions.create(
            model="gpt-5.5",
            messages=[{"role": "user", "content": prompt}],
        )

tasks = [call(f"task {i}") for i in range(1000)]
results = await asyncio.gather(*tasks)

import asyncio, random

async def call_with_retry(prompt, max_retries=5):
    delay = 1
    for attempt in range(max_retries):
        try:
            return await client.chat.completions.create(
                model="gpt-5.5",
                messages=[{"role": "user", "content": prompt}],
            )
        except openai.RateLimitError as e:
            retry_after = float(e.response.headers.get("Retry-After", delay))
            await asyncio.sleep(retry_after + random.uniform(0, 0.5))
            delay = min(delay * 2, 30)
    raise RuntimeError("max retries exceeded")

from aiolimiter import AsyncLimiter

# 200 requests per minute
rate = AsyncLimiter(max_rate=200, time_period=60)

async def call(prompt):
    async with rate:
        return await client.chat.completions.create(...)

remaining_req = int(resp.response.headers.get("x-ratelimit-remaining-requests", "100"))
remaining_tok = int(resp.response.headers.get("x-ratelimit-remaining-tokens", "1000000"))

# proactively sleep when remaining < 10%
if remaining_req < 20:
    await asyncio.sleep(2)