限流与并发

import asyncio
from openai import AsyncOpenAI

client = AsyncOpenAI(api_key="sk-...", base_url="https://openp.ai/v1")
sem = asyncio.Semaphore(32)  # 同时最多 32 个并发请求

async def call(prompt: str):
    async with sem:
        return await client.chat.completions.create(
            model="gpt-5.5",
            messages=[{"role": "user", "content": prompt}],
        )

tasks = [call(f"task {i}") for i in range(1000)]
results = await asyncio.gather(*tasks)

import asyncio, random

async def call_with_retry(prompt, max_retries=5):
    delay = 1
    for attempt in range(max_retries):
        try:
            return await client.chat.completions.create(
                model="gpt-5.5",
                messages=[{"role": "user", "content": prompt}],
            )
        except openai.RateLimitError as e:
            retry_after = float(e.response.headers.get("Retry-After", delay))
            await asyncio.sleep(retry_after + random.uniform(0, 0.5))
            delay = min(delay * 2, 30)
    raise RuntimeError("max retries exceeded")

from aiolimiter import AsyncLimiter

# 每分钟 200 个请求
rate = AsyncLimiter(max_rate=200, time_period=60)

async def call(prompt):
    async with rate:
        return await client.chat.completions.create(...)

remaining_req = int(resp.response.headers.get("x-ratelimit-remaining-requests", "100"))
remaining_tok = int(resp.response.headers.get("x-ratelimit-remaining-tokens", "1000000"))

# 剩余 < 10% 时主动 sleep
if remaining_req < 20:
    await asyncio.sleep(2)