429 Too Many Requests

# the status and any rate-limit headers
curl -s -o /dev/null -D - -x http://USER:[email protected]:8000 https://api.example.com/items \
  | grep -iE '^(HTTP|retry-after|x-ratelimit)'

# --retry retries a 429 and waits as long as Retry-After says
curl --retry 3 --retry-max-time 120 -x http://USER:[email protected]:8000 https://api.example.com/items

import random
import time
from email.utils import parsedate_to_datetime

import requests

PROXY = "http://USER:[email protected]:8000"


def pause_for(r, attempt):
    header = r.headers.get("Retry-After")
    if header is None:
        return 2 ** attempt + random.random()
    if header.isdigit():
        return int(header)
    return max(0, parsedate_to_datetime(header).timestamp() - time.time())


def get(url, tries=4):
    for attempt in range(tries):
        r = requests.get(url, proxies={"http": PROXY, "https": PROXY}, timeout=30)
        if r.status_code != 429:
            return r
        pause = pause_for(r, attempt)
        print(f"429 on {url}, waiting {pause:.1f}s")
        time.sleep(pause)
    return r

ROBOTSTXT_OBEY = True
CONCURRENT_REQUESTS_PER_DOMAIN = 2
DOWNLOAD_DELAY = 1.0
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
RETRY_HTTP_CODES = [429, 500, 502, 503, 504]
RETRY_TIMES = 3

from playwright.sync_api import Error, sync_playwright

PROXY = {"server": "http://resi.proxymonkey.io:8000", "username": "USER", "password": "PASS"}

URLS = ["https://example.com/a", "https://example.com/b"]


def log_limits(response):
    if response.status == 429:
        print("429 on", response.url, "retry-after:", response.headers.get("retry-after"))


with sync_playwright() as p:
    browser = p.chromium.launch(proxy=PROXY)
    page = browser.new_page()
    page.on("response", log_limits)
    for url in URLS:
        page.goto(url)
        page.wait_for_timeout(2000)
    browser.close()

import { setTimeout as sleep } from "node:timers/promises";
import { fetch, ProxyAgent } from "undici";

const dispatcher = new ProxyAgent("http://USER:[email protected]:8000");

async function get(url, tries = 4) {
  for (let attempt = 0; attempt < tries; attempt++) {
    const res = await fetch(url, { dispatcher });
    if (res.status !== 429) return res;
    const after = Number(res.headers.get("retry-after"));
    const pause = after > 0 ? after * 1000 : 2 ** attempt * 1000 + Math.random() * 1000;
    await res.body?.cancel();
    await sleep(pause);
  }
  throw new Error("still 429 after " + tries + " tries: " + url);
}