403 Forbidden

URL=https://example.com/page

# through the proxy: the status, then who answered
curl -s -o block.html -D - -x http://USER:[email protected]:8000 "$URL" \
  | grep -iE '^(HTTP|server|cf-ray|x-amz-cf-id|content-type)'

# the same request without the proxy
curl -s -o /dev/null -w 'direct: %{http_code}\n' "$URL"

import requests

URL = "https://example.com/page"
PROXY = "http://USER:[email protected]:8000"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
    "Accept": "text/html,application/xhtml+xml,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

for label, proxies in [("proxy", {"http": PROXY, "https": PROXY}), ("direct", None)]:
    r = requests.get(URL, headers=HEADERS, proxies=proxies, timeout=30)
    print(label, r.status_code, r.headers.get("server"), r.text[:200])

# settings.py
ROBOTSTXT_OBEY = True
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"


# spiders/page.py
import scrapy


class PageSpider(scrapy.Spider):
    name = "page"
    handle_httpstatus_list = [403]
    start_urls = ["https://example.com/page"]

    def parse(self, response):
        if response.status == 403:
            self.logger.warning("403 at %s, server=%s", response.url, response.headers.get("Server"))
            with open("block.html", "wb") as f:
                f.write(response.body)

from playwright.sync_api import Error, sync_playwright

PROXY = {"server": "http://resi.proxymonkey.io:8000", "username": "USER", "password": "PASS"}

with sync_playwright() as p:
    browser = p.chromium.launch(proxy=PROXY)
    page = browser.new_page()
    response = page.goto("https://example.com/page")
    print(response.status, response.headers.get("server"), page.title())
    if response.status == 403:
        page.screenshot(path="block.png", full_page=True)
    browser.close()

import { writeFile } from "node:fs/promises";
import { fetch, ProxyAgent } from "undici";

const dispatcher = new ProxyAgent("http://USER:[email protected]:8000");

const res = await fetch("https://example.com/page", {
  dispatcher,
  headers: {
    "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
    "accept-language": "en-US,en;q=0.9",
  },
});

console.log(res.status, res.headers.get("server"), res.headers.get("cf-ray"));
if (res.status === 403) await writeFile("block.html", await res.text());