/**
 * Crawlie fetch backend — Cloudflare Worker
 * ------------------------------------------
 * Server-side fetches don't hit CORS at all, and using a real browser
 * User-Agent gets past many of the basic bot checks that block the public
 * CORS relays (which are well-known and often IP-blocklisted outright).
 *
 * DEPLOY (2 minutes, free):
 *   1. Go to https://dash.cloudflare.com -> Workers & Pages -> Create -> Worker.
 *   2. Delete the sample code and paste this entire file in.
 *   3. Click Deploy. You'll get a URL like https://crawlie-fetch.<you>.workers.dev
 *   4. Paste that URL into Crawlie's "Use your own fetch backend" box
 *      (shown on the report page when a scan fails) and save.
 *
 * Free tier: 100,000 requests/day — far more than a personal audit tool needs.
 *
 * HONEST LIMITS: this performs a plain HTTP fetch, not a real browser. Sites
 * using heavy JS-challenge bot protection (Cloudflare "Under Attack" mode,
 * full browser fingerprinting, etc.) can still block even this. There is no
 * way to guarantee every site is fetchable without running a real headless
 * browser somewhere — but this covers the large majority of sites that were
 * blocking the public relays for simpler reasons (missing/generic headers,
 * blocklisted relay IPs, no User-Agent).
 */
export default {
  async fetch(request) {
    const corsHeaders = {
      'Access-Control-Allow-Origin': '*',
      'Access-Control-Allow-Methods': 'GET, OPTIONS',
      'Access-Control-Allow-Headers': 'Content-Type',
    };

    if (request.method === 'OPTIONS') {
      return new Response(null, { headers: corsHeaders });
    }

    const { searchParams } = new URL(request.url);
    const target = searchParams.get('url');
    if (!target) {
      return new Response('Missing ?url= parameter', { status: 400, headers: corsHeaders });
    }

    let targetUrl;
    try {
      targetUrl = new URL(target);
    } catch (e) {
      return new Response('Invalid URL', { status: 400, headers: corsHeaders });
    }
    if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') {
      return new Response('Only http/https URLs are allowed', { status: 400, headers: corsHeaders });
    }

    try {
      const upstream = await fetch(targetUrl.toString(), {
        redirect: 'follow',
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.9',
        },
        cf: { cacheTtl: 0 },
      });

      const html = await upstream.text();
      return new Response(html, {
        status: upstream.status,
        headers: { ...corsHeaders, 'Content-Type': 'text/html; charset=utf-8' },
      });
    } catch (e) {
      return new Response('Upstream fetch failed: ' + (e && e.message ? e.message : 'unknown error'), {
        status: 502,
        headers: corsHeaders,
      });
    }
  },
};

