<?php
/**
 * Full-site crawl endpoint — the core "faster, no missing links" piece.
 *
 *   GET /backend-php/api/crawl.php?url=<site>&mode=discover   -> just the URL list
 *   GET /backend-php/api/crawl.php?url=<site>&mode=posts      -> title/description/date per page (for RSS)
 *   GET /backend-php/api/crawl.php?url=<site>&mode=products   -> detected products only (for the Merchant feed tool)
 *
 * Runs entirely server-side: sitemap-first discovery (robots.txt -> sitemap.xml /
 * sitemap index), falling back to a real breadth-first crawl with curl_multi
 * concurrency if no sitemap exists. Because this never goes through a public
 * CORS relay, none of the "relay was blocked/rate-limited" failures that cause
 * missing links in the pure-browser version apply here — every page that's
 * genuinely reachable gets fetched, in one request instead of dozens of
 * separate round trips from the browser.
 */
require_once __DIR__ . '/../includes/crawler.php';
require_once __DIR__ . '/../includes/auth.php';
send_cors_headers();

$target = $_GET['url'] ?? '';
$mode = $_GET['mode'] ?? 'discover';
if ($target === '') json_response(['error' => 'Missing ?url= parameter'], 400);
if (!in_array($mode, ['discover', 'posts', 'products'], true)) json_response(['error' => 'mode must be discover, posts, or products'], 400);

$startedAt = microtime(true);
$deadline = $startedAt + CRAWL_TOTAL_TIME_BUDGET;
$siteUrl = crawlie_normalize_url($target);

$sitemap = crawlie_discover_sitemap_urls($siteUrl);
$source = $sitemap['source'];
$urls = $sitemap['urls'];
$failedCount = 0;

if (!$source) {
    // No sitemap — fall back to a real crawl from the homepage.
    $seed = crawlie_fetch($siteUrl);
    if (!$seed['ok']) {
        json_response(['error' => 'Could not fetch ' . $siteUrl . ': ' . $seed['error']], 502);
    }
    $crawl = crawlie_crawl_site($seed['url'], $seed['html'], CRAWL_MAX_PAGES, $deadline);
    $urls = $crawl['urls'];
    $failedCount = $crawl['failedCount'];
    $source = 'crawl';
}

$urls = array_slice($urls, 0, CRAWL_MAX_PAGES);

$response = [
    'source' => $source,
    'pagesScanned' => count($urls),
];

if ($mode === 'discover') {
    $response['urls'] = $urls;
} else {
    // posts/products modes need each page's content, not just its URL.
    $fetchResults = crawlie_fetch_many($urls, CRAWL_CONCURRENCY);
    $items = [];
    $thinCount = 0;
    foreach ($fetchResults as $i => $res) {
        if (!$res['ok']) { $failedCount++; continue; }
        if ($mode === 'posts') {
            $items[] = crawlie_extract_page_details($res['html'], $res['url']);
        } else {
            $product = crawlie_extract_product_if_any($res['html'], $res['url']);
            if ($product) $items[] = $product;
            if (crawlie_body_text_length($res['html']) < 200) $thinCount++;
        }
    }
    if ($mode === 'products' && count($items) === 0 && count($urls) >= 3 && ($thinCount / max(1, count($urls))) >= 0.7) {
        $response['likelyJsRendered'] = true;
    }
    $response['items'] = $items;
}

$response['failedCount'] = $failedCount;
$response['durationMs'] = (int) round((microtime(true) - $startedAt) * 1000);

// Best-effort activity log — never lets a DB hiccup break the actual crawl result.
try {
    require_once __DIR__ . '/../includes/db.php';
    $stmt = db()->prepare('INSERT INTO crawl_jobs (target_url, mode, source, pages_scanned, items_found, duration_ms) VALUES (?, ?, ?, ?, ?, ?)');
    $stmt->execute([$siteUrl, $mode, $source, count($urls), $mode === 'discover' ? count($urls) : count($response['items'] ?? []), $response['durationMs']]);
} catch (Throwable $e) {
    // Ignored on purpose — logging is a nice-to-have, not a dependency of the crawl itself.
}

json_response($response);
