<?php
require_once __DIR__ . '/../config.php';

/**
 * Fetches one URL server-side. This is the whole point of this file existing:
 * a server-to-server HTTP request has no CORS restriction at all — nothing to
 * route through a public relay, nothing that can be blocked because a relay's
 * IP range is known and blocklisted. It can still be blocked by the *target*
 * site's own bot detection, same as any fetch method, but the entire class of
 * "the relay itself failed" errors this project has dealt with disappears.
 */
function crawlie_is_safe_url(string $url): bool {
    $host = parse_url($url, PHP_URL_HOST);
    if (!$host) return false;
    if (in_array(strtolower($host), ['localhost', '127.0.0.1', '0.0.0.0', '::1'])) return false;
    $ip = gethostbyname($host);
    if (filter_var($ip, FILTER_VALIDATE_IP) && filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) === false) {
        return false;
    }
    return true;
}

function crawlie_fetch(string $url, int $timeout = CRAWL_PER_REQUEST_TIMEOUT): array {
    if (!crawlie_is_safe_url($url)) {
        return ['ok' => false, 'error' => 'Blocked: Unsafe URL or private IP', 'url' => $url];
    }
    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_MAXREDIRS => 5,
        CURLOPT_TIMEOUT => $timeout,
        CURLOPT_CONNECTTIMEOUT => min(8, $timeout),
        CURLOPT_SSL_VERIFYPEER => true,
        CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; CrawlieBot/1.0; +https://www.crawlie.app)',
        CURLOPT_HTTPHEADER => ['Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'],
    ]);
    $body = curl_exec($ch);
    $info = curl_getinfo($ch);
    $error = curl_error($ch);
    curl_close($ch);

    if ($body === false) {
        return ['ok' => false, 'error' => $error ?: 'curl error', 'url' => $url];
    }
    $code = $info['http_code'] ?? 0;
    if ($code < 200 || $code >= 400) {
        return ['ok' => false, 'error' => 'HTTP ' . $code, 'url' => $url, 'status' => $code];
    }
    return ['ok' => true, 'html' => $body, 'url' => $info['url'] ?: $url, 'status' => $code];
}

/**
 * Fetches many URLs concurrently using curl_multi — this is the real speed win
 * over the browser's public-relay approach: dozens of pages fetched in
 * parallel from the server, no per-relay rate limits, no per-page browser
 * round trip. Returns results in the same order as $urls.
 */
function crawlie_fetch_many(array $urls, int $concurrency = CRAWL_CONCURRENCY, int $timeout = CRAWL_PER_REQUEST_TIMEOUT): array {
    $results = array_fill(0, count($urls), null);
    $mh = curl_multi_init();
    $handles = [];
    $queue = array_values($urls);
    $indexByHandleId = [];
    $nextIndex = 0;

    $addHandle = function () use (&$queue, &$nextIndex, &$handles, &$indexByHandleId, &$results, $mh, $timeout) {
        if ($nextIndex >= count($queue)) return false;
        $i = $nextIndex++;
        if (!crawlie_is_safe_url($queue[$i])) {
            $results[$i] = ['ok' => false, 'error' => 'Blocked: Unsafe URL or private IP', 'url' => $queue[$i]];
            return true; // We pretend we added it but we just filled the result immediately
        }
        $ch = curl_init($queue[$i]);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 5,
            CURLOPT_TIMEOUT => $timeout,
            CURLOPT_CONNECTTIMEOUT => min(8, $timeout),
            CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; CrawlieBot/1.0; +https://www.crawlie.app)',
        ]);
        curl_multi_add_handle($mh, $ch);
        $handles[(int) $ch] = $ch;
        $indexByHandleId[(int) $ch] = $i;
        return true;
    };

    for ($n = 0; $n < $concurrency; $n++) { if (!$addHandle()) break; }

    $running = null;
    do {
        curl_multi_exec($mh, $running);
        curl_multi_select($mh, 1.0);
        while ($info = curl_multi_info_read($mh)) {
            $ch = $info['handle'];
            $id = (int) $ch;
            $i = $indexByHandleId[$id];
            $body = curl_multi_getcontent($ch);
            $httpInfo = curl_getinfo($ch);
            $err = curl_error($ch);
            $code = $httpInfo['http_code'] ?? 0;
            if ($body !== false && $body !== '' && $code >= 200 && $code < 400) {
                $results[$i] = ['ok' => true, 'html' => $body, 'url' => $httpInfo['url'] ?: $queue[$i], 'status' => $code];
            } else {
                $results[$i] = ['ok' => false, 'error' => $err ?: ('HTTP ' . $code), 'url' => $queue[$i]];
            }
            curl_multi_remove_handle($mh, $ch);
            curl_close($ch);
            unset($handles[$id]);
            $addHandle();
        }
    } while ($running > 0 || count($handles) > 0);

    curl_multi_close($mh);
    return $results;
}

/** Mirrors extractPageDetails() in tools/rss-feed-generator.html. */
function crawlie_extract_page_details(string $html, string $pageUrl): array {
    $doc = new DOMDocument();
    libxml_use_internal_errors(true);
    $doc->loadHTML('<?xml encoding="utf-8"?>' . $html);
    libxml_clear_errors();
    $xpath = new DOMXPath($doc);

    $titleNode = $doc->getElementsByTagName('title')->item(0);
    $title = $titleNode ? trim($titleNode->textContent) : $pageUrl;

    $description = '';
    foreach (['description', 'og:description'] as $key) {
        $attr = $key === 'description' ? 'name' : 'property';
        $nodes = $xpath->query("//meta[translate(@{$attr}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$key}']");
        if ($nodes->length) { $description = trim($nodes->item(0)->getAttribute('content')); break; }
    }

    $pubDate = null;
    $timeMetaNodes = $xpath->query("//meta[translate(@property, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='article:published_time' or translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='date']");
    if ($timeMetaNodes->length) $pubDate = $timeMetaNodes->item(0)->getAttribute('content');
    if (!$pubDate) {
        $timeEls = $doc->getElementsByTagName('time');
        if ($timeEls->length && $timeEls->item(0)->getAttribute('datetime')) $pubDate = $timeEls->item(0)->getAttribute('datetime');
    }
    if (!$pubDate) {
        foreach ($doc->getElementsByTagName('script') as $script) {
            if (strtolower($script->getAttribute('type')) !== 'application/ld+json') continue;
            $data = json_decode($script->textContent, true);
            if (!$data) continue;
            $nodes = isset($data['@graph']) ? $data['@graph'] : (array_is_list($data) ? $data : [$data]);
            foreach ($nodes as $node) { if (!empty($node['datePublished'])) { $pubDate = $node['datePublished']; break 2; } }
        }
    }
    $ts = $pubDate ? strtotime($pubDate) : false;
    $pubDateStr = $ts !== false ? gmdate('D, d M Y H:i:s \G\M\T', $ts) : gmdate('D, d M Y H:i:s \G\M\T');

    return ['url' => $pageUrl, 'title' => $title, 'description' => $description, 'pubDate' => $pubDateStr];
}

const CRAWLIE_AVAILABILITY_MAP = ['instock' => 'in_stock', 'outofstock' => 'out_of_stock', 'preorder' => 'preorder', 'backorder' => 'backorder', 'discontinued' => 'out_of_stock'];
const CRAWLIE_CONDITION_MAP = ['newcondition' => 'new', 'usedcondition' => 'used', 'refurbishedcondition' => 'refurbished', 'damagedcondition' => 'used'];

/** Mirrors extractProductIfAny() in tools/merchant-feed-generator.html. Returns null for
 *  non-product pages so they're excluded from the feed, same as the client-side logic. */
function crawlie_extract_product_if_any(string $html, string $pageUrl): ?array {
    $doc = new DOMDocument();
    libxml_use_internal_errors(true);
    $doc->loadHTML('<?xml encoding="utf-8"?>' . $html);
    libxml_clear_errors();
    $xpath = new DOMXPath($doc);

    $out = ['title' => '', 'description' => '', 'price' => '', 'currency' => '', 'brand' => '', 'image' => '', 'availability' => '', 'condition' => '', 'sku' => ''];
    $foundStructured = false;

    foreach ($doc->getElementsByTagName('script') as $script) {
        if (strtolower($script->getAttribute('type')) !== 'application/ld+json') continue;
        $data = json_decode($script->textContent, true);
        if (!$data) continue;
        $nodes = isset($data['@graph']) ? $data['@graph'] : (array_is_list($data) ? $data : [$data]);
        foreach ($nodes as $node) {
            if (!is_array($node) || empty($node['@type']) || stripos((string) $node['@type'], 'product') === false) continue;
            $foundStructured = true;
            $out['title'] = $out['title'] ?: ($node['name'] ?? '');
            $out['description'] = $out['description'] ?: ($node['description'] ?? '');
            $brand = $node['brand'] ?? '';
            $out['brand'] = $out['brand'] ?: (is_array($brand) ? ($brand['name'] ?? '') : $brand);
            $image = $node['image'] ?? '';
            $out['image'] = $out['image'] ?: (array_is_list((array) $image) && is_array($image) ? ($image[0] ?? '') : $image);
            $out['sku'] = $out['sku'] ?: ($node['sku'] ?? $node['mpn'] ?? '');
            if (!empty($node['offers'])) {
                $offer = array_is_list($node['offers']) ? $node['offers'][0] : $node['offers'];
                $out['price'] = $out['price'] ?: ($offer['price'] ?? $offer['lowPrice'] ?? '');
                $out['currency'] = $out['currency'] ?: ($offer['priceCurrency'] ?? '');
                if (!empty($offer['availability'])) $out['availability'] = $out['availability'] ?: strtolower(str_replace('https://schema.org/', '', $offer['availability']));
                if (!empty($offer['itemCondition'])) $out['condition'] = $out['condition'] ?: strtolower(str_replace('https://schema.org/', '', $offer['itemCondition']));
            }
        }
    }

    $meta = function (string $attr, string $key) use ($xpath) {
        $nodes = $xpath->query("//meta[translate(@{$attr}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$key}']");
        return $nodes->length ? trim($nodes->item(0)->getAttribute('content')) : '';
    };
    $ogType = strtolower($meta('property', 'og:type'));
    $out['title'] = $out['title'] ?: $meta('property', 'og:title');
    $out['description'] = $out['description'] ?: ($meta('name', 'description') ?: $meta('property', 'og:description'));
    $out['image'] = $out['image'] ?: $meta('property', 'og:image');
    $out['price'] = $out['price'] ?: ($meta('property', 'product:price:amount') ?: $meta('itemprop', 'price'));
    $out['currency'] = $out['currency'] ?: $meta('property', 'product:price:currency');
    $out['brand'] = $out['brand'] ?: $meta('property', 'product:brand');
    if (!$out['availability']) { $av = strtolower($meta('property', 'product:availability')); if ($av) $out['availability'] = $av; }

    if (!$out['title']) {
        $h1 = $doc->getElementsByTagName('h1')->item(0);
        if ($h1) $out['title'] = trim($h1->textContent);
    }

    $looksLikeProduct = $foundStructured || $ogType === 'product' || $out['price'] !== '';
    if (!$looksLikeProduct || !$out['title'] || !$out['price']) return null;

    return [
        'id' => $out['sku'] ?: crawlie_slug_id($pageUrl),
        'title' => $out['title'],
        'description' => $out['description'] ?: $out['title'],
        'link' => $pageUrl,
        'image_link' => $out['image'],
        'price' => $out['price'] . ' ' . ($out['currency'] ?: 'USD'),
        'brand' => $out['brand'],
        'condition' => CRAWLIE_CONDITION_MAP[$out['condition']] ?? 'new',
        'availability' => CRAWLIE_AVAILABILITY_MAP[$out['availability']] ?? 'in_stock',
    ];
}

function crawlie_slug_id(string $url): string {
    $path = trim((string) parse_url($url, PHP_URL_PATH), '/');
    $parts = explode('/', $path);
    return end($parts) ?: (string) parse_url($url, PHP_URL_HOST);
}

function crawlie_body_text_length(string $html): int {
    $doc = new DOMDocument();
    libxml_use_internal_errors(true);
    $doc->loadHTML('<?xml encoding="utf-8"?>' . $html);
    libxml_clear_errors();
    $body = $doc->getElementsByTagName('body')->item(0);
    return $body ? strlen(trim(preg_replace('/\s+/', ' ', $body->textContent))) : 0;
}

function crawlie_normalize_url(string $raw): string {
    $v = trim($raw);
    if (!preg_match('#^https?://#i', $v)) $v = 'https://' . $v;
    return $v;
}

/** Same asset-extension skip list as the client-side extractInternalLinks, kept in sync. */
const CRAWLIE_ASSET_EXT_RE = '/\.(jpg|jpeg|png|gif|svg|webp|ico|css|js|mjs|pdf|zip|rar|7z|mp4|mp3|wav|avi|mov|woff2?|ttf|eot|xml|json|txt|csv|doc|docx|xls|xlsx)$/i';

function crawlie_extract_internal_links(string $html, string $pageUrl): array {
    $host = parse_url($pageUrl, PHP_URL_HOST);
    if (!$host) return [];
    $host = preg_replace('/^www\./i', '', $host);

    $doc = new DOMDocument();
    libxml_use_internal_errors(true);
    $doc->loadHTML('<?xml encoding="utf-8"?>' . $html);
    libxml_clear_errors();

    $results = [];
    $seen = [];
    foreach ($doc->getElementsByTagName('a') as $a) {
        $href = $a->getAttribute('href');
        if (!$href || preg_match('~^(mailto:|tel:|javascript:|#)~i', $href)) continue;
        $abs = crawlie_resolve_url($href, $pageUrl);
        if (!$abs) continue;
        $scheme = parse_url($abs, PHP_URL_SCHEME);
        if (!in_array($scheme, ['http', 'https'], true)) continue;
        $linkHost = preg_replace('/^www\./i', '', (string) parse_url($abs, PHP_URL_HOST));
        if (strcasecmp($linkHost, $host) !== 0) continue;
        $path = (string) parse_url($abs, PHP_URL_PATH);
        if (preg_match(CRAWLIE_ASSET_EXT_RE, $path)) continue;
        $normalized = preg_replace('/#.*$/', '', $abs);
        $normalized = rtrim($normalized, '/');
        if ($normalized === '' || isset($seen[$normalized])) continue;
        $seen[$normalized] = true;
        $results[] = $normalized;
    }
    return $results;
}

function crawlie_resolve_url(string $href, string $base): ?string {
    if (preg_match('#^https?://#i', $href)) return $href;
    $b = parse_url($base);
    if (!$b) return null;
    $scheme = $b['scheme'] ?? 'https';
    $host = $b['host'] ?? '';
    $port = isset($b['port']) ? ':' . $b['port'] : '';
    if (str_starts_with($href, '//')) return $scheme . ':' . $href;
    if (str_starts_with($href, '/')) return $scheme . '://' . $host . $port . $href;
    $basePath = $b['path'] ?? '/';
    $dir = substr($basePath, 0, strrpos($basePath, '/') + 1);
    $combined = $dir . $href;
    // Collapse ./ and ../ segments.
    $segments = [];
    foreach (explode('/', $combined) as $seg) {
        if ($seg === '.' || $seg === '') continue;
        if ($seg === '..') { array_pop($segments); continue; }
        $segments[] = $seg;
    }
    return $scheme . '://' . $host . $port . '/' . implode('/', $segments);
}

/** Robots.txt + sitemap.xml/sitemap-index discovery — the server-side twin of
 *  CrawlieEngine.discoverSitemapUrls() in assets/audit-engine.js. Kept logically
 *  identical so both paths behave the same; this one just runs without any
 *  CORS/relay layer in between. */
function crawlie_discover_sitemap_urls(string $siteUrl, int $maxSitemaps = 20, int $maxUrls = 5000): array {
    $origin = crawlie_origin($siteUrl);
    if (!$origin) return ['urls' => [], 'source' => null, 'sitemapUrls' => []];

    $candidates = [];
    $robotsRes = crawlie_fetch($origin . '/robots.txt', 8);
    $robotsFound = $robotsRes['ok'];
    if ($robotsFound) {
        foreach (preg_split('/\r?\n/', $robotsRes['html']) as $line) {
            if (preg_match('/^\s*sitemap\s*:\s*(\S+)/i', $line, $m)) $candidates[] = trim($m[1]);
        }
    }
    if (!$candidates) {
        $candidates[] = $origin . '/sitemap.xml';
        $candidates[] = $origin . '/sitemap_index.xml';
    }

    $seen = [];
    $queue = $candidates;
    $urls = [];
    $usedSitemaps = [];
    $declaredInRobots = $robotsFound && count($candidates) > 0;

    while ($queue && count($usedSitemaps) < $maxSitemaps && count($urls) < $maxUrls) {
        $sUrl = array_shift($queue);
        if (isset($seen[$sUrl])) continue;
        $seen[$sUrl] = true;
        $res = crawlie_fetch($sUrl, 8);
        if (!$res['ok']) continue;
        $parsed = crawlie_parse_sitemap_xml($res['html']);
        if (!$parsed) continue;
        $usedSitemaps[] = $sUrl;
        if ($parsed['type'] === 'sitemapindex') {
            foreach ($parsed['locs'] as $l) { if (!isset($seen[$l])) $queue[] = $l; }
        } else {
            foreach ($parsed['locs'] as $l) { if (count($urls) < $maxUrls) $urls[] = $l; }
        }
    }

    return [
        'urls' => $urls,
        'source' => $usedSitemaps ? ($declaredInRobots ? 'robots.txt' : 'sitemap.xml') : null,
        'sitemapUrls' => $usedSitemaps,
    ];
}

function crawlie_parse_sitemap_xml(string $xml): ?array {
    libxml_use_internal_errors(true);
    $doc = simplexml_load_string($xml);
    libxml_clear_errors();
    if (!$doc) return null;
    $name = $doc->getName();
    $locs = [];
    if ($name === 'sitemapindex') {
        foreach ($doc->sitemap as $s) { if (isset($s->loc)) $locs[] = trim((string) $s->loc); }
        if (!$locs) return null;
        return ['type' => 'sitemapindex', 'locs' => $locs];
    }
    if ($name === 'urlset') {
        foreach ($doc->url as $u) { if (isset($u->loc)) $locs[] = trim((string) $u->loc); }
        if (!$locs) return null;
        return ['type' => 'urlset', 'locs' => $locs];
    }
    return null;
}

function crawlie_origin(string $url): ?string {
    $p = parse_url($url);
    if (!$p || !isset($p['scheme'], $p['host'])) return null;
    return $p['scheme'] . '://' . $p['host'] . (isset($p['port']) ? ':' . $p['port'] : '');
}

/**
 * Full breadth-first crawl using real concurrency (curl_multi) — discovers
 * every reachable page from a seed URL. Used as the fallback when no sitemap
 * exists. Because there's no per-relay rate limit or blocklisted-IP failure
 * mode here, this reliably finds everything reachable rather than losing
 * pages to sporadic relay failures, which is the actual "missing links" cause
 * in the pure client-side version.
 */
function crawlie_crawl_site(string $seedUrl, string $seedHtml, int $maxPages, float $deadline): array {
    $visited = [$seedUrl => true];
    $queue = crawlie_extract_internal_links($seedHtml, $seedUrl);
    $queue = array_values(array_diff($queue, array_keys($visited)));
    $failedCount = 0;

    while ($queue && count($visited) < $maxPages && microtime(true) < $deadline) {
        $room = $maxPages - count($visited);
        $batch = array_splice($queue, 0, min(CRAWL_CONCURRENCY, count($queue), $room));
        $results = crawlie_fetch_many($batch, CRAWL_CONCURRENCY);
        foreach ($results as $i => $res) {
            $url = $batch[$i];
            if (isset($visited[$url])) continue;
            $visited[$url] = true;
            if ($res['ok']) {
                foreach (crawlie_extract_internal_links($res['html'], $res['url']) as $l) {
                    if (!isset($visited[$l]) && !in_array($l, $queue, true)) $queue[] = $l;
                }
            } else {
                $failedCount++;
            }
        }
    }

    return ['urls' => array_keys($visited), 'failedCount' => $failedCount, 'hitDeadline' => microtime(true) >= $deadline];
}
