/*
 * Crawlie audit engine — runs entirely in the browser.
 * Given raw HTML (fetched or pasted) it extracts on-page signals and
 * turns them into category scores + a ranked issue list.
 * No data ever leaves the browser except the one request needed to
 * fetch the target page's HTML (via a public read-only CORS relay).
 */
(function (global) {

  var STOPWORDS = ("a an the and or but if of at by for with about against between into "
    + "through during before after above below to from up down in out on off over under "
    + "again further then once here there when where why how all any both each few more "
    + "most other some such no nor not only own same so than too very s t can will just don "
    + "should now is are was were be been being have has had do does did this that these those "
    + "it its it's you your yours we our ours they their theirs he she him her his i me my mine as").split(/\s+/);
  var STOP = {}; STOPWORDS.forEach(function (w) { STOP[w] = true; });

  // A live URL fetch from a static, backend-less site has to go through a public CORS relay
  // (the target server almost never sends the headers a browser needs to allow it directly).
  // We try several, in order, because any single one can rate-limit or block a given site.
  //
  // NOTE (2026-07): corsproxy.io's free tier now only serves requests from localhost,
  // *.github.io, ngrok/trycloudflare tunnels, and similar dev origins — it returns a
  // hard 403 for any real deployed site's origin, which is exactly this project's case.
  // That's not a bug here to fix; it's a policy change on their end, so it's been
  // replaced with corsfix.com (still free, no key, works from any origin as of this
  // writing) rather than left in as permanent dead weight.
  var PROXIES = [
    { name: 'direct', build: function (u) { return u; }, json: false }
  ];

  // Session-level circuit breaker: a relay that fails several times in a row for this
  // site is very likely blocked outright (not just having a bad moment), so skip it for
  // subsequent pages instead of paying its full timeout on every single fetch during a
  // long crawl. This is the main lever for crawl speed — most of the time lost during a
  // multi-page crawl is waiting out timeouts on relays that were never going to work for
  // this particular site. Resets each page load.
  var FAILURE_STREAK_LIMIT = 2;
  var proxyFailureStreak = {};
  function activeProxies() {
    var enabled = PROXIES.filter(function (p) { return (proxyFailureStreak[p.name] || 0) < FAILURE_STREAK_LIMIT; });
    // Safety: never fully lock ourselves out — if everything has tripped the breaker,
    // reset and give them all another chance rather than permanently failing.
    if (enabled.length < 2) { proxyFailureStreak = {}; return PROXIES; }
    return enabled;
  }
  function recordProxyResult(name, ok) {
    proxyFailureStreak[name] = ok ? 0 : (proxyFailureStreak[name] || 0) + 1;
  }

  function normalizeUrl(raw) {
    var v = (raw || '').trim();
    if (!v) return '';
    if (!/^https?:\/\//i.test(v)) v = 'https://' + v;
    return v;
  }

  // Public CORS relays fail "successfully" surprisingly often — they return their own
  // rate-limit, captcha, or "access denied" page as a normal 200 OK HTML response.
  // Without this check that error page gets analyzed as if it were the real site, which
  // is why every scan could end up producing the same generic result. Reject anything
  // that looks like one of these before it reaches the analyzer.
  var BLOCK_SIGNATURES = [
    'rate limit', 'too many requests', 'access denied', 'checking your browser',
    'attention required', 'cf-error', 'cf-browser-verification', 'captcha',
    'blocked by', '403 forbidden', 'quota exceeded', 'please slow down',
    'you have exceeded', 'proxy error', 'bad gateway', 'origin is unreachable',
    'cors anywhere', 'requests limit', 'please try again later', 'permission denied',
    'unusual traffic', 'automated requests', 'enable javascript and cookies'
  ];
  function looksBlocked(html, targetHost, expectHtml) {
    if (!html || html.length < (expectHtml === false ? 5 : 500)) return true;
    var lower = html.toLowerCase();
    for (var i = 0; i < BLOCK_SIGNATURES.length; i++) {
      if (lower.indexOf(BLOCK_SIGNATURES[i]) !== -1) return true;
    }
    // A genuine page almost always references its own host somewhere (links, canonical,
    // assets). A relay's own error/landing page never will. Only applied when there's
    // enough text to make the absence meaningful, so short-but-real pages aren't punished.
    // Skipped for non-HTML resources (robots.txt, sitemap.xml) since a sitemap can
    // legitimately be hosted on a different subdomain/CDN than the page host.
    if (expectHtml !== false && targetHost && html.length < 4000 && lower.indexOf(targetHost.toLowerCase()) === -1) return true;
    return false;
  }

  function getCustomBackend() {
    // try { return (localStorage.getItem('crawlie_backend_url') || '/backend-php/api/fetch.php').trim(); } catch (e) { return '/backend-php/api/fetch.php'; }
    return 'http://localhost/crawler/site/backend-php/api/fetch.php';
  }
  function setCustomBackend(url) {
    try {
      if (url) localStorage.setItem('crawlie_backend_url', url);
      else localStorage.removeItem('crawlie_backend_url');
    } catch (e) {}
  }

  async function raceProxies(proxies, url, host, perTry, expectHtml) {
    function attempt(proxy) {
      return new Promise(function (resolve) {
        var controller = new AbortController();
        var timer = setTimeout(function () { controller.abort(); }, perTry);
        var started = performance.now();
        fetch(proxy.build(url), { signal: controller.signal, redirect: 'follow' })
          .then(function (res) {
            if (!res.ok) { recordProxyResult(proxy.name, false); return resolve({ ok: false, error: proxy.name + ': HTTP ' + res.status }); }
            return (proxy.json ? res.json() : res.text()).then(function (data) {
              var html = proxy.json ? (data && (data.contents || data.body || '')) : data;
              var elapsed = Math.round(performance.now() - started);
              var minLen = expectHtml === false ? 1 : 40;
              if (!html || html.length < minLen) { recordProxyResult(proxy.name, false); return resolve({ ok: false, error: proxy.name + ': empty response' }); }
              if (expectHtml !== false && !/<html|<HTML|<!doctype html/i.test(html)) { recordProxyResult(proxy.name, false); return resolve({ ok: false, error: proxy.name + ': response was not HTML' }); }
              if (looksBlocked(html, host, expectHtml)) { recordProxyResult(proxy.name, false); return resolve({ ok: false, error: proxy.name + ': relay returned a blocked/rate-limit page, not the real site' }); }
              recordProxyResult(proxy.name, true);
              resolve({ ok: true, html: html, url: url, fetchMs: elapsed, via: proxy.name });
            });
          })
          .catch(function (e) {
            recordProxyResult(proxy.name, false);
            var reason = e && e.name === 'AbortError' ? 'timed out after ' + Math.round(perTry / 1000) + 's' : (e && e.message ? e.message : 'network error');
            resolve({ ok: false, error: proxy.name + ': ' + reason });
          })
          .finally(function () { clearTimeout(timer); });
      });
    }

    var pending = proxies.map(attempt);
    var errors = [];
    for (var remaining = pending.length; remaining > 0; remaining--) {
      var result = await Promise.race(pending.map(function (p, idx) {
        return p.then(function (r) { return { r: r, idx: idx }; });
      }));
      if (result.r.ok) return result.r;
      errors.push(result.r.error);
      pending.splice(result.idx, 1);
      if (!pending.length) break;
    }
    return { ok: false, errors: errors };
  }

  async function fetchPage(rawUrl, timeoutMs, opts) {
    opts = opts || {};
    var url = normalizeUrl(rawUrl);
    var host = ''; try { host = new URL(url).hostname.replace(/^www\./, ''); } catch (e) {}
    // 12s per attempt (was 7s): several public relays (especially under their own rate
    // limits) can genuinely take longer than 7s to respond even when they'd have
    // eventually succeeded. Since every candidate races in parallel rather than one
    // after another, raising this doesn't multiply total wait time by the number of
    // relays — it just gives slower-but-working ones a real chance before we give up.
    var perTry = timeoutMs || 12000;
    var custom = getCustomBackend();
    var proxiesToTry = activeProxies();

    // The custom backend (if configured and not currently circuit-broken) races IN
    // PARALLEL with the public relays, not before them. Racing them together means a
    // healthy backend still wins quickly (it's usually the fastest option), but a backend
    // that's currently failing for this site doesn't cost every single page an extra full
    // timeout before the public relays even get a chance to run.
    if (custom && (proxyFailureStreak['your-backend'] || 0) < FAILURE_STREAK_LIMIT) {
      var sep = custom.indexOf('?') > -1 ? '&' : '?';
      var customProxy = { name: 'your-backend', build: function (u) { return custom + sep + 'url=' + encodeURIComponent(u); }, json: false };
      proxiesToTry = [customProxy].concat(proxiesToTry);
    }

    var result = await raceProxies(proxiesToTry, url, host, perTry, opts.expectHtml);
    if (result.ok) return result;

    return {
      ok: false, url: url,
      error: result.errors[result.errors.length - 1] || 'Could not fetch this URL from the browser.',
      attempts: result.errors, attemptCount: proxiesToTry.length
    };
  }

  function textWordCount(str) {
    var words = (str || '').trim().split(/\s+/).filter(Boolean);
    return words.length;
  }

  function countSyllables(word) {
    word = word.toLowerCase().replace(/[^a-z]/g, '');
    if (!word) return 0;
    if (word.length <= 3) return 1;
    word = word.replace(/(?:[^laeiouy]es|ed|[^laeiouy]e)$/, '');
    word = word.replace(/^y/, '');
    var m = word.match(/[aeiouy]{1,2}/g);
    return m ? m.length : 1;
  }

  // Fraction of letters in the text that are Latin-script (covers English and most
  // European languages). Used to decide whether the English-specific Flesch/syllable
  // model is meaningful, or whether we should fall back to a script-neutral estimate.
  function latinRatio(text) {
    var letters = text.match(/\p{L}/gu) || [];
    if (!letters.length) return 1;
    var latin = letters.filter(function (ch) { return /[A-Za-z\u00C0-\u024F]/.test(ch); }).length;
    return latin / letters.length;
  }

  function readability(text) {
    text = text || '';
    // Sentence terminators across scripts: . ! ? plus Devanagari danda (।॥) and CJK full-width punctuation.
    var sentences = (text.match(/[.!?।॥。！？]+(\s|$)/gu) || []).length || 1;
    // \p{L}\p{N} matches letters/numbers in ANY script (Tamil, Hindi, Arabic, CJK, etc.),
    // not just Latin — the previous [A-Za-z'-]+ pattern silently produced 0 words for
    // any non-English text.
    var words = (text.match(/[\p{L}\p{N}'-]+/gu) || []);
    var wc = words.length || 1;
    var avgWordsPerSentence = Math.round((wc / sentences) * 10) / 10;
    var isLatin = latinRatio(text) > 0.6;

    if (isLatin) {
      var syllables = words.reduce(function (a, w) { return a + countSyllables(w); }, 0);
      var flesch = 206.835 - 1.015 * (wc / sentences) - 84.6 * (syllables / wc);
      flesch = Math.max(0, Math.min(100, Math.round(flesch)));
      var grade;
      if (flesch >= 90) grade = '5th grade — very easy';
      else if (flesch >= 80) grade = '6th grade — easy';
      else if (flesch >= 70) grade = '7th grade — fairly easy';
      else if (flesch >= 60) grade = '8th–9th grade — plain English';
      else if (flesch >= 50) grade = '10th–12th grade — fairly difficult';
      else if (flesch >= 30) grade = 'College — difficult';
      else grade = 'College graduate — very difficult';
      return { sentences: sentences, words: wc, syllables: syllables, flesch: flesch, grade: grade,
        avgWordsPerSentence: avgWordsPerSentence, method: 'flesch' };
    }

    // Script-neutral fallback for non-Latin text: the Flesch formula's constants are
    // tuned to English syllable patterns and don't transfer to other scripts. Instead,
    // estimate difficulty from sentence length and word length alone (longer sentences
    // and longer words both make text harder to read, regardless of script).
    var avgWordLen = words.length ? (words.join('').length / words.length) : 0;
    var estimate = 100 - (avgWordsPerSentence * 1.3) - (avgWordLen * 4);
    estimate = Math.max(0, Math.min(100, Math.round(estimate)));
    var generalGrade = estimate >= 70 ? 'Easy to read' : estimate >= 45 ? 'Moderate difficulty' : 'Difficult to read';
    return { sentences: sentences, words: wc, syllables: 0, flesch: estimate, grade: generalGrade,
      avgWordsPerSentence: avgWordsPerSentence, method: 'general' };
  }

  function keywordDensity(text, opts) {
    opts = opts || {};
    var phraseLen = opts.phraseLen || 1;
    // \p{L}\p{N} keeps letters/numbers from ANY script — the previous [^a-z0-9] pattern
    // stripped every non-ASCII character, silently deleting all Tamil/Hindi/Arabic/CJK
    // text and leaving nothing to analyze.
    var raw = (text || '').toLowerCase().replace(/[^\p{L}\p{N}\s'-]/gu, ' ');
    var words = raw.split(/\s+/).filter(Boolean).filter(function (w) { return !STOP[w] && w.length > 1; });
    var total = words.length;
    var counts = {};
    for (var i = 0; i <= words.length - phraseLen; i++) {
      var phrase = words.slice(i, i + phraseLen).join(' ');
      counts[phrase] = (counts[phrase] || 0) + 1;
    }
    var rows = Object.keys(counts).map(function (k) {
      return { term: k, count: counts[k], density: total ? (counts[k] / total * 100) : 0 };
    });
    rows.sort(function (a, b) { return b.count - a.count; });
    return { total: total, rows: rows };
  }

  function slugify(str) {
    return (str || '')
      .toLowerCase()
      .trim()
      .replace(/['"]/g, '')
      .replace(/[^a-z0-9]+/g, '-')
      .replace(/^-+|-+$/g, '')
      .replace(/-{2,}/g, '-');
  }

  // ---- HTML analysis ----------------------------------------------------
  function analyzeHtml(html, pageUrl) {
    var doc = new DOMParser().parseFromString(html, 'text/html');
    var host = '';
    try { host = new URL(pageUrl).hostname.replace(/^www\./, ''); } catch (e) {}

    var title = (doc.querySelector('title') || {}).textContent || '';
    title = title.trim();

    var metaDesc = doc.querySelector('meta[name="description" i]');
    var descContent = metaDesc ? (metaDesc.getAttribute('content') || '').trim() : '';

    var canonical = doc.querySelector('link[rel="canonical" i]');
    var viewport = doc.querySelector('meta[name="viewport" i]');
    var charset = doc.querySelector('meta[charset]') || doc.querySelector('meta[http-equiv="Content-Type" i]');
    var robotsMeta = doc.querySelector('meta[name="robots" i]');
    var robotsContent = robotsMeta ? (robotsMeta.getAttribute('content') || '').toLowerCase() : '';
    var noindex = /noindex/.test(robotsContent);

    var h1s = Array.prototype.slice.call(doc.querySelectorAll('h1'));
    var headingCounts = {};
    ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].forEach(function (t) { headingCounts[t] = doc.querySelectorAll(t).length; });

    var imgs = Array.prototype.slice.call(doc.querySelectorAll('img'));
    var imgsMissingAlt = imgs.filter(function (i) { return !i.getAttribute('alt') || !i.getAttribute('alt').trim(); }).length;

    var anchors = Array.prototype.slice.call(doc.querySelectorAll('a[href]'));
    var internal = 0, external = 0, nofollow = 0;
    anchors.forEach(function (a) {
      var href = a.getAttribute('href') || '';
      if (/^(mailto:|tel:|javascript:|#)/i.test(href)) return;
      var isExternal = /^https?:\/\//i.test(href) && host && href.indexOf(host) === -1;
      if (isExternal) external++; else internal++;
      var rel = (a.getAttribute('rel') || '').toLowerCase();
      if (rel.indexOf('nofollow') !== -1) nofollow++;
    });

    var ogTitle = doc.querySelector('meta[property="og:title" i]');
    var ogDesc = doc.querySelector('meta[property="og:description" i]');
    var ogImage = doc.querySelector('meta[property="og:image" i]');
    var twitterCard = doc.querySelector('meta[name="twitter:card" i]');

    var ldJson = doc.querySelectorAll('script[type="application/ld+json" i]');
    var htmlLang = doc.documentElement ? doc.documentElement.getAttribute('lang') : null;
    var favicon = doc.querySelector('link[rel*="icon" i]');

    // Strip script/style/noscript before computing visible text + word count.
    var bodyClone = doc.body ? doc.body.cloneNode(true) : document.createElement('body');
    bodyClone.querySelectorAll('script,style,noscript,template').forEach(function (n) { n.remove(); });
    var visibleText = (bodyClone.textContent || '').replace(/\s+/g, ' ').trim();
    var wordCount = textWordCount(visibleText);

    var headScripts = doc.querySelectorAll('head script[src]');
    var renderBlocking = Array.prototype.slice.call(headScripts).filter(function (s) {
      return !s.hasAttribute('async') && !s.hasAttribute('defer');
    }).length;

    var sizeKb = Math.round((new Blob([html]).size / 1024) * 10) / 10;
    var isHttps = /^https:\/\//i.test(pageUrl);

    return {
      url: pageUrl, host: host, sizeKb: sizeKb, isHttps: isHttps,
      title: title, titleLen: title.length,
      description: descContent, descLen: descContent.length,
      canonical: canonical ? canonical.getAttribute('href') : null,
      viewport: !!viewport,
      charset: !!charset,
      noindex: noindex,
      h1Count: h1s.length, h1Text: h1s[0] ? h1s[0].textContent.trim() : '',
      headingCounts: headingCounts,
      imgCount: imgs.length, imgsMissingAlt: imgsMissingAlt,
      linkInternal: internal, linkExternal: external, linkNofollow: nofollow,
      ogComplete: !!(ogTitle && ogDesc && ogImage),
      ogTitle: !!ogTitle, ogDesc: !!ogDesc, ogImage: !!ogImage,
      twitterCard: !!twitterCard,
      structuredDataCount: ldJson.length,
      htmlLang: htmlLang,
      favicon: !!favicon,
      wordCount: wordCount,
      visibleText: visibleText,
      renderBlockingScripts: renderBlocking,
      readability: readability(visibleText)
    };
  }

  // ---- Scoring ------------------------------------------------------------
  function clamp(n, lo, hi) { return Math.max(lo, Math.min(hi, n)); }

  function scoreOnPage(m) {
    var s = 100, issues = [];
    if (!m.title) { s -= 30; issues.push(mk('critical', 'On-page SEO', 'Missing <title> tag', 'Every page needs a unique, descriptive title tag — it is the single strongest on-page ranking signal.')); }
    else if (m.titleLen < 15 || m.titleLen > 65) { s -= 12; issues.push(mk('warning', 'On-page SEO', 'Title tag length is ' + m.titleLen + ' characters', 'Aim for 15–65 characters so it isn\'t truncated in search results.')); }
    if (!m.description) { s -= 22; issues.push(mk('critical', 'On-page SEO', 'Missing meta description', 'Add a compelling 50–160 character description to improve click-through from search results.')); }
    else if (m.descLen < 50 || m.descLen > 160) { s -= 10; issues.push(mk('warning', 'On-page SEO', 'Meta description length is ' + m.descLen + ' characters', 'Keep it between 50–160 characters so Google doesn\'t rewrite or truncate it.')); }
    if (m.h1Count === 0) { s -= 16; issues.push(mk('critical', 'On-page SEO', 'No H1 heading found', 'Add one clear H1 that describes the page\'s main topic.')); }
    else if (m.h1Count > 1) { s -= 8; issues.push(mk('warning', 'On-page SEO', m.h1Count + ' H1 tags found', 'Use a single H1 per page and structure the rest with H2/H3.')); }
    if (m.imgCount > 0 && m.imgsMissingAlt > 0) {
      var pct = Math.round((m.imgsMissingAlt / m.imgCount) * 100);
      var sev = pct > 40 ? 'critical' : 'warning';
      s -= sev === 'critical' ? 14 : 7;
      issues.push(mk(sev, 'On-page SEO', m.imgsMissingAlt + ' image' + (m.imgsMissingAlt > 1 ? 's' : '') + ' missing alt text', 'Alt text helps accessibility and lets Google understand what your images show.'));
    }
    if (!m.canonical) { s -= 6; issues.push(mk('warning', 'On-page SEO', 'No canonical tag', 'A self-referencing canonical avoids duplicate-content ambiguity.')); }
    return { score: clamp(Math.round(s), 0, 100), issues: issues };
  }

  function scoreContent(m) {
    var s = 100, issues = [];
    if (m.wordCount === 0) { s -= 55; issues.push(mk('critical', 'Content', 'No readable content found on the page', 'The page has no visible text at all — search engines have nothing to index or rank.')); }
    else if (m.wordCount < 50) { s -= 42; issues.push(mk('critical', 'Content', 'Almost no content — only ' + m.wordCount + ' words', 'Pages this thin are rarely indexed as useful; add substantive, unique copy.')); }
    else if (m.wordCount < 150) { s -= 30; issues.push(mk('critical', 'Content', 'Thin content — only ' + m.wordCount + ' words', 'Pages under ~150 words rarely rank; add substantive, useful copy.')); }
    else if (m.wordCount < 300) { s -= 12; issues.push(mk('warning', 'Content', 'Light content — ' + m.wordCount + ' words', 'Consider expanding to 300+ words to fully cover the topic.')); }
    var fl = m.readability.flesch;
    if (m.wordCount > 0 && fl < 40) { s -= 12; issues.push(mk('warning', 'Content', 'Readability is difficult (score ' + fl + '/100)', 'Shorter sentences and simpler words widen your audience.')); }
    if (m.wordCount > 0 && m.readability.avgWordsPerSentence > 26) { s -= 8; issues.push(mk('warning', 'Content', 'Long average sentence length (' + m.readability.avgWordsPerSentence + ' words)', 'Break long sentences up to improve scannability.')); }
    return { score: clamp(Math.round(s), 0, 100), issues: issues };
  }

  function scoreTechnical(m) {
    var s = 100, issues = [];
    if (!m.isHttps) { s -= 30; issues.push(mk('critical', 'Technical', 'Not served over HTTPS', 'HTTPS is a confirmed ranking signal and required for user trust.')); }
    if (m.noindex) { s -= 40; issues.push(mk('critical', 'Technical', 'Page is set to noindex', 'This explicitly tells search engines to exclude the page from results.')); }
    if (!m.htmlLang) { s -= 6; issues.push(mk('warning', 'Technical', 'Missing lang attribute on <html>', 'Declaring a language helps search engines and screen readers.')); }
    if (m.structuredDataCount === 0) { s -= 10; issues.push(mk('warning', 'Technical', 'No structured data (JSON-LD) found', 'Schema markup can unlock rich results like stars, FAQs and breadcrumbs.')); }
    if (!m.favicon) { s -= 4; issues.push(mk('warning', 'Technical', 'No favicon declared', 'A favicon improves recognition in tabs, bookmarks and search results.')); }
    return { score: clamp(Math.round(s), 0, 100), issues: issues };
  }

  function scoreMobile(m) {
    var s = 100, issues = [];
    if (!m.viewport) { s -= 45; issues.push(mk('critical', 'Mobile', 'Missing responsive viewport meta tag', 'Without it, mobile browsers render a desktop-width layout and zoom out.')); }
    return { score: clamp(Math.round(s), 0, 100), issues: issues };
  }

  function scorePerformance(m) {
    var s = 100, issues = [];
    if (m.sizeKb > 1500) { s -= 24; issues.push(mk('critical', 'Performance', 'Very large HTML payload (' + m.sizeKb + ' KB)', 'Trim unused markup/inline content — large HTML slows first paint.')); }
    else if (m.sizeKb > 700) { s -= 10; issues.push(mk('warning', 'Performance', 'Heavy HTML payload (' + m.sizeKb + ' KB)', 'Consider reducing inline scripts/styles and unused markup.')); }
    if (m.renderBlockingScripts > 2) { s -= 18; issues.push(mk('critical', 'Performance', m.renderBlockingScripts + ' render-blocking scripts in <head>', 'Add async or defer so scripts don\'t block the first render.')); }
    else if (m.renderBlockingScripts > 0) { s -= 8; issues.push(mk('warning', 'Performance', m.renderBlockingScripts + ' render-blocking script(s) in <head>', 'Add async or defer where possible.')); }
    if (m.fetchMs && m.fetchMs > 1800) { s -= 12; issues.push(mk('warning', 'Performance', 'Server responded slowly (' + m.fetchMs + ' ms)', 'A faster time-to-first-byte improves both rankings and user experience.')); }
    return { score: clamp(Math.round(s), 0, 100), issues: issues };
  }

  function scoreSocial(m) {
    var s = 100, issues = [];
    if (!m.ogComplete) { s -= 40; issues.push(mk('warning', 'Social', 'Incomplete Open Graph tags', 'Add og:title, og:description and og:image so shared links look complete on social platforms.')); }
    if (!m.twitterCard) { s -= 20; issues.push(mk('warning', 'Social', 'No Twitter/X card meta tag', 'Add twitter:card so links render as rich previews on X.')); }
    return { score: clamp(Math.round(s), 0, 100), issues: issues };
  }

  var SEVERITY_ORDER = { critical: 0, warning: 1, good: 2 };
  function mk(severity, category, title, detail) { return { severity: severity, category: category, title: title, detail: detail }; }

  function buildReport(m) {
    var cats = {
      onpage: scoreOnPage(m),
      content: scoreContent(m),
      technical: scoreTechnical(m),
      mobile: scoreMobile(m),
      performance: scorePerformance(m),
      social: scoreSocial(m)
    };
    var weights = { onpage: 0.26, content: 0.18, technical: 0.22, mobile: 0.14, performance: 0.12, social: 0.08 };
    var overall = 0;
    Object.keys(weights).forEach(function (k) { overall += cats[k].score * weights[k]; });
    overall = Math.round(overall);

    var issues = [].concat(cats.onpage.issues, cats.content.issues, cats.technical.issues, cats.mobile.issues, cats.performance.issues, cats.social.issues);
    issues.sort(function (a, b) { return SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]; });

    return { metrics: m, categories: cats, overall: overall, issues: issues, generatedAt: new Date() };
  }

  // ---- Sitemap-first URL discovery -----------------------------------------
  // Most real sites already publish a sitemap.xml (often declared in robots.txt).
  // Reading that directly is faster and far more complete than crawling page-by-page
  // via <a href> links — and it's the *site's own* standard file, not a third-party
  // API, so this adds no external dependency beyond the site being audited itself.
  function parseSitemapXml(xml) {
    var doc;
    try { doc = new DOMParser().parseFromString(xml, 'text/xml'); } catch (e) { return { type: null, locs: [] }; }
    if (!doc || doc.querySelector('parsererror')) return { type: null, locs: [] };
    var root = doc.documentElement;
    if (!root) return { type: null, locs: [] };
    var tag = root.tagName.toLowerCase();
    var locs = Array.prototype.slice.call(doc.getElementsByTagName('loc'))
      .map(function (el) { return (el.textContent || '').trim(); })
      .filter(Boolean);
    if (!locs.length) return { type: null, locs: [] };
    return { type: tag.indexOf('sitemapindex') !== -1 ? 'sitemapindex' : 'urlset', locs: locs };
  }

  async function discoverSitemapUrls(siteUrl, opts) {
    opts = opts || {};
    var maxSitemaps = opts.maxSitemaps || 20;
    var maxUrls = opts.maxUrls || 5000;
    var origin;
    try { origin = new URL(normalizeUrl(siteUrl)).origin; } catch (e) { return { urls: [], source: null, sitemapUrls: [], robotsFound: false }; }

    var candidates = [];
    var robotsFound = false;
    var robotsRes = await fetchPage(origin + '/robots.txt', 9000, { expectHtml: false });
    if (robotsRes.ok) {
      robotsFound = true;
      robotsRes.html.split(/\r?\n/).forEach(function (line) {
        var m = line.match(/^\s*sitemap\s*:\s*(\S+)/i);
        if (m) candidates.push(m[1].trim());
      });
    }
    if (!candidates.length) {
      candidates.push(origin + '/sitemap.xml');
      candidates.push(origin + '/sitemap_index.xml');
    }

    var seen = {};
    var queue = candidates.slice();
    var urls = [];
    var usedSitemaps = [];
    var declaredInRobots = candidates.length && robotsFound;

    while (queue.length && usedSitemaps.length < maxSitemaps && urls.length < maxUrls) {
      var sUrl = queue.shift();
      if (seen[sUrl]) continue;
      seen[sUrl] = true;
      var res = await fetchPage(sUrl, 9000, { expectHtml: false });
      if (!res.ok) continue;
      var parsed = parseSitemapXml(res.html);
      if (!parsed.type) continue;
      usedSitemaps.push(sUrl);
      if (parsed.type === 'sitemapindex') {
        parsed.locs.forEach(function (l) { if (!seen[l]) queue.push(l); });
      } else {
        parsed.locs.forEach(function (l) { if (urls.length < maxUrls) urls.push(l); });
      }
    }

    return {
      urls: urls,
      source: usedSitemaps.length ? (declaredInRobots ? 'robots.txt' : 'sitemap.xml') : null,
      sitemapUrls: usedSitemaps,
      robotsFound: robotsFound
    };
  }

  // Common non-page file extensions to skip when discovering "pages" for a sitemap.
  var ASSET_EXT_RE = /\.(jpg|jpeg|png|gif|svg|webp|ico|css|js|mjs|pdf|zip|rar|7z|mp4|mp3|wav|avi|mov|woff2?|ttf|eot|xml|json|txt|csv|doc|docx|xls|xlsx)$/i;

  function extractInternalLinks(html, pageUrl) {
    var doc = new DOMParser().parseFromString(html, 'text/html');
    var base;
    try { base = new URL(pageUrl); } catch (e) { return []; }
    var host = base.hostname.replace(/^www\./, '');
    var anchors = Array.prototype.slice.call(doc.querySelectorAll('a[href]'));
    var seen = {};
    var results = [];
    anchors.forEach(function (a) {
      var href = a.getAttribute('href') || '';
      if (!href || /^(mailto:|tel:|javascript:|#)/i.test(href)) return;
      var abs;
      try { abs = new URL(href, base); } catch (e) { return; }
      if (abs.protocol !== 'http:' && abs.protocol !== 'https:') return;
      if (abs.hostname.replace(/^www\./, '') !== host) return;
      if (ASSET_EXT_RE.test(abs.pathname)) return;
      abs.hash = '';
      var normalized = abs.toString().replace(/\/$/, '') || abs.toString();
      if (seen[normalized]) return;
      seen[normalized] = true;
      results.push(normalized);
    });
    // Always include the page itself.
    var selfUrl = pageUrl.replace(/\/$/, '');
    if (!seen[selfUrl]) results.unshift(selfUrl);
    return results;
  }

  /**
   * If the configured custom backend is this project's own PHP backend (ends in
   * /fetch.php, by convention), derive its sibling /crawl.php endpoint. Returns
   * null for anything else (a Cloudflare Worker, a differently-named endpoint,
   * or no backend at all) — callers must fall back to the client-side crawl in
   * that case, which they all already do.
   */
  function getCustomCrawlBackend() {
    var fetchBackend = getCustomBackend();
    if (!fetchBackend) return null;
    if (/\/fetch\.php(\?.*)?$/i.test(fetchBackend)) {
      return fetchBackend.replace(/\/fetch\.php(\?.*)?$/i, '/crawl.php');
    }
    return null;
  }

  /**
   * One-shot server-side crawl via this project's own PHP backend, when
   * configured. mode is 'discover' | 'posts' | 'products' — matching
   * backend-php/api/crawl.php exactly. Returns {ok:false} for any reason
   * (no backend configured, backend unreachable, backend errored) so callers
   * can transparently fall back to the existing client-side crawl.
   */
  async function crawlViaBackend(siteUrl, mode, timeoutMs) {
    var crawlUrl = getCustomCrawlBackend();
    if (!crawlUrl) return { ok: false, reason: 'no-backend' };
    var controller = new AbortController();
    var timer = setTimeout(function () { controller.abort(); }, timeoutMs || 60000);
    try {
      var sep = crawlUrl.indexOf('?') > -1 ? '&' : '?';
      var res = await fetch(crawlUrl + sep + 'url=' + encodeURIComponent(siteUrl) + '&mode=' + mode, { signal: controller.signal });
      clearTimeout(timer);
      if (!res.ok) return { ok: false, reason: 'HTTP ' + res.status };
      var data = await res.json();
      if (data.error) return { ok: false, reason: data.error };
      return { ok: true, data: data };
    } catch (e) {
      clearTimeout(timer);
      return { ok: false, reason: e && e.name === 'AbortError' ? 'timed out' : (e && e.message) };
    }
  }

  global.CrawlieEngine = {
    fetchPage: fetchPage,
    analyzeHtml: analyzeHtml,
    buildReport: buildReport,
    readability: readability,
    keywordDensity: keywordDensity,
    slugify: slugify,
    normalizeUrl: normalizeUrl,
    getCustomBackend: getCustomBackend,
    setCustomBackend: setCustomBackend,
    extractInternalLinks: extractInternalLinks,
    discoverSitemapUrls: discoverSitemapUrls,
    getCustomCrawlBackend: getCustomCrawlBackend,
    crawlViaBackend: crawlViaBackend
  };
})(window);

