// Fetches each policy page and checks that every quoted phrase appears verbatim
// (after decoding HTML entities and normalizing whitespace, quotes and dashes).
// Pages that block automated fetches (HTTP 403) or render text with JavaScript are
// reported as "unchecked" and must be verified in a browser.
const { U, R } = require('./policy-rows');

// TikTok embeds its policy text URL-encoded inside a script tag, so script
// contents are kept and %XX / \uXXXX escapes are decoded before tags are stripped.
const norm = s => s.replace(/%([0-9A-F]{2})/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
  .replace(/\\u([0-9a-f]{4})/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
  .replace(/<[^>]+>/g, ' ')
  .replace(/\\u0026/g, '&').replace(/\\u003c/g, '<').replace(/\\u003e/g, '>').replace(/\\"/g, '"').replace(/\\n/g, ' ')
  .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(+d))
  .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
  .replace(/&quot;/g, '"').replace(/&amp;/g, '&').replace(/&apos;|&#39;/g, "'").replace(/&nbsp;/g, ' ')
  .replace(/&lt;/g, '<').replace(/&gt;/g, '>')
  .replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/[–—]/g, '-')
  .replace(/\s+/g, ' ').toLowerCase();

(async () => {
  const pages = {};
  for (const [id, url] of Object.entries(U)) {
    try {
      const r = await fetch(url, { headers: { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126 Safari/537.36', 'accept-language': 'en-US' } });
      pages[id] = r.ok ? norm(await r.text()) : null;
      if (!r.ok) console.error(`${id}: HTTP ${r.status}`);
    } catch (e) { pages[id] = null; console.error(`${id}: ${e.message}`); }
  }
  const out = [];
  for (const [platform, topic, , , quote, sid] of R) {
    if (!quote) continue;
    const p = pages[sid];
    const status = p == null ? 'unchecked' : (p.includes(norm(quote)) ? 'found' : 'NOT-FOUND');
    out.push({ platform, topic, sid, status, quote });
  }
  require('fs').writeFileSync(__dirname + '/quote-check.json', JSON.stringify(out, null, 1));
  const tally = out.reduce((a, o) => (a[o.status] = (a[o.status] || 0) + 1, a), {});
  console.log(tally);
  for (const o of out.filter(o => o.status === 'NOT-FOUND')) console.log('NOT-FOUND', o.platform, o.topic, o.sid, '|', o.quote);
})();