// Merge coded I-645 media rows (pipe tables in the researchers' coding notes) into
// data/i645_media_coding.csv and print per-outlet / per-group tallies.
// Usage: node millionaires-tax/data/media_audit.js <notes_dir>
const fs = require('fs');
const path = require('path');

const notesDir = process.argv[2];
const COLS = ['story_id', 'outlet', 'outlet_group', 'date', 'author', 'headline', 'url', 'read_status', 'type',
  'yes_sources', 'no_sources', 'neutral_sources', 'first_word', 'last_word', 'rebuttal', 'headline_term',
  'loaded_terms', 'no_claims_unchecked', 'yes_claims_unchecked', 'claims_checked',
  'O1', 'O2', 'O3a', 'O3b', 'O4', 'O5', 'O6', 'O7', 'O8', 'O9',
  'errors', 'ballot_sentence', 'dhm_poll', 'key_quote', 'notes'];
const OMIT = ['O1', 'O2', 'O3a', 'O3b', 'O4', 'O5', 'O6', 'O7', 'O8', 'O9'];

const files = fs.readdirSync(notesDir).filter(f => /^media-audit-.*\.md$/.test(f) && f !== 'media-audit-codebook.md');
const rows = [];
for (const f of files) {
  for (const line of fs.readFileSync(path.join(notesDir, f), 'utf8').split(/\r?\n/)) {
    if ((line.match(/\|/g) || []).length < COLS.length - 2) continue;
    const cells = line.trim().replace(/^\||\|$/g, '').split('|').map(s => s.trim());
    if (cells.length < COLS.length - 1) continue;
    if (cells[0] === 'story_id' || /^-+$/.test(cells[0].replace(/[:\s]/g, ''))) continue;
    const r = {};
    COLS.forEach((c, i) => r[c] = cells[i] ?? '');
    // a key_quote containing '|' would add cells; fold extras into notes
    if (cells.length > COLS.length) r.notes = cells.slice(COLS.length - 1).join(' / ');
    r.source_file = f;
    rows.push(r);
  }
}

const num = v => { const n = parseFloat(v); return isNaN(n) ? null : n; };
const CODE_RE = /\b(N13B|N460K|NVOTE|N05|NMEALS|N300B|NHEDGE|NBREAK|YCOME|YPENS|YUNCON|YUNEMP|Y90|YLEAVE|YLEAK|YNOK12)\b/g;
const codes = v => [...new Set((v || '').match(CODE_RE) || [])];
const bin = v => { const m = String(v || '').trim().match(/^[01]/); return m ? m[0] : ''; };
const hasErr = v => !!v && !/^(none|no\b|n\/a|-|$)/i.test(String(v).trim());

// Canonical outlet (masthead that published the story) and group, per the audit brief's outlet list.
const CANON = [
  [/Center Square/, 'The Center Square', 'conservative'], [/Seattle Red/, 'Seattle Red/KTTH', 'conservative'],
  [/MyNorthwest/, 'MyNorthwest/KIRO Newsradio', 'conservative'], [/Washington Policy Center/, 'Washington Policy Center', 'conservative'],
  [/Daily Wire/, 'The Daily Wire', 'conservative'], [/Lynnwood/, 'Lynnwood Times', 'conservative'],
  [/^Seattle Times/, 'Seattle Times', 'mainstream'], [/^Spokesman/, 'Spokesman-Review', 'mainstream'],
  [/Columbian/, 'The Columbian', 'mainstream'], [/Everett Herald/, 'Everett Herald', 'mainstream'],
  [/Tri-City/, 'Tri-City Herald', 'mainstream'], [/State Standard/, 'Washington State Standard', 'mainstream'],
  [/KUOW/, 'KUOW', 'mainstream'], [/Cascade PBS/, 'Cascade PBS', 'mainstream'], [/Stranger/, 'The Stranger', 'mainstream'],
  [/Deseret/, 'Deseret News', 'mainstream'], [/Cascadia Daily/, 'Cascadia Daily News', 'mainstream'],
  [/KOMO/, 'KOMO', 'mainstream'], [/KING 5/, 'KING 5', 'mainstream'], [/KIRO 7/, 'KIRO 7', 'mainstream'],
  [/KXLY/, 'KXLY', 'mainstream'], [/Fox 13/, 'Fox 13 Seattle', 'mainstream'], [/Axios/, 'Axios Seattle', 'mainstream'],
  [/KREM/, 'KREM', 'mainstream'], [/NPI/, 'NPI Cascadia Advocate', 'progressive'], [/The Stand/, 'The Stand', 'progressive'],
];
// A later read of a story replaces its "unread" placeholder (same story_id and URL).
for (let i = rows.length - 1; i >= 0; i--) {
  const r = rows[i];
  if (r.read_status === 'unread' && rows.some(o => o !== r && o.story_id === r.story_id && o.url === r.url && o.read_status !== 'unread'))
    rows.splice(i, 1);
}
// Voice re-pass for O3b and O4 (see media_coding_overrides.json).
const OV = JSON.parse(fs.readFileSync(path.join(__dirname, "media_coding_overrides.json"), "utf8"));
const seenIds = new Set();
for (const r of rows) {
  if (seenIds.has(r.story_id)) r.story_id = r.outlet.replace(/[^A-Z0-9]/g, '') + '-' + r.story_id;
  seenIds.add(r.story_id);
  r.outlet_detail = r.outlet;
  const c = CANON.find(([re]) => re.test(r.outlet));
  if (c) { r.outlet = c[1]; r.outlet_group = c[2]; } else console.error('unmapped outlet', r.outlet);
  if (!r.type) r.type = 'unknown';
  for (const k of ["O3b", "O4"]) {
    const v = (OV[k] || {})[r.story_id] || "";
    if (v && r.read_status !== "unread" && !/opinion|editorial/.test(r.type)) r[k] = "1";
    r[k + "_voice"] = bin(r[k]) === "1" ? (v || (/opinion|editorial/.test(r.type) ? "not-classified-opinion" : "unclassified")) : "";
  }
  const addCk = ((OV.claims_checked_add || {})[r.story_id]) || [];
  if (addCk.length) r.claims_checked = [r.claims_checked, ...addCk].filter(x => x && !/^none$/i.test(x)).join("; ");
  r.sources_detail = `yes: ${r.yes_sources}; no: ${r.no_sources}; neutral: ${r.neutral_sources}`;
}

const OUT_COLS = ['story_id', 'outlet', 'outlet_detail', 'outlet_group', 'date', 'author', 'headline', 'url', 'read_status', 'type',
  'yes_sources', 'no_sources', 'neutral_sources', 'first_word', 'last_word', 'rebuttal', 'headline_term', 'loaded_terms',
  'no_claims_unchecked', 'yes_claims_unchecked', 'claims_checked', ...OMIT, 'O3b_voice', 'O4_voice', 'has_error', 'errors', 'ballot_sentence', 'dhm_poll',
  'key_quote', 'sources_detail', 'notes'];
const clean = r => {
  const o = { ...r };
  const urls = String(r.url).match(/https?:\/\/[^\s,)]+/g) || [];
  o.url = urls[0] || r.url;
  if (urls.length > 1) o.notes = `[also/read via: ${urls.slice(1).join(' ')}] ${r.notes}`;
  for (const k of ['yes_sources', 'no_sources', 'neutral_sources']) { const n = num(r[k]); o[k] = n === null ? '' : n; }
  for (const k of ['no_claims_unchecked', 'yes_claims_unchecked', 'claims_checked']) o[k] = codes(r[k]).join(';');
  for (const k of OMIT) o[k] = r.read_status === 'unread' ? '' : bin(r[k]);
  o.has_error = r.read_status === 'unread' ? '' : (hasErr(r.errors) ? 1 : 0);
  o.ballot_sentence = (String(r.ballot_sentence).match(/critiqued|quoted|mentioned|none/) || [''])[0];
  o.dhm_poll = /^Y/.test(r.dhm_poll) ? 'Y' : /^N \(only/.test(r.dhm_poll) ? 'partial' : /^N/.test(r.dhm_poll) ? 'N' : '';
  return o;
};
const esc = v => /[",\n]/.test(String(v)) ? '"' + String(v).replace(/"/g, '""') + '"' : String(v);
const out = path.join(__dirname, 'i645_media_coding.csv');
rows.sort((a, b) => a.outlet_group.localeCompare(b.outlet_group) || a.outlet.localeCompare(b.outlet) || a.date.localeCompare(b.date));
fs.writeFileSync(out, [OUT_COLS.join(',')].concat(rows.map(clean).map(r => OUT_COLS.map(c => esc(r[c] ?? '')).join(','))).join('\n') + '\n');
const isOp = r => /opinion|editorial/.test(r.type);

function tally(group) {
  const read = group.filter(r => r.read_status !== 'unread');
  const news = read.filter(r => !isOp(r));
  const op = read.filter(r => isOp(r));
  const avg = (arr, k) => { const v = arr.map(r => num(r[k])).filter(x => x !== null); return v.length ? (v.reduce((a, b) => a + b, 0) / v.length).toFixed(1) : '-'; };
  const cnt = (arr, k) => arr.reduce((a, r) => a + codes(r[k]).length, 0);
  const omit = {};
  for (const o of OMIT) {
    const v = news.filter(r => bin(r[o]));
    omit[o] = v.length ? `${v.filter(r => bin(r[o]) === '1').length}/${v.length}` : '-';
  }
  return {
    stories: group.length, read: read.length, news: news.length, opinion: op.length,
    yes_src: avg(news, 'yes_sources'), no_src: avg(news, 'no_sources'),
    first_yes: news.filter(r => r.first_word === 'yes').length, first_no: news.filter(r => r.first_word === 'no').length,
    last_yes: news.filter(r => r.last_word === 'yes').length, last_no: news.filter(r => r.last_word === 'no').length,
    rebuttal_N: news.filter(r => r.rebuttal === 'N').length,
    noUnchk_news: cnt(news, 'no_claims_unchecked'), yesUnchk_news: cnt(news, 'yes_claims_unchecked'),
    noUnchk_op: cnt(op, 'no_claims_unchecked'), yesUnchk_op: cnt(op, 'yes_claims_unchecked'),
    errors: read.filter(r => hasErr(r.errors)).length,
    omit,
  };
}

const by = (k) => rows.reduce((m, r) => ((m[r[k]] = m[r[k]] || []).push(r), m), {});
const result = { total: rows.length, files, byGroup: {}, byOutlet: {}, claimFreq: {} };
for (const [g, rs] of Object.entries(by('outlet_group'))) result.byGroup[g] = tally(rs);
for (const [o, rs] of Object.entries(by('outlet'))) result.byOutlet[o] = { group: rs[0].outlet_group, ...tally(rs) };
for (const r of rows.filter(r => r.read_status !== 'unread')) {
  for (const k of ['no_claims_unchecked', 'yes_claims_unchecked', 'claims_checked'])
    for (const c of codes(r[k])) {
      const key = `${k}:${c}:${r.outlet_group}:${isOp(r) ? 'op' : 'news'}`;
      result.claimFreq[key] = (result.claimFreq[key] || 0) + 1;
    }
}
fs.writeFileSync(path.join(__dirname, 'media_audit_results.json'), JSON.stringify(result, null, 2));
console.log(JSON.stringify(result, null, 1));