// Derived tables for dataset wa-i645-media-coding, computed from i645_media_coding.csv.
// Usage: node millionaires-tax/data/media_audit_derived.js
// Writes media_omission_rates.csv, media_claim_checks.csv, media_sourcing_balance.csv and
// media_derived.json (rows for the dataset's saved analyses).
const fs = require('fs');
const path = require('path');
const dir = __dirname;

function parseCsv(t) {
  const rows = []; let row = [], f = '', q = false;
  for (let i = 0; i < t.length; i++) {
    const c = t[i];
    if (q) { if (c === '"') { if (t[i + 1] === '"') { f += '"'; i++; } else q = false; } else f += c; }
    else if (c === '"') q = true; else if (c === ',') { row.push(f); f = ''; }
    else if (c === '\n') { row.push(f); rows.push(row); row = []; f = ''; } else if (c !== '\r') f += c;
  }
  return rows;
}
const R = parseCsv(fs.readFileSync(path.join(dir, 'i645_media_coding.csv'), 'utf8'));
const H = R[0];
const all = R.slice(1).filter(r => r.length > 1).map(r => Object.fromEntries(H.map((h, i) => [h, r[i]])));
const read = all.filter(r => r.read_status !== 'unread');
const isOp = r => /opinion|editorial/.test(r.type);
// News groups use fully read stories only, so every count shares one denominator
// (a paywalled, partly read story could be coded for only some facts).
const isNews = r => !isOp(r) && r.read_status === 'full';
const TV = ['KOMO', 'KING 5', 'KIRO 7', 'KXLY', 'Fox 13 Seattle', 'KREM'];
const esc = v => /[",\n]/.test(String(v)) ? '"' + String(v).replace(/"/g, '""') + '"' : String(v);
const writeCsv = (name, cols, rows) => fs.writeFileSync(path.join(dir, name),
  [cols.join(',')].concat(rows.map(r => cols.map(c => esc(r[c] ?? '')).join(','))).join('\n') + '\n');
const r3 = x => Math.round(x * 1000) / 1000;

const GROUPS = [
  ['Mainstream news', r => r.outlet_group === 'mainstream' && isNews(r)],
  ['Mainstream TV news', r => r.outlet_group === 'mainstream' && isNews(r) && TV.includes(r.outlet)],
  ['Mainstream print/public/online news', r => r.outlet_group === 'mainstream' && isNews(r) && !TV.includes(r.outlet)],
  ['Conservative news', r => r.outlet_group === 'conservative' && isNews(r)],
  ['Conservative opinion', r => r.outlet_group === 'conservative' && isOp(r)],
  ['Mainstream opinion', r => r.outlet_group === 'mainstream' && isOp(r)],
  ['Progressive (all)', r => r.outlet_group === 'progressive'],
];
const FACTS = [
  ['O1', 'Tax relief survives repeal', 'No side'], ['O2', 'Revenue not dedicated to K-12', 'No side'],
  ['O3a', 'Collections start in 2029', 'Neither (timing)'], ['O3b', 'No current appropriation relies on it', 'No side'],
  ['O4', '2027-29 budget outlook counts on it', 'Yes side'], ['O5', 'Threshold can be lowered by simple majority', 'No side'],
  ['O6', 'Estimate is static / revenue volatile', 'No side'], ['O7', 'Union funding / No side outraised Yes', 'No side'],
  ['O8', "Heywood's personal financial interest", 'Yes side'], ['O9', 'Legal risk to capital gains tax', 'No side'],
];

const omission = [];
for (const [g, f] of GROUPS) {
  const s = read.filter(f);
  for (const [code, label, rebuts] of FACTS) {
    const coded = s.filter(r => r[code] === '0' || r[code] === '1');
    const inc = coded.filter(r => r[code] === '1').length;
    omission.push({ group: g, fact_code: code, fact: label, rebuts_claims_of: rebuts,
      stories_including: inc, stories_coded: coded.length, share_including: coded.length ? r3(inc / coded.length) : null });
  }
}
// Voice split for the two budget facts: stated as fact vs only in an advocate quote.
for (const [g, f] of GROUPS) {
  const s = read.filter(f);
  for (const code of ["O3b", "O4"]) {
    const coded = s.filter(r => r[code] === "0" || r[code] === "1");
    const label = FACTS.find(x => x[0] === code);
    for (const voice of ["fact", "advocate"]) {
      const n = coded.filter(r => r[code + "_voice"] === voice).length;
      omission.push({ group: g, fact_code: code + "-" + voice, fact: label[1] + (voice === "fact" ? " (stated as fact)" : " (only in an advocate quote)"), rebuts_claims_of: label[2],
        stories_including: n, stories_coded: coded.length, share_including: coded.length ? r3(n / coded.length) : null });
    }
  }
}
writeCsv('media_omission_rates.csv', ['group', 'fact_code', 'fact', 'rebuts_claims_of', 'stories_including', 'stories_coded', 'share_including'], omission);

const CLAIMS = [
  ['N13B', 'No', '"$13 billion in cuts" to schools, health care, child care'], ['NBREAK', 'No', '"$13 billion tax break" for millionaires'],
  ['N460K', 'No', 'Repeal raises taxes on 460,000-500,000 families'], ['NVOTE', 'No', 'Voters always have the final say on expansion'],
  ['N05', 'No', 'Only ~0.5% / 20,000-25,000 households pay (as permanent scope)'], ['NMEALS', 'No', 'Free school meals end'],
  ['N300B', 'No', 'Jayapal: "$300 billion tax break"'], ['NHEDGE', 'No', '"Hedge-fund mogul" buying a tax cut'],
  ['YCOME', 'Yes', '"It will come for everyone"'], ['YPENS', 'Yes', 'Pensions and Social Security taxed'],
  ['YUNCON', 'Yes', '"Unconstitutional" stated as fact'], ['YUNEMP', 'Yes', 'WA unemployment "tied for highest"'],
  ['Y90', 'Yes', '"90% of family businesses"'], ['YLEAVE', 'Yes', 'Millionaires are leaving (as fact)'],
  ['YLEAK', 'Yes', '"Leaked emails"'], ['YNOK12', 'Yes', '"Doesn\'t do anything with K-12 funding" / "not a dime"'],
];
const has = (r, k, c) => (r[k] || '').split(';').includes(c);
const claims = [];
for (const [g, f] of GROUPS.filter(([g]) => !/TV|print/.test(g))) {
  const s = read.filter(f);
  for (const [code, side, label] of CLAIMS) {
    const un = s.filter(r => has(r, side === 'No' ? 'no_claims_unchecked' : 'yes_claims_unchecked', code)).length;
    const ck = s.filter(r => has(r, 'claims_checked', code)).length;
    if (un + ck) claims.push({ group: g, claim_code: code, claim_side: side, claim: label, stories_unchecked: un, stories_contextualized: ck,
      share_contextualized: r3(ck / (un + ck)) });
  }
}
writeCsv('media_claim_checks.csv', ['group', 'claim_code', 'claim_side', 'claim', 'stories_unchecked', 'stories_contextualized', 'share_contextualized'], claims);

const sourcing = [];
for (const [g, f] of GROUPS) {
  const s = read.filter(f).filter(r => !isOp(r) || /Progressive/.test(g));
  const n = s.length; if (!n) continue;
  const num = k => s.map(r => r[k]).filter(v => v !== '').map(Number);
  const avg = k => { const v = num(k); return v.length ? r3(v.reduce((a, b) => a + b, 0) / v.length) : null; };
  const instances = side => {
    let u = 0, c = 0;
    for (const r of s) {
      u += (r[side === 'No' ? 'no_claims_unchecked' : 'yes_claims_unchecked'] || '').split(';').filter(Boolean).length;
      c += (r.claims_checked || '').split(';').filter(x => x && x[0] === side[0]).length;
    }
    return [u, c];
  };
  const [nu, nc] = instances('No'), [yu, yc] = instances('Yes');
  sourcing.push({ group: g, stories: n, avg_yes_sources: avg('yes_sources'), avg_no_sources: avg('no_sources'),
    first_word_yes: s.filter(r => r.first_word === 'yes').length, first_word_no: s.filter(r => r.first_word === 'no').length,
    last_word_yes: s.filter(r => r.last_word === 'yes').length, last_word_no: s.filter(r => r.last_word === 'no').length,
    no_rebuttal: s.filter(r => r.rebuttal === 'N').length,
    stories_with_unchecked_no_claim: s.filter(r => r.no_claims_unchecked).length,
    stories_with_unchecked_yes_claim: s.filter(r => r.yes_claims_unchecked).length,
    no_claim_instances: nu + nc, no_claim_instances_contextualized: nc,
    yes_claim_instances: yu + yc, yes_claim_instances_contextualized: yc,
    stories_with_error: s.filter(r => r.has_error === '1').length });
}
writeCsv('media_sourcing_balance.csv', Object.keys(sourcing[0]), sourcing);

const SHORT = { O1: "Relief survives repeal", O2: "Not dedicated to K-12", "O3b-fact": "No current budget relies on it", "O4-fact": "2027-29 outlook counts on it", O5: "Lawmakers can lower threshold", O7: "Union money / No outraised Yes", O8: "Heywood's own stake" };
// Chart rows: the two headline groups only, with a display label that names which side each fact rebuts.
const chartRows = omission.filter(r => r.group === 'Mainstream news' || r.group === 'Conservative news')
  .filter(r => ['O1','O2','O3b-fact','O4-fact','O5','O7','O8'].includes(r.fact_code))
  .map(r => ({ fact_short: SHORT[r.fact_code] + (r.rebuts_claims_of === "Yes side" ? " [rebuts Yes]" : " [rebuts No]"), fact: `${r.fact} (rebuts ${r.rebuts_claims_of})`, group: r.group, share_including: r.share_including,
    stories_including: r.stories_including, stories_coded: r.stories_coded }));
fs.writeFileSync(path.join(dir, 'media_derived.json'), JSON.stringify({ omission, claims, sourcing, chartRows }, null, 1));
console.log(JSON.stringify({ stories: all.length, read: read.length, sourcing }, null, 1));
console.log(claims.filter(c => c.group === 'Mainstream news').map(c => `${c.claim_code} ${c.stories_unchecked}/${c.stories_contextualized}`).join('  '));