// Recomputes fact-rates.csv, sourcing-balance.csv and claim-checks.csv from stories.csv.
// Usage: node derive.js [outDir]   (reads stories.csv next to this script)
const fs = require('fs'), path = require('path');
const OUT = process.argv[2] || __dirname;
function parse(t) { t = t.replace(/^/, ''); const R = []; let row = [], f = '', q = false; for (let i = 0; i < t.length; i++) { const c = t[i]; if (q) { if (c == '"') { if (t[i + 1] == '"') { f += '"'; i++ } else q = false } else f += c } else if (c == '"') q = true; else if (c == ',') { row.push(f); f = '' } else if (c == '\n' || c == '\r') { if (c == '\r' && t[i + 1] == '\n') i++; row.push(f); f = ''; if (row.length > 1) R.push(row); row = [] } else f += c } if (f || row.length) { row.push(f); R.push(row) } const h = R[0].map(s => s.trim()); return R.slice(1).map(r => Object.fromEntries(h.map((k, j) => [k, (r[j] || '').trim()]))) }
const all = parse(fs.readFileSync(path.join(__dirname, 'stories.csv'), 'utf8'));
const esc = v => /[",\n\r]/.test(v) ? '"' + v.replace(/"/g, '""') + '"' : v;
const FACTS = { F1: ['Genetic or hormone tests allowed; no exam mandated', 'No'], F2: ["The girl's own provider verifies; no school role", 'No'], F3: ['Any measure of the male performance gap', 'No'], F4: ["Trans girls can still play on boys' or open teams", 'No'], F5: ['27+ states, IOC or federal policy', 'No'], F6: ['A documented WA placement, or the 269 count', 'No'], F7: ['Who funds the No side', 'No'], F8: ['Heywood / Let\'s Go Washington funds the Yes side', 'Yes'], F9: ['The AAP objected to the Yes pamphlet citation', 'Yes'], F10: ['Few trans girl athletes in Washington', 'Yes'], F11: ['Cost or insurance of genetic or hormone tests', 'Yes'], F12: ['Forms sent to schools have no sex field; no form has the provider verify sex', 'Yes'] };
const SHORT = { F1: 'F1 Tests instead of exam', F2: 'F2 Own provider', F3: 'F3 Performance gap', F4: "F4 Can play on boys' teams", F5: 'F5 Other states/IOC', F6: 'F6 WA placements', F7: 'F7 No-side funders', F8: 'F8 Yes-side funder', F9: 'F9 AAP objection', F10: 'F10 Few trans athletes', F11: 'F11 Test cost', F12: 'F12 School forms lack sex' };
const groups = ['mainstream', 'conservative', 'progressive']; const rates = [], bal = [];
const has = v => v && !['none', 'n/a', 'na', '-'].includes(v.toLowerCase()); const num = v => +v || 0;
for (const g of groups) {
  const n = all.filter(r => r.group == g && r.counts_as_news == '1');
  for (const [k, [label, rebuts]] of Object.entries(FACTS)) { const c = n.filter(r => r[k] == '1').length; rates.push({ fact: k, fact_label: label, fact_short: SHORT[k] + (rebuts == 'No' ? ' [answers No]' : ' [answers Yes]'), answers_side: rebuts, group: g, news_stories: n.length, stories_including: c, share_including: +(c / n.length).toFixed(4) }) }
  bal.push({ group: g, news_stories: n.length, avg_yes_sources: +(n.reduce((a, r) => a + num(r.yes_sources), 0) / n.length).toFixed(2), avg_no_sources: +(n.reduce((a, r) => a + num(r.no_sources), 0) / n.length).toFixed(2), first_word_yes: n.filter(r => r.first_word == 'yes').length, first_word_no: n.filter(r => r.first_word == 'no').length, last_word_yes: n.filter(r => r.last_word == 'yes').length, last_word_no: n.filter(r => r.last_word == 'no').length, no_rebuttal: n.filter(r => r.rebuttal.toUpperCase() == 'N').length, stories_unchecked_no_claim: n.filter(r => has(r.no_claims_unchecked)).length, stories_unchecked_yes_claim: n.filter(r => has(r.yes_claims_unchecked)).length, stories_with_outlet_voice_error: n.filter(r => ['1', 'y', 'yes'].includes(r.has_error.toLowerCase())).length })
}
const toCsv = a => { const c = Object.keys(a[0]); return [c.join(','), ...a.map(r => c.map(k => esc(String(r[k]))).join(','))].join('\n') + '\n' };
fs.writeFileSync(path.join(OUT, 'fact-rates.csv'), toCsv(rates)); fs.writeFileSync(path.join(OUT, 'sourcing-balance.csv'), toCsv(bal));
const cl = [];
for (const g of groups) {
  const n = all.filter(r => r.group == g && r.counts_as_news == '1'); const u = {}, ch = {};
  for (const r of n) { for (const c of (r.no_claims_unchecked + ';' + r.yes_claims_unchecked).split(/[;,]/).map(s => s.trim()).filter(has)) u[c] = (u[c] || 0) + 1; for (const c of (r.claims_checked || '').split(/[;,]/).map(s => s.trim()).filter(has)) ch[c] = (ch[c] || 0) + 1 }
  for (const c of new Set([...Object.keys(u), ...Object.keys(ch)])) cl.push({ group: g, claim_code: c, side: c[0] == 'N' ? 'No' : 'Yes', stories_unchecked: u[c] || 0, stories_checked: ch[c] || 0 })
}
cl.sort((a, b) => a.group.localeCompare(b.group) || b.stories_unchecked - a.stories_unchecked || a.claim_code.localeCompare(b.claim_code));
fs.writeFileSync(path.join(OUT, 'claim-checks.csv'), toCsv(cl));
console.log('stories', all.length, 'news', groups.map(g => g + ':' + all.filter(r => r.group == g && r.counts_as_news == '1').length).join(' '));