// Builds the content parts for FactHub dataset wa-i645-media-coding:
// media_dataset_parts.json (sources, tables, headline analysis rows) and prints artifact declarations.
// Usage: node millionaires-tax/data/media_dataset_build.js
const fs = require('fs'), path = require('path'), crypto = require('crypto');
const dir = __dirname;
const RET = '2026-10-21';

const OUTLETS = [
  ['seattle-times', 'Seattle Times', 'https://www.seattletimes.com/', 'news'],
  ['spokesman-review', 'Spokesman-Review', 'https://www.spokesman.com/', 'news'],
  ['columbian', 'The Columbian', 'https://www.columbian.com/', 'news'],
  ['everett-herald', 'Everett Herald', 'https://www.heraldnet.com/', 'news'],
  ['tri-city-herald', 'Tri-City Herald', 'https://www.tri-cityherald.com/', 'news'],
  ['wa-state-standard', 'Washington State Standard', 'https://washingtonstatestandard.com/', 'news'],
  ['kuow', 'KUOW', 'https://www.kuow.org/', 'news'],
  ['cascade-pbs', 'Cascade PBS', 'https://www.cascadepbs.org/', 'news'],
  ['the-stranger', 'The Stranger', 'https://www.thestranger.com/', 'news'],
  ['deseret-news', 'Deseret News', 'https://www.deseret.com/', 'news'],
  ['cascadia-daily', 'Cascadia Daily News', 'https://www.cascadiadaily.com/', 'news'],
  ['komo', 'KOMO', 'https://komonews.com/', 'news'],
  ['king5', 'KING 5', 'https://www.king5.com/', 'news'],
  ['kiro7', 'KIRO 7', 'https://www.kiro7.com/', 'news'],
  ['kxly', 'KXLY', 'https://www.kxly.com/', 'news'],
  ['fox13', 'Fox 13 Seattle', 'https://www.fox13seattle.com/', 'news'],
  ['axios-seattle', 'Axios Seattle', 'https://www.axios.com/local/seattle', 'news'],
  ['krem', 'KREM', 'https://www.krem.com/', 'news'],
  ['center-square', 'The Center Square', 'https://www.thecentersquare.com/washington/', 'news'],
  ['seattle-red', 'Seattle Red/KTTH', 'https://seattlered.com/', 'news'],
  ['mynorthwest', 'MyNorthwest/KIRO Newsradio', 'https://mynorthwest.com/', 'news'],
  ['wpc', 'Washington Policy Center', 'https://www.washingtonpolicy.org/', 'advocacy'],
  ['daily-wire', 'The Daily Wire', 'https://www.dailywire.com/', 'news'],
  ['lynnwood-times', 'Lynnwood Times', 'https://lynnwoodtimes.com/', 'news'],
  ['npi', 'NPI Cascadia Advocate', 'https://www.nwprogressive.org/weblog/', 'partyToDispute'],
  ['the-stand', 'The Stand', 'https://www.thestand.org/', 'partyToDispute'],
];
const NOTE = {
  'wpc': 'Free-market think tank that argued for I-645 (repeal); its staff are quoted as Yes-side sources. Its own I-645 posts are coded rows (opinion).',
  'npi': 'Northwest Progressive Institute, part of the coalition opposing I-645; it published the No coalition\'s legal memo. Its blog posts are coded rows (opinion).',
  'the-stand': 'News site of the Washington State Labor Council, AFL-CIO, which opposes I-645 and whose president debated for the No side. Its posts are coded rows.',
};
const sources = OUTLETS.map(([id, name, url, kind]) => ({
  id: 'outlet-' + id, name, url, retrievedAt: RET,
  role: `Publisher of coded I-645 stories. Each story's exact URL is in the stories table (column url); the publisher's site is listed here.`,
  licenseName: 'All rights reserved (publisher copyright)', licenseUrl: null,
  redistribution: 'Story text is not redistributed. The dataset holds coding, story metadata and quotes of 25 words or fewer.',
  artifactId: null, readUrl: null,
  reliability: { publisherKind: kind, interest: 'none',
    note: NOTE[id] || `News publisher whose coverage is the subject of the coding; its I-645 stories (June 1 to Oct 21, 2026) are rows in the stories table. It has no stake in the counts beyond its own coverage.` },
}));
sources.push(
  { id: 'wrc-i645-brief', name: 'Washington Research Council, "New brief: I-645: Repealing the State Individual Income Tax" (Sept. 22, 2026)',
    url: 'https://researchcouncil.org/new-brief-i-645-repealing-the-state-individual-income-tax/', retrievedAt: RET,
    role: 'Anchor for facts O3b (no current appropriation relies on the tax) and O4 (the 2027-29 outlook assumes the revenue).',
    licenseName: 'All rights reserved', licenseUrl: null, redistribution: 'Linked only.', artifactId: null, readUrl: 'https://researchcouncil.org/wp-content/uploads/Initiative645.pdf',
    reliability: { publisherKind: 'other', interest: 'none', note: 'Business-funded nonpartisan fiscal research group. Its brief states both that no current appropriation relies on the tax and that the four-year outlook assumes $2.698 billion from it in 2027-29.' } },
  { id: 'sos-voters-pamphlet', name: 'Washington Secretary of State, 2026 General Election Voters\' Pamphlet (Edition 10, Spokane)',
    url: 'https://www.sos.wa.gov/sites/default/files/2026-10/Voters%20Pamphlet%202026%20-%20Edition%2010%20-%20Spokane.pdf', retrievedAt: RET,
    role: 'Anchor for fact O1 (the Attorney General\'s explanatory statement says the tax relief would remain in place) and for the ballot sentence text.',
    licenseName: 'Washington State government publication', licenseUrl: null, redistribution: 'Linked only.', artifactId: null, readUrl: null,
    reliability: { publisherKind: 'government', interest: 'none', note: 'Official state publication. The explanatory statement and ballot disclosure are written by the Attorney General\'s office; the statewide text is the same in every edition.' } },
  { id: 'erfc-outlook-apr-2026', name: 'Economic and Revenue Forecast Council, enacted 2026 supplemental budget outlook (Apr. 22, 2026)',
    url: 'https://erfc.wa.gov/sites/default/files/2026-04/20260422_Enacted_Sup_BO_Adopted.pdf', retrievedAt: RET,
    role: 'Anchor for fact O4: the outlook counts ESSB 6346 revenue in 2027-29.',
    licenseName: 'Washington State government publication', licenseUrl: null, redistribution: 'Linked only.', artifactId: null, readUrl: null,
    reliability: { publisherKind: 'government', interest: 'none', note: 'State council that adopts the official four-year balanced-budget outlook; lists ESSB 6346 at +$2,309.18 million (net) for 2027-29.' } },
);

const STORY_FIELDS = [
  ['story_id', 'string', 'Row identifier assigned by the coder (stable within this release).'],
  ['outlet', 'string', 'Publishing outlet (masthead), one of 26.'],
  ['outlet_detail', 'string', 'Outlet as recorded by the coder, e.g. "Seattle Times via Spokesman-Review" for a syndicated copy.'],
  ['outlet_group', 'string', 'mainstream, conservative or progressive, assigned by the audit design before coding (not by our judgment of each outlet).'],
  ['date', 'date', 'Publication date (YYYY-MM-DD).'], ['author', 'string', 'Byline as published.'],
  ['headline', 'string', 'Headline as published.'], ['url', 'string', 'Story URL. Copies used to read blocked pages are listed in notes.'],
  ['read_status', 'string', 'full or partial (paywall). No row is unread in this release.'],
  ['type', 'string', 'news, explainer, fact_check, broadcast (counted as news), opinion or editorial.'],
  ['yes_sources', 'integer', 'Distinct Yes-side (repeal) advocates quoted or paraphrased.'],
  ['no_sources', 'integer', 'Distinct No-side advocates quoted or paraphrased.'],
  ['neutral_sources', 'integer', 'Neutral sources (agencies, courts, pollsters, nonpartisan analysts).'],
  ['first_word', 'string', 'Side of the first advocate quoted: yes, no, neutral or none.'],
  ['last_word', 'string', 'Side of the last advocate quoted.'],
  ['rebuttal', 'string', 'Y if the other side responds to the story\'s lead claim; N if not; NA for opinion.'],
  ['headline_term', 'string', 'How the headline names the tax.'], ['loaded_terms', 'string', 'Loaded words; Q: prefix = only inside a quotation.'],
  ['no_claims_unchecked', 'string', 'Semicolon list of No-campaign claim codes repeated with no context on that claim (codebook).'],
  ['yes_claims_unchecked', 'string', 'Semicolon list of Yes-campaign claim codes repeated with no context.'],
  ['claims_checked', 'string', 'Claim codes the story itself contextualized or checked.'],
  ['O1', 'integer', '1 if the story says the tax relief survives repeal (AG statement); 0 if not.'],
  ['O2', 'integer', '1 if it says the revenue goes to the general fund / is not dedicated to K-12.'],
  ['O3a', 'integer', '1 if it says collections start in 2029.'],
  ['O3b', 'integer', '1 if it says no current appropriation relies on the tax (in any voice; see O3b_voice).'],
  ['O4', 'integer', '1 if it says the 2027-29 budget outlook counts on the revenue (in any voice; see O4_voice).'],
  ['O5', 'integer', '1 if it says lawmakers, not voters, could lower the $1 million threshold.'],
  ['O6', 'integer', '1 if it says the revenue estimate is static or the revenue volatile.'],
  ['O7', 'integer', '1 if it reports the No side\'s union funding or that No outraised Yes.'],
  ['O8', 'integer', '1 if it mentions Heywood\'s (or Yes donors\') personal financial interest.'],
  ['O9', 'integer', '1 if it mentions the legal risk I-645\'s text poses to the capital gains tax.'],
  ['O3b_voice', 'string', 'fact (reporter or neutral source) or advocate (only in a campaign side\'s quote) when O3b = 1.'],
  ['O4_voice', 'string', 'fact or advocate when O4 = 1.'],
  ['has_error', 'integer', '1 if the coder recorded a factual error in reporter or outlet voice, minor ones included.'],
  ['errors', 'string', 'The error, the correct fact, and whether a correction was found.'],
  ['ballot_sentence', 'string', 'none, mentioned, quoted or critiqued.'], ['dhm_poll', 'string', 'Y if the story reports the DHM poll\'s with/without-sentence result; partial; N.'],
  ['key_quote', 'string', 'Verbatim passage of 25 words or fewer illustrating the coding.'],
  ['sources_detail', 'string', 'Who was counted on each side.'], ['notes', 'string', 'Coder notes.'],
];
const OMIT_UNIT = { O1: 1, O2: 1, O3a: 1, O3b: 1, O4: 1, O5: 1, O6: 1, O7: 1, O8: 1, O9: 1, yes_sources: 1, no_sources: 1, neutral_sources: 1, has_error: 1 };
const field = ([name, type, description], extra = {}) => ({ name, type, description, units: OMIT_UNIT[name] ? (/sources/.test(name) ? 'count' : '0/1') : null,
  nullable: !['story_id', 'outlet', 'outlet_group', 'url', 'read_status', 'type'].includes(name),
  valueType: ['O3b_voice', 'O4_voice'].includes(name) ? 'derived' : 'observed', derivation: ['O3b_voice', 'O4_voice'].includes(name) ? 'Voice re-pass of every news text, recorded in media_coding_overrides.json.' : null,
  sourceIds: [], ...extra });

const derivedField = (name, type, description, units = null) => ({ name, type, description, units, nullable: false, valueType: 'derived',
  derivation: 'Computed from the stories table by media_audit_derived.js.', sourceIds: [] });
const tables = [
  { id: 'stories', title: 'Coded I-645 stories', description: 'One row per story about I-645 published June 1 to Oct 21, 2026 by 26 Washington and national outlets, coded with one codebook (media_coding_codebook.md).',
    artifactId: 'stories-csv', unitOfObservation: 'One news story, explainer, broadcast write-up, opinion piece or editorial', fields: STORY_FIELDS.map(f => field(f)) },
  { id: 'omission-rates', title: 'How often stories included each key fact', description: 'For each group of stories and each fact (O1-O9, plus the voice split for O3b and O4), how many stories included it.',
    artifactId: 'omission-csv', unitOfObservation: 'Story group by fact',
    fields: [derivedField('group', 'string', 'Story group, e.g. "Mainstream news" (news, explainer, fact check and broadcast stories from mainstream outlets).'),
      derivedField('fact_code', 'string', 'Codebook fact code; -fact / -advocate suffix = voice split.'), derivedField('fact', 'string', 'Plain-language fact.'),
      derivedField('rebuts_claims_of', 'string', 'Which campaign\'s claims the fact rebuts.'), derivedField('stories_including', 'integer', 'Stories that included the fact.', 'stories'),
      derivedField('stories_coded', 'integer', 'Stories coded for the fact.', 'stories'), derivedField('share_including', 'number', 'stories_including / stories_coded.', 'fraction')] },
  { id: 'claim-checks', title: 'Campaign claims: unchecked vs contextualized', description: 'For each story group and graded campaign claim, how many stories repeated it with no context and how many contextualized it.',
    artifactId: 'claims-csv', unitOfObservation: 'Story group by campaign claim',
    fields: [derivedField('group', 'string', 'Story group.'), derivedField('claim_code', 'string', 'Codebook claim code.'), derivedField('claim_side', 'string', 'Yes or No campaign.'),
      derivedField('claim', 'string', 'The claim.'), derivedField('stories_unchecked', 'integer', 'Stories repeating it with no context.', 'stories'),
      derivedField('stories_contextualized', 'integer', 'Stories that contextualized or checked it.', 'stories'), derivedField('share_contextualized', 'number', 'Contextualized share.', 'fraction')] },
  { id: 'sourcing-balance', title: 'Sourcing balance by story group', description: 'Average sources per side, first and last word, missing rebuttals, unchecked-claim counts and claim-check rates by story group.',
    artifactId: 'sourcing-csv', unitOfObservation: 'Story group',
    fields: ['group', 'stories', 'avg_yes_sources', 'avg_no_sources', 'first_word_yes', 'first_word_no', 'last_word_yes', 'last_word_no', 'no_rebuttal',
      'stories_with_unchecked_no_claim', 'stories_with_unchecked_yes_claim', 'no_claim_instances', 'no_claim_instances_contextualized', 'yes_claim_instances',
      'yes_claim_instances_contextualized', 'stories_with_error'].map(n => derivedField(n, n === 'group' ? 'string' : /avg/.test(n) ? 'number' : 'integer',
      n === 'group' ? 'Story group.' : n.replace(/_/g, ' ') + '.', n === 'group' ? null : /avg/.test(n) ? 'sources per story' : 'count')) },
  { id: 'coder-agreement', title: 'Agreement between the first coding and a blind second coder', description: 'Percent agreement and Cohen\'s kappa on a stratified random sample of 20 news stories (14 mainstream, 6 conservative; seed 645).',
    artifactId: 'agreement-csv', unitOfObservation: 'Coded field',
    fields: [derivedField('field', 'string', 'Coded field; O3b and O4 shown before and after the voice re-pass.'), derivedField('stories', 'integer', 'Stories compared.', 'stories'),
      derivedField('agree', 'integer', 'Stories where both coders agreed.', 'stories'), derivedField('percent_agreement', 'number', 'Agreement.', 'percent'),
      derivedField('cohens_kappa', 'number', 'Cohen\'s kappa (blank when one category only).', null), derivedField('disagreements', 'string', 'story_id:first→second for each disagreement.')] },
];

// Headline analysis rows (inline).
const der = JSON.parse(fs.readFileSync(path.join(dir, 'media_derived.json'), 'utf8'));
const pick = (g, c) => der.omission.find(r => r.group === g && r.fact_code === c);
const headline = [];
for (const g of ['Mainstream news', 'Conservative news'])
  for (const c of ['O3b-fact', 'O3b', 'O1', 'O2', 'O4-fact', 'O4', 'O8']) {
    const r = pick(g, c); headline.push({ group: g, fact_code: c, fact: r.fact, rebuts_claims_of: r.rebuts_claims_of, stories_including: r.stories_including, stories_coded: r.stories_coded, share_including: r.share_including });
  }
fs.writeFileSync(path.join(dir, 'media_dataset_parts.json'), JSON.stringify({ sources, tables, headline }, null, 1));
const f = 'media_dataset_parts.json', b = fs.readFileSync(path.join(dir, f));
console.log(f, b.length, 'sha256:' + crypto.createHash('sha256').update(b).digest('hex'));
console.log(JSON.stringify(headline.map(r => `${r.group} ${r.fact_code} ${r.stories_including}/${r.stories_coded}`)));