// Builds the platform policy matrix CSV and the FactHub dataset content JSON.
// Usage: node build-policy-dataset.js
const fs = require('fs');
const crypto = require('crypto');
const { U, TOPICS, R } = require('./policy-rows');

const KEY = 'social-media-viewpoint-policies';
const CHECKED = '2026-10-10';
const OUT = __dirname + '/../out/' + KEY;
fs.mkdirSync(OUT, { recursive: true });

// archive_source results (2026-10-10). Missing entries: capture failed (Wayback rate limiting) or site blocked fetch.
const A = {
  'x-abusive': ['https://web.archive.org/web/20261001205650/https://help.x.com/en/rules-and-policies/abusive-behavior', '2026-10-01T20:56:50Z', null],
  'x-hateful': ['https://web.archive.org/web/20261001072939/https://help.x.com/en/rules-and-policies/hateful-conduct-policy', '2026-10-01T07:29:39Z', null],
  'x-election': ['https://web.archive.org/web/20261006231331/https://help.x.com/en/rules-and-policies/election-integrity-policy', '2026-10-06T23:13:31Z', null],
  'x-violent-entities': ['https://web.archive.org/web/20260821070844/https://help.x.com/en/rules-and-policies/violent-entities', '2026-08-21T07:08:44Z', null],
  'x-violent-content': ['https://web.archive.org/web/20260925090125/https://help.x.com/en/rules-and-policies/violent-content', '2026-09-25T09:01:25Z', null],
  'ts-tos': ['https://web.archive.org/web/20261006193924/https://help.truthsocial.com/legal/terms-of-service', '2026-10-06T19:39:24Z', null],
  'ts-guidelines': ['https://web.archive.org/web/20261010222002/https://help.truthsocial.com/community-guidelines-page/', '2026-10-10T22:20:02Z', null],
  'rumble-terms': ['https://web.archive.org/web/20261010222018/https://rumble.com/s/terms', '2026-10-10T22:20:18Z', 'sha256:4934bc655e799e0d6a2283417897e7988c3f9c528e46c14d8e17048f0ad9aeed'],
  'reddit-rules': ['https://web.archive.org/web/20261010222036/https://redditinc.com/policies/reddit-rules', '2026-10-10T22:20:36Z', 'sha256:9bc598f3d20ffb8c6a0dcca9a4b2a89b9ed2ca9433bcd8627835878e4b75b94a'],
  'reddit-transactions': ['https://web.archive.org/web/20260820050103/https://support.reddithelp.com/hc/en-us/articles/360043513471-Illegal-or-Prohibited-Transactions', '2026-08-20T05:01:03Z', null],
  'yt-hate': [null, null, 'sha256:3a92ff75153b556a264e5f0181c0f968cc7a44a2daf7e4e37704c6cd491969a2'],
  'yt-harassment': ['https://web.archive.org/web/20261008234842/https://support.google.com/youtube/answer/2802268', '2026-10-08T23:48:42Z', 'sha256:c79fc24bd878770b54561cc5369ff14213aab232d77cd680429ca6ae45870059'],
  'yt-medical': [null, null, 'sha256:0a8c9dc8f840f9ce75e581c0422172cc1e2ab1e681782dcc78d5423d14462f85'],
  'yt-elections': ['https://web.archive.org/web/20261010222204/https://support.google.com/youtube/answer/10835034', '2026-10-10T22:22:04Z', 'sha256:8425b088e6bb972a326185f3a6d86fadb33fa439528024769b17c57c3d81fe72'],
  'yt-extremism': ['https://web.archive.org/web/20261010222257/https://support.google.com/youtube/answer/9229472', '2026-10-10T22:22:57Z', 'sha256:619ebaf8a3d74abae17ee0014e2040b98ba657cea2a7f9a17b3bc4ad03f12f3d'],
  'yt-regulated-goods': ['https://web.archive.org/web/20261010222316/https://support.google.com/youtube/answer/9229611', '2026-10-10T22:23:16Z', 'sha256:f2bb95a3a49020849582f46668368c514d53171a4bbadd0b408c93b31d93c10c'],
  'yt-graphic': ['https://web.archive.org/web/20261010222423/https://support.google.com/youtube/answer/2802008', '2026-10-10T22:24:23Z', 'sha256:a2fc9356fdd2f689231c93ac7d1f1365cb3a6d8835432805911d997d3b1f68c0'],
  'adsense-climate': ['https://web.archive.org/web/20261010223011/https://support.google.com/adsense/answer/11214106', '2026-10-10T22:30:11Z', 'sha256:31a91bead1016dc05046e2671b33b60a01b71b86d089f690eff1ab947374cb47'],
  'meta-hateful': ['https://web.archive.org/web/20261010222531/https://transparency.meta.com/policies/community-standards/hateful-conduct/', '2026-10-10T22:25:31Z', 'sha256:7d08e96e4d6de8ea09ca2d0e6fba1407600c1a7c7f3ab7a711c4e9b8f316c4c6'],
  'meta-bullying': ['https://web.archive.org/web/20261010222548/https://transparency.meta.com/policies/community-standards/bullying-harassment/', '2026-10-10T22:25:48Z', 'sha256:7a0403209138fc203623d05c083738aca2ff257d512e2031b6fbd4c99462259a'],
  'meta-misinfo': ['https://web.archive.org/web/20261010222606/https://transparency.meta.com/policies/community-standards/misinformation/', '2026-10-10T22:26:06Z', 'sha256:e265f1f773711310158278bc43bdd151976dd3830a46a83dac69a7da889fc5a1'],
  'meta-doi': ['https://web.archive.org/web/20261010222625/https://transparency.meta.com/policies/community-standards/dangerous-individuals-organizations/', '2026-10-10T22:26:25Z', 'sha256:8caf21be337547e246be68fe881dac72df6ecef2fe8d4c90d75690c5761f6e51'],
  'meta-restricted': ['https://web.archive.org/web/20261010222643/https://transparency.meta.com/policies/community-standards/restricted-goods-services/', '2026-10-10T22:26:43Z', 'sha256:6716a23028045939be556fa641d2de2e1027b73d4feed4842c06f1b60a7af6cf'],
  'meta-graphic': [null, null, 'sha256:3525847d45a6a6e6838e130822f44a702b727edeeeaf8dbc8065019ecb95f4ad'],
  'oversight-river': ['https://web.archive.org/web/20261010222708/https://www.oversightboard.com/decision/bun-86tj0rk5/', '2026-10-10T22:27:08Z', 'sha256:19929cc8637dcfa9ea2a388cef2ad63ad8f9d88eade1a2b1b70628fde606d0df'],
  'bsky-guidelines': ['https://web.archive.org/web/20261010222735/https://bsky.social/about/support/community-guidelines', '2026-10-10T22:27:35Z', 'sha256:bf85682402204379f3a67423a1f3aef5771b8b7ec69a91c012f5f0cf9dcfa97b'],
  'tiktok-sc': ['https://web.archive.org/web/20261006144951/https://www.tiktok.com/safety/en/policies-and-engagement/safety-civility', '2026-10-06T14:49:51Z', 'sha256:b05084ab2131622223b65368ffbb1327706e8acc4801526baf5cd3bcc99574f5'],
  'tiktok-ia': [null, null, 'sha256:f4eed1af868852ddb0c7408382640c34ac12b16dcdfe2242a5d1155ca26b4a56'],
  'tiktok-rca': ['https://web.archive.org/web/20260928082833/https://www.tiktok.com/safety/en/policies-and-engagement/regulated-commercial-activities', '2026-09-28T08:28:33Z', 'sha256:8886cf4cd84251e98cb564c45cadfba4e46ea7608518736e8d773fbb63966ff7'],
  'tiktok-smt': ['https://web.archive.org/web/20261010222809/https://www.tiktok.com/safety/en/policies-and-engagement/sensitive-mature-themes', '2026-10-10T22:28:09Z', 'sha256:83bef2726e0b28fb676462f241d8d31bfe9416e0474a2c56e133e1a5b3d264d1'],
  'mastodon-rules': ['https://web.archive.org/web/20261010222825/https://mastodon.social/api/v1/instance/rules', '2026-10-10T22:28:25Z', 'sha256:1459ff6e0b436c655b7f232c03a2f12208c0855c72937d75704657a88936b19f'],
  'mastodon-covenant': ['https://web.archive.org/web/20261010222835/https://joinmastodon.org/covenant', '2026-10-10T22:28:35Z', 'sha256:14e8e700ef95a1ae19a07f95aa184c3a112489c3d266e3a049f4ea5d905ae983'],
};

const NAMES = {
  'x-abusive': 'X Help Center: Abuse and harassment policy', 'x-hateful': 'X Help Center: Hateful conduct policy',
  'x-election': 'X Help Center: Civic integrity policy', 'x-rules': 'X Help Center: The X Rules',
  'x-violent-entities': 'X Help Center: Violent and hateful entities policy', 'x-violent-content': 'X Help Center: Violent content policy',
  'x-illegal-regulated': 'X Help Center: Illegal or regulated goods and services policy',
  'ts-tos': 'Truth Social: Terms of Service', 'ts-guidelines': 'Truth Social: Community Guidelines',
  'rumble-terms': 'Rumble: Terms and Conditions (last updated September 1, 2026)',
  'reddit-rules': 'Reddit Rules', 'reddit-hate': 'Reddit Help: Hate', 'reddit-quarantine': 'Reddit Help: Quarantined communities',
  'reddit-transactions': 'Reddit Help: Illegal or prohibited transactions', 'reddit-terrorism': 'Reddit Help: How Reddit fights the dissemination of terrorist content',
  'reddit-violence': 'Reddit Help: Violence',
  'yt-hate': 'YouTube Help: Hate speech policy', 'yt-harassment': 'YouTube Help: Harassment and cyberbullying policies',
  'yt-medical': 'YouTube Help: Medical misinformation policy', 'yt-elections': 'YouTube Help: Elections misinformation policies',
  'yt-extremism': 'YouTube Help: Violent extremist or criminal organizations policy', 'yt-regulated-goods': 'YouTube Help: Sale of illegal or regulated goods or services policies',
  'yt-graphic': 'YouTube Help: Violent or graphic content policies', 'adsense-climate': 'Google Publisher Policies: Climate change misinformation (monetization)',
  'meta-hateful': 'Meta Community Standards: Hateful conduct', 'meta-bullying': 'Meta Community Standards: Bullying and harassment',
  'meta-misinfo': 'Meta Community Standards: Misinformation', 'meta-doi': 'Meta Community Standards: Dangerous organizations and individuals',
  'meta-restricted': 'Meta Community Standards: Restricted goods and services', 'meta-graphic': 'Meta Community Standards: Violent and graphic content',
  'oversight-river': 'Oversight Board decision on "From the River to the Sea" posts (September 4, 2024)',
  'wt-zionist': 'Washington Times: Meta to remove posts targeting "Zionists" (July 9, 2024)',
  'bsky-guidelines': 'Bluesky Community Guidelines (last updated September 19, 2025)',
  'tiktok-sc': 'TikTok Community Guidelines: Safety and civility', 'tiktok-ia': 'TikTok Community Guidelines: Integrity and authenticity',
  'tiktok-rca': 'TikTok Community Guidelines: Regulated and commercial activities', 'tiktok-smt': 'TikTok Community Guidelines: Sensitive and mature themes',
  'mastodon-rules': 'mastodon.social server rules (instance rules API)', 'mastodon-covenant': 'Mastodon Server Covenant',
};

const csvCell = v => { const s = v == null ? '' : String(v); return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s; };
const header = ['platform', 'topic_id', 'topic', 'tier', 'in_viewpoint_index', 'verdict', 'verdict_detail', 'policy_quote', 'quote_verified', 'source_id', 'policy_url', 'checked_date'];
const rows = R.map(([platform, tid, verdict, detail, quote, sid]) => {
  const [topic, tier, idx] = TOPICS[tid];
  return [platform, tid, topic, tier, idx ? 'true' : 'false', verdict, detail, quote, quote ? 'verbatim' : 'no_quote', sid, U[sid], CHECKED];
});
const csv = [header, ...rows].map(r => r.map(csvCell).join(',')).join('\n') + '\n';
fs.writeFileSync(OUT + '/policy-matrix.csv', csv);

// Viewpoint index: topics T01-T05 and T06B, counting removed/restricted.
const platforms = [...new Set(R.map(r => r[0]))];
const index = platforms.map(p => {
  const rs = R.filter(r => r[0] === p && TOPICS[r[1]][2]);
  const c = v => rs.filter(r => r[2] === v).length;
  return { platform: p, restricted_topics: c('removed') + c('restricted'), removed: c('removed'), restricted: c('restricted'), vague: c('vague'), allowed_or_not_addressed: c('allowed') + c('not_addressed'), topics_counted: rs.length };
}).sort((a, b) => a.restricted_topics - b.restricted_topics || a.vague - b.vague);

const sha = f => 'sha256:' + crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
const art = (id, role, file, mediaType, desc) => ({ id, role, fileName: require('path').basename(file), mediaType, byteLength: fs.statSync(file).size, contentDigest: sha(file), description: desc, access: 'public' });
fs.copyFileSync(__dirname + '/policy-rows.js', OUT + '/policy-rows.js');
fs.copyFileSync(__dirname + '/verify-quotes.js', OUT + '/verify-quotes.js');
fs.copyFileSync(__filename, OUT + '/build-policy-dataset.js');

const sourceIds = Object.keys(U);
const sources = sourceIds.map(id => {
  const s = { id, name: NAMES[id], url: U[id], retrievedAt: CHECKED, role: id === 'wt-zionist' ? 'Secondary news report of a policy change (Meta announcement page no longer loads)' : (id === 'oversight-river' ? 'Primary: Oversight Board decision applying Meta policy' : 'Primary: platform policy page'), licenseName: 'Publisher copyright; short quotations used for commentary', redistribution: 'Not redistributed; linked only' };
  const a = A[id];
  if (a && a[0]) { s.archiveUrl = a[0]; s.archivedAt = a[1]; }
  if (a && a[2]) s.contentDigest = a[2];
  return s;
});

const field = (name, type, description, extra = {}) => ({ name, type, description, nullable: false, valueType: 'observed', ...extra });
const content = {
  key: KEY, status: 'Final', rulesVersion: '2.14',
  title: 'Written viewpoint rules on nine US social platforms (October 2026)',
  description: 'For nine social platforms and 19 contested topics, what each platform\'s own written policy says it does: remove, restrict reach, allow, or not address the content, with a verbatim quote and link for every cell.',
  selectionRationale: 'Built to test the claim that X is the only social media site that does not censor. Platforms are the largest US networks plus the alternatives promoted by politicians on the left and right (Bluesky, Mastodon, Truth Social, Rumble). Topics are current political flashpoints where moderation rules can restrict a viewpoint, plus two baseline rules every platform shares (voter suppression and support for designated terrorist groups) for comparison.',
  topics: ['content moderation', 'free speech', 'social media', 'censorship'],
  geography: ['United States'],
  temporalCoverage: 'Policies as published on 2026-10-10',
  unitOfObservation: 'One platform and one topic',
  updateCadence: 'Irregular; re-check when platforms revise policies',
  curator: 'FactHub',
  reproducibilityStatus: 'Partially reproducible',
  reproducibilityExplanation: 'Every verdict cites a live policy page and, where one exists, an archived copy. Quotes were checked verbatim against the live pages on 2026-10-10: automatically with verify-quotes.js for pages that allow automated fetches, and in a browser for X, Truth Social, Reddit Help and Meta, which block automated fetches or render text with JavaScript. Verdicts are FactHub\'s reading of each policy and involve judgment, especially where rules are vague.',
  limitations: [
    'Written policy only. The dataset does not measure how often or how evenly platforms enforce these rules; no cross-platform enforcement data exists.',
    'Verdicts are interpretations. "Vague" marks a general rule that could cover the topic but does not name it; reasonable readers may classify some cells differently.',
    'mastodon.social is one server. Other Mastodon servers set their own rules and can block whole servers (defederation).',
    'Reddit rows are site-wide rules only. Individual subreddits add their own rules and moderator bans, which are not captured.',
    'Bluesky users can also subscribe to user-run labelers and block lists, which are not Bluesky\'s rules and are not captured.',
    'Truth Social\'s discrimination clause applies to "Interactive Content" (comments and replies); whether it covers ordinary posts is unclear.',
    'Meta\'s July 2024 "Zionist" rule (T11C) is sourced to contemporaneous news coverage because Meta\'s announcement page no longer loads.',
    'Some sources lack an archived copy because the Wayback Machine was rate-limiting captures on 2026-10-10; X, Truth Social and Reddit Help also block FactHub\'s server-side fetch, so no content digest was recorded for them.',
    'The viewpoint index counts six topics equally. Counting vaccine and climate misinformation rules as viewpoint restrictions is itself a judgment; some readers treat them as accuracy rules.',
    'Platforms change policies often; verdicts reflect 2026-10-10 only.',
  ],
  sources,
  artifacts: [
    art('policy-matrix', 'curatedTable', OUT + '/policy-matrix.csv', 'text/csv', 'One row per platform and topic: verdict, explanation, verbatim policy quote, and policy URL.'),
    art('policy-rows-js', 'transformationManifest', OUT + '/policy-rows.js', 'text/javascript', 'Source data for the matrix: policy URLs, topic definitions, and each verdict with its quote.'),
    art('verify-quotes-js', 'qualityReport', OUT + '/verify-quotes.js', 'text/javascript', 'Script that fetches each policy page and checks every quote appears verbatim.'),
    art('build-js', 'transformationManifest', OUT + '/build-policy-dataset.js', 'text/javascript', 'Script that builds the CSV and this dataset\'s content from policy-rows.js.'),
  ],
  tables: [{
    id: 'policy-matrix', title: 'Platform policy by topic', artifactId: 'policy-matrix', unitOfObservation: 'One platform and one topic',
    description: 'What each platform\'s written policy says it does with content on each topic, as of 2026-10-10.',
    fields: [
      field('platform', 'string', 'Platform name. Meta covers Facebook, Instagram and Threads, which share Meta\'s Community Standards. mastodon.social is the largest Mastodon server.'),
      field('topic_id', 'string', 'Stable topic code (T01-T12B).'),
      field('topic', 'string', 'Plain-language topic description.'),
      field('tier', 'string', 'viewpoint (a contested opinion or claim), baseline (a rule nearly every platform shares), or scope (which groups a hate rule protects).', { valueType: 'derived', derivation: 'Assigned by FactHub per topic.' }),
      field('in_viewpoint_index', 'boolean', 'Whether the topic counts toward the viewpoint index (T01-T05 and T06B).', { valueType: 'derived', derivation: 'Assigned by FactHub per topic.' }),
      field('verdict', 'string', 'removed, restricted (reduced reach, label, warning, demonetized, or not recommended), allowed (policy explicitly permits it), not_addressed (no rule found), vague (a general rule could cover it but does not name it), or n/a for scope rows.', { valueType: 'derived', derivation: 'FactHub reading of the quoted policy text.' }),
      field('verdict_detail', 'string', 'Explanation of the verdict, including exceptions and dated changes.', { valueType: 'derived', derivation: 'FactHub summary of the policy.' }),
      field('policy_quote', 'string', 'Verbatim excerpt from the policy page supporting the verdict; empty when no rule exists to quote.', { nullable: true, sourceIds }),
      field('quote_verified', 'string', 'verbatim if the quote was confirmed on the live page on 2026-10-10; no_quote if empty.', { valueType: 'derived', derivation: 'verify-quotes.js and browser checks.' }),
      field('source_id', 'string', 'Id of the source entry for the policy page.'),
      field('policy_url', 'string', 'URL of the policy page.'),
      field('checked_date', 'date', 'Date the policy was read.'),
    ],
  }],
  analyses: [{
    id: 'viewpoint-index', title: 'Viewpoint topics each platform restricts by written rule',
    question: 'Of six contested viewpoint topics, how many does each platform\'s written policy remove or restrict?',
    operation: 'Count rows with in_viewpoint_index = true by platform and verdict.',
    parameters: { topics: ['T01', 'T02', 'T03', 'T04', 'T05', 'T06B'], counted_as_restricting: ['removed', 'restricted'] },
    scope: 'Nine platforms, six topics, policies as of 2026-10-10.',
    resultShape: 'table', structuredResult: { rows: index },
    displayResult: index.map(r => `${r.platform}: ${r.restricted_topics} of 6 (${r.vague} vague)`).join('; '),
    methodologicalNotes: ['Topics: misgendering, gender-identity viewpoints, vaccine or medical claims, stolen-election claims, climate denial, and calls to exclude or deport immigrants.', 'Vague rules are counted separately and not as restrictions.'],
    limitations: ['Counts written rules, not enforcement.', 'Each topic is weighted equally.', 'Classifying vaccine and climate misinformation rules as viewpoint restrictions is a judgment call.'],
    executedAt: '2026-10-10T23:00:00Z',
  }],
  events: [
    { id: 'youtube-election-change', eventType: 'policy change', date: '2023-06-02', label: 'YouTube stops removing claims that past US presidential elections were stolen', description: 'Reported by news outlets at the time; the current elections policy linked here does not cover claims about past elections.', sourceIds: ['yt-elections'] },
    { id: 'meta-zionist-update', eventType: 'policy change', date: '2024-07-09', label: 'Meta removes attacks on "Zionists" used as a proxy for Jews or Israelis', description: 'Covers dehumanizing comparisons, calls for harm and denials of existence.', sourceIds: ['wt-zionist'] },
    { id: 'oversight-river-decision', eventType: 'ruling', date: '2024-09-04', label: 'Oversight Board upholds leaving up "From the River to the Sea" posts', description: 'The majority found the phrase does not have a single meaning.', sourceIds: ['oversight-river'] },
    { id: 'meta-jan-2025', eventType: 'policy change', date: '2025-01-07', label: 'Meta loosens hateful-conduct rules and ends US fact-checking', description: 'Explicitly allows some gender-identity and immigration viewpoints; Community Notes replace US fact-checkers from April 2025.', sourceIds: ['meta-hateful', 'meta-misinfo'] },
    { id: 'bluesky-guidelines-2025', eventType: 'policy change', date: '2025-09-19', label: 'Bluesky revises Community Guidelines', description: 'Current guidelines carry this last-updated date.', sourceIds: ['bsky-guidelines'] },
  ],
  changelog: [{ date: CHECKED, summary: 'First release: 9 platforms, 19 topics, 171 platform-topic rows, quotes verified against live pages on 2026-10-10.' }],
};
const parts = { sources: content.sources, tables: content.tables, analyses: content.analyses, events: content.events };
const partsBytes = Buffer.from(JSON.stringify(parts, null, 1) + '\n');
fs.writeFileSync(OUT + '/dataset-parts.json', partsBytes);
content.artifacts.push(art('dataset-parts', 'documentation', OUT + '/dataset-parts.json', 'application/json', 'Sources, table definitions, saved analysis and events for this dataset, as JSON.'));
for (const k of Object.keys(parts)) content[k] = { $artifact: 'dataset-parts', pointer: '/' + k };
fs.writeFileSync(OUT + '/content.json', JSON.stringify(content, null, 1));
fs.writeFileSync(OUT + '/content.min.json', JSON.stringify(content));
console.log('rows', rows.length, 'csv bytes', csv.length);
console.log(index);