// Builds the FactHub dataset content for Twitter/X legal demands to remove content.
// Input: ../x-government-removal-requests.csv and notes (compiled 2026-10-10 from X transparency reports).
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');

const KEY = 'x-content-removal-demands';
const SRC = path.join(__dirname, '..');
const OUT = path.join(__dirname, '..', 'out', KEY);
fs.mkdirSync(OUT, { recursive: true });
fs.copyFileSync(path.join(SRC, 'x-government-removal-requests.csv'), path.join(OUT, 'x-removal-demands.csv'));
fs.copyFileSync(path.join(SRC, 'x-government-removal-requests-notes.md'), path.join(OUT, 'notes.md'));
fs.copyFileSync(__filename, path.join(OUT, 'build-takedown-dataset.js'));

const parse = l => { const o = []; let c = '', q = false; for (let i = 0; i < l.length; i++) { const ch = l[i]; if (q) { if (ch === '"' && l[i + 1] === '"') { c += '"'; i++; } else if (ch === '"') q = false; else c += ch; } else if (ch === '"') q = true; else if (ch === ',') { o.push(c); c = ''; } else c += ch; } o.push(c); return o; };
const lines = fs.readFileSync(path.join(OUT, 'x-removal-demands.csv'), 'utf8').trim().split(/\r?\n/);
const head = parse(lines[0]);
const rows = lines.slice(1).map(l => { const r = parse(l); return Object.fromEntries(head.map((h, i) => [h, r[i]])); });
const num = v => (v === '' || v == null ? null : Number(v));

const volume = rows.filter(r => r.removal_requests_total !== '').map(r => ({ period: r.period, company: r.company, removal_requests: num(r.removal_requests_total), report_status: r.report_status }));
const rate = rows.filter(r => r.compliance_rate_pct !== '').map(r => ({ period: r.period, company: r.company, rate_pct: num(r.compliance_rate_pct), metric: r.company === 'X' ? 'X action rate' : 'Twitter compliance rate' }));

const sha = f => 'sha256:' + crypto.createHash('sha256').update(fs.readFileSync(f)).digest('hex');
const art = (id, role, file, mediaType, description) => ({ id, role, fileName: path.basename(file), mediaType, byteLength: fs.statSync(file).size, contentDigest: sha(file), description, access: 'public' });
const src = (id, name, url, role, a) => { const s = { id, name, url, retrievedAt: '2026-10-10', role, licenseName: 'Publisher copyright; figures reproduced as facts', redistribution: 'Figures extracted; source files not redistributed' }; if (a && a[0]) { s.archiveUrl = a[0]; s.archivedAt = a[1]; } if (a && a[2]) s.contentDigest = a[2]; return s; };

const sources = [
  src('twitter-data-2021h2', 'Twitter Transparency Report #20 data download (Jul-Dec 2021), removal and information request CSVs covering 2012-2021', 'https://transparency.x.com/content/dam/transparency-twitter/download/2021-jul-dec/Twitter-Transparency-Report-20-Jul-Dec-2021.zip', 'Primary: official data file for 2012-H1 to 2021-H2', ['https://web.archive.org/web/20261010224422/https://transparency.x.com/content/dam/transparency-twitter/download/2021-jul-dec/Twitter-Transparency-Report-20-Jul-Dec-2021.zip', '2026-10-10T22:44:22Z', 'sha256:bbf38aedb86879d1b0afb54b78c39e4b7f8208ee3f9102ff3cf1b70a3ee94aa8']),
  src('twitter-page-2021h2', 'Twitter Transparency: Removal requests, July-December 2021', 'https://transparency.x.com/en/reports/removal-requests/2021-jul-dec', 'Primary: headline compliance rate and cross-check of totals', ['https://web.archive.org/web/20261010224431/https://transparency.x.com/en/reports/removal-requests/2021-jul-dec', '2026-10-10T22:44:31Z', 'sha256:e4ea3cb66004d39fa06b573944cac3e394a738fd9ff939c0cef43b5dc876ffce']),
  src('x-blog-2023', 'X blog: An update on Twitter Transparency Reporting (April 25, 2023)', 'https://blog.x.com/en_us/topics/company/2023/an-update-on-twitter-transparency-reporting', 'Primary: summary figures for 2022-H1', null),
  src('x-report-2024h1', 'X Global Transparency Report, H1 2024 (PDF)', 'https://transparency.x.com/content/dam/transparency-twitter/2024/x-global-transparency-report-h1.pdf', 'Primary: 2024-H1 figures (p. 13)', ['https://web.archive.org/web/20260929115040/https://transparency.x.com/content/dam/transparency-twitter/2024/x-global-transparency-report-h1.pdf', '2026-09-29T11:50:40Z', 'sha256:ff3ef346ea5e5330170e35f0ed861232d0f16a83fc7d67d4081d8d7546bd7419']),
  src('x-report-2024h2', 'X Global Transparency Report, H2 2024 (PDF)', 'https://transparency.x.com/content/dam/transparency-twitter/2025/x-global-transparency-report_h2_2024.pdf', 'Primary: 2024-H2 figures (p. 13)', ['https://web.archive.org/web/20261010224537/https://transparency.x.com/content/dam/transparency-twitter/2025/x-global-transparency-report_h2_2024.pdf', '2026-10-10T22:45:37Z', 'sha256:faf1a08ae38485d577bfbe1f25853702fd482c5380ef060144506aeba94fb531']),
  src('x-report-2024h2-html', 'X 2025 Transparency Report (HTML version of the H2 2024 report)', 'https://transparency.x.com/en/reports/global-reports/2025-transparency-report', 'Primary: cross-check of 2024-H2 figures; also the latest report linked as of 2026-10-10', ['https://web.archive.org/web/20261009012749/https://transparency.x.com/en/reports/global-reports/2025-transparency-report', '2026-10-09T01:27:49Z', null]),
  src('rest-of-world-2023', 'Rest of World: Twitter is complying with more government demands under Elon Musk (2023)', 'https://restofworld.org/2023/elon-musk-twitter-government-orders/', 'Secondary context only: analysis of Lumen database submissions; not part of the series', ['https://web.archive.org/web/20261010224635/https://restofworld.org/2023/elon-musk-twitter-government-orders/', '2026-10-10T22:46:35Z', 'sha256:a794a4a6b70780c54c9bd82b6e36aa77905c3bbfc8296cabdf523daed7f44c6b']),
];
const ALL = ['twitter-data-2021h2', 'x-blog-2023', 'x-report-2024h1', 'x-report-2024h2'];
const f = (name, type, description, extra = {}) => ({ name, type, description, nullable: true, valueType: 'observed', sourceIds: ALL, ...extra });

const content = {
  key: KEY, status: 'Final', rulesVersion: '2.14',
  title: 'Twitter/X legal demands to remove content, 2012-2024',
  description: 'Half-yearly counts of legal demands to remove content that Twitter and X received worldwide, and the share they acted on, from the companies\' own transparency reports.',
  selectionRationale: 'Tests whether X under Elon Musk resists or complies with government censorship demands more than Twitter did. Uses only the companies\' own published reports so the comparison rests on figures X itself stands behind.',
  topics: ['content moderation', 'government censorship', 'transparency reports', 'X (Twitter)'],
  geography: ['Worldwide'],
  temporalCoverage: '2012-H1 to 2026-H1 (data for 2012-H1 to 2022-H1, 2024-H1 and 2024-H2; other periods unpublished)',
  unitOfObservation: 'One half-year reporting period',
  updateCadence: 'Half-yearly when X publishes a report',
  curator: 'FactHub',
  reproducibilityStatus: 'Reproducible',
  reproducibilityExplanation: '2012-2021 values are read directly from Twitter\'s official removal-requests CSV in the report #20 download and match the totals and headline rates printed on each report page for 2019-H2 to 2021-H2. 2022-H1 comes from an X blog post. 2024 values are copied from page 13 of each X Global Transparency Report and match the HTML version. notes.md documents every check and discrepancy.',
  limitations: [
    'Twitter\'s "legal demands" (2012-2021) include demands from lawyers representing private individuals, not only governments; no government-only total was published. X\'s 2024 reports call the data government requests but do not define it.',
    'The rates are not like-for-like. Twitter\'s compliance rate counts demands where it withheld or removed any reported content, including removals for breaking its own rules; X\'s 2024 action rate is cases actioned divided by total requests, with "actioned" undefined. Treat the change from about 51% (2021-H2) to 71-82% (2024) as indicative.',
    'No global compliance rate exists before 2019-H2; those cells are blank, not zero.',
    'X published no report for 2022-H2, 2023-H1 or 2023-H2, only a summary for 2022-H1, and none after 2024-H2 as of 2026-10-10. Missing periods are left empty, not interpolated.',
    '2024 breakdowns group countries into jurisdictions (the EU as one bloc, "All Others"), so the top-jurisdiction columns are not comparable with earlier country rankings. X did not report US removal demands for 2024.',
    'Totals are driven by a few countries: Japan alone accounted for 46,648 of 72,703 demands in 2024-H1 and 69,186 of 97,006 in 2024-H2. A rising worldwide total mostly reflects demands from Japan and Turkey.',
    'Batch demands can inflate accounts specified (for example, Indonesia named 102,363 accounts in two demands in 2021-H1).',
    'Rest of World\'s 2023 finding of 83% full compliance comes from Twitter\'s submissions to the Lumen database, a different and partial source, and is not part of this series.',
  ],
  sources,
  artifacts: [
    art('x-removal-demands', 'curatedTable', path.join(OUT, 'x-removal-demands.csv'), 'text/csv', 'One row per half-year, 2012-H1 to 2026-H1: removal demands, compliance or action rate, US figures, top jurisdictions, information requests, and source.'),
    art('notes', 'documentation', path.join(OUT, 'notes.md'), 'text/markdown', 'Sources, definitions, cross-checks, gaps and discrepancies for the series.'),
    art('build-js', 'transformationManifest', path.join(OUT, 'build-takedown-dataset.js'), 'text/javascript', 'Script that builds this dataset\'s content and saved analyses from the CSV.'),
  ],
  tables: [{
    id: 'x-removal-demands', title: 'Twitter/X legal demands to remove content by half-year', artifactId: 'x-removal-demands', unitOfObservation: 'One half-year reporting period',
    description: 'Worldwide legal demands to remove content, the share acted on, and related figures, as published by Twitter (2012-2022) and X (2024).',
    fields: [
      f('period', 'string', 'Half-year, e.g. 2021-H2 (July-December 2021).', { nullable: false }),
      f('period_start', 'date', 'First day of the period.', { nullable: false }),
      f('period_end', 'date', 'Last day of the period.', { nullable: false }),
      f('company', 'string', 'Twitter, Twitter/X, or X, as the reporting company.', { nullable: false }),
      f('report_status', 'string', 'published, summary only, or missing (no report published).', { nullable: false, valueType: 'derived', derivation: 'FactHub classification of what X published for the period.' }),
      f('removal_requests_total', 'integer', 'Legal demands to remove content received worldwide. Empty when not published.', { units: 'demands' }),
      f('court_orders', 'integer', 'Demands that were court orders (2012-2021 only).', { units: 'demands' }),
      f('other_legal_demands', 'integer', 'Other legal demands (2012-2021 only).', { units: 'demands' }),
      f('accounts_specified', 'integer', 'Accounts named in the demands, where reported.', { units: 'accounts' }),
      f('compliance_rate_pct', 'number', 'Share of demands acted on. Twitter compliance rate (2019-H2 to 2021-H2) or X action rate (2024); see compliance_metric.', { units: 'percent' }),
      f('compliance_metric', 'string', 'Definition of compliance_rate_pct for the row.'),
      f('us_requests', 'integer', 'Removal demands from the United States, where reported.', { units: 'demands' }),
      f('us_compliance_pct', 'number', 'Compliance rate for US demands, where reported.', { units: 'percent' }),
      f('top1_jurisdiction', 'string', 'Jurisdiction with the most demands.'),
      f('top1_requests', 'integer', 'Demands from top1_jurisdiction.', { units: 'demands' }),
      f('top1_compliance_pct', 'number', 'Compliance or action rate for top1_jurisdiction.', { units: 'percent' }),
      f('top2_jurisdiction', 'string', 'Second jurisdiction by demands.'),
      f('top2_requests', 'integer', 'Demands from top2_jurisdiction.', { units: 'demands' }),
      f('top2_compliance_pct', 'number', 'Compliance or action rate for top2_jurisdiction.', { units: 'percent' }),
      f('top3_jurisdiction', 'string', 'Third jurisdiction by demands.'),
      f('top3_requests', 'integer', 'Demands from top3_jurisdiction.', { units: 'demands' }),
      f('top3_compliance_pct', 'number', 'Compliance or action rate for top3_jurisdiction.', { units: 'percent' }),
      f('info_requests_total', 'integer', 'Government requests for user account information.', { units: 'requests' }),
      f('info_disclosure_pct', 'number', 'Share of information requests where some data was disclosed. For 2012-2021, a request-weighted average of published per-country rates (about +/-0.5 points).', { units: 'percent', valueType: 'derived', derivation: 'Request-weighted mean of per-country disclosure rates for 2012-2021; as published for 2024.' }),
      f('us_info_requests', 'integer', 'US government information requests, where reported.', { units: 'requests' }),
      f('us_info_disclosure_pct', 'number', 'Disclosure rate for US information requests, where reported.', { units: 'percent' }),
      f('source_type', 'string', 'primary or none.', { nullable: false }),
      f('source_url', 'string', 'Source for the row.'),
      f('notes', 'string', 'Row-level notes and definitional caveats.'),
    ],
  }],
  analyses: [
    {
      id: 'removal-demands-series', title: 'Legal demands to remove content, by half-year',
      question: 'How many legal demands to remove content did Twitter and X receive in each half-year they reported?',
      operation: 'Select periods with a published removal_requests_total.', scope: 'Worldwide, 2012-H1 to 2024-H2; unpublished periods omitted.',
      resultShape: 'timeSeries', structuredResult: { rows: volume },
      displayResult: volume.map(r => `${r.period}: ${r.removal_requests.toLocaleString('en-US')}`).join('; '),
      methodologicalNotes: ['2022-H1 is X\'s rounded summary figure ("approximately 53,000").'],
      limitations: ['Includes demands from private lawyers for 2012-2021.', 'Periods 2022-H2 to 2023-H2 and after 2024-H2 were not published.', 'Growth is concentrated in a few countries, especially Japan and Turkey.'],
      executedAt: '2026-10-10T23:30:00Z',
    },
    {
      id: 'compliance-rate-series', title: 'Share of removal demands acted on, by half-year',
      question: 'What share of legal demands to remove content did Twitter (2019-2021) and X (2024) act on?',
      operation: 'Select periods with a published global compliance or action rate.', scope: 'Worldwide; 2019-H2 to 2021-H2 (Twitter compliance rate) and 2024-H1 to 2024-H2 (X action rate).',
      resultShape: 'timeSeries', structuredResult: { rows: rate },
      displayResult: rate.map(r => `${r.period}: ${r.rate_pct}% (${r.metric})`).join('; '),
      methodologicalNotes: ['Twitter compliance rate: share of demands where some or all reported content was withheld or removed, including for breaking Twitter\'s rules. X action rate: cases actioned divided by total requests.'],
      limitations: ['The two metrics are defined differently; the before-and-after comparison is indicative, not exact.', 'No global rate was published before 2019-H2 or for 2022-H1 to 2023-H2.', 'A descriptive comparison of periods; it does not show why rates changed.'],
      executedAt: '2026-10-10T23:30:00Z',
    },
  ],
  events: [
    { id: 'musk-acquisition', eventType: 'ownership change', date: '2022-10-27', label: 'Elon Musk completes purchase of Twitter', description: 'Divides the pre- and post-acquisition periods; 2022-H2 straddles the change and was never reported.' },
  ],
  changelog: [{ date: '2026-10-10', summary: 'First release: 2012-H1 to 2026-H1, with data for 22 periods from Twitter and X transparency reports.' }],
};
fs.writeFileSync(path.join(OUT, 'content.json'), JSON.stringify(content));
console.log(JSON.stringify(rate), volume.length, content.artifacts.map(a => a.id + ' ' + a.byteLength + ' ' + a.contentDigest).join('\n'));

// Move the large members into a staged JSON artifact and reference them.
const parts = { sources: content.sources, tables: content.tables, analyses: content.analyses };
fs.writeFileSync(path.join(OUT, 'dataset-parts.json'), JSON.stringify(parts, null, 1) + '\n');
content.artifacts.push(art('dataset-parts', 'documentation', path.join(OUT, 'dataset-parts.json'), 'application/json', 'Sources, table definition and saved analyses for this dataset, as JSON.'));
for (const k of Object.keys(parts)) content[k] = { $artifact: 'dataset-parts', pointer: '/' + k };
fs.writeFileSync(path.join(OUT, 'content.min.json'), JSON.stringify(content));