// Builds FactHub dataset content for two I-645 datasets:
//   wa-capital-gains-collections  (DOR collections by tax year, payment concentration, fiscal-note forecasts)
//   wa-millionaire-income-irs     (IRS SOI $1M+ returns, WA revenue by source, illustrative back-cast, crash scenarios)
// Run from this folder: node build_datasets.js   (first run ../data/volatility.js)
// Writes artifacts to ./artifacts and dataset content to ./dataset-*.json
const fs = require("fs"), path = require("path"), crypto = require("crypto");
const here = __dirname, dataDir = path.join(here, "..", "data"), artDir = path.join(here, "artifacts");
fs.mkdirSync(artDir, { recursive: true });
const TODAY = "2026-10-22", EXEC = "2026-10-22T12:00:00Z";
const r1 = x => x == null ? null : Math.round(x * 10) / 10;
const r2 = x => x == null ? null : Math.round(x * 100) / 100;
const r3 = x => x == null ? null : Math.round(x * 1000) / 1000;
const pct = (a, b) => (a == null || b == null) ? null : r1(100 * (a / b - 1));

function readCsv(file) {
  const text = fs.readFileSync(file, "utf8").trim(), rows = []; let row = [], f = "", q = false;
  for (let i = 0; i < text.length; i++) {
    const c = text[i];
    if (q) { if (c === '"') { if (text[i + 1] === '"') { f += '"'; i++; } else q = false; } else f += c; }
    else if (c === '"') q = true; else if (c === ",") { row.push(f); f = ""; }
    else if (c === "\n" || c === "\r") { if (c === "\r" && text[i + 1] === "\n") i++; row.push(f); rows.push(row); row = []; f = ""; }
    else f += c;
  }
  row.push(f); rows.push(row);
  const h = rows[0];
  return rows.slice(1).map(v => Object.fromEntries(h.map((k, i) => [k, v[i] === "" ? null : (isNaN(+v[i]) ? v[i] : +v[i])])));
}
function toCsv(rows, cols) {
  const esc = v => v == null ? "" : (/[",\n]/.test(String(v)) ? `"${String(v).replace(/"/g, '""')}"` : String(v));
  return [cols.join(","), ...rows.map(r => cols.map(c => esc(r[c])).join(","))].join("\n") + "\n";
}
function artifact(id, role, fileName, mediaType, bytes, description) {
  fs.writeFileSync(path.join(artDir, fileName), bytes);
  const buf = fs.readFileSync(path.join(artDir, fileName));
  return { id, role, fileName, mediaType, byteLength: buf.length,
    contentDigest: "sha256:" + crypto.createHash("sha256").update(buf).digest("hex"), description, access: "public" };
}
const fld = (name, type, description, units, nullable, valueType, derivation, sourceIds) =>
  ({ name, type, description, units, nullable, valueType, derivation, sourceIds });
const gov = note => ({ publisherKind: "government", interest: "none", note });

// =====================================================================================
// 1. wa-capital-gains-collections
// =====================================================================================
const cgYears = [
  // DOR, "Capital Gains Tax - Tax Year 2025 Initial Collections", as of June 9, 2026
  { taxYear: 2022, netCollectionsMillion: 782.0, netPayments: 3625, fiscalNoteMillion: 442.0, fiscalNoteFY: "FY2023" },
  { taxYear: 2023, netCollectionsMillion: 394.7, netPayments: 3649, fiscalNoteMillion: 469.0, fiscalNoteFY: "FY2024" },
  { taxYear: 2024, netCollectionsMillion: 583.8, netPayments: 6727, fiscalNoteMillion: 505.0, fiscalNoteFY: "FY2025" },
  { taxYear: 2025, netCollectionsMillion: 1500, netPayments: 9050, fiscalNoteMillion: 519.0, fiscalNoteFY: "FY2026" },
];
cgYears.forEach((r, i) => {
  r.asOf = "2026-06-09";
  r.status = r.taxYear === 2025 ? "initial; DOR rounds to $1.5 billion; extended returns were due Oct 15, 2026" : "net payments to date";
  r.changeFromPriorYearPct = i ? pct(r.netCollectionsMillion, cgYears[i - 1].netCollectionsMillion) : null;
  r.surchargeNoteMillion = r.taxYear === 2025 ? 151.0 : null;
  r.actualToBaseEstimateRatio = r2(r.netCollectionsMillion / r.fiscalNoteMillion);
});
const cgConc = [
  // DOR Research & Fiscal Analysis, "Capital gains tax payments by decile", as of January 6, 2025
  { period: "Tax year 2022", periodType: "taxYear", netPaymentsCount: 3380, totalNetPayments: 780401874, top10Payments: 394300000, top10SharePctReported: 51, topDecileCount: 341, topDecilePayments: 668028512 },
  { period: "Tax year 2023", periodType: "taxYear", netPaymentsCount: 3401, totalNetPayments: 416681025, top10Payments: 129120000, top10SharePctReported: 31, topDecileCount: 344, topDecilePayments: 306180005 },
  { period: "Fiscal year 2023", periodType: "fiscalYear", netPaymentsCount: 3175, totalNetPayments: 847519106, top10Payments: 461630000, top10SharePctReported: 54, topDecileCount: 317, topDecilePayments: 729959936 },
  { period: "Fiscal year 2024", periodType: "fiscalYear", netPaymentsCount: 3706, totalNetPayments: 360593887, top10Payments: 91227741, top10SharePctReported: 25, topDecileCount: 366, topDecilePayments: 282363933 },
];
cgConc.forEach(r => { r.asOf = "2025-01-06"; r.topDecileSharePct = r1(100 * r.topDecilePayments / r.totalNetPayments); });

const cgCols1 = ["taxYear", "netCollectionsMillion", "netPayments", "asOf", "status", "changeFromPriorYearPct", "fiscalNoteFY", "fiscalNoteMillion", "surchargeNoteMillion", "actualToBaseEstimateRatio"];
const cgCols2 = ["period", "periodType", "asOf", "netPaymentsCount", "totalNetPayments", "top10Payments", "top10SharePctReported", "topDecileCount", "topDecilePayments", "topDecileSharePct"];
const cgArts = [
  artifact("cg-collections", "curatedTable", "wa_cg_collections_by_tax_year.csv", "text/csv", toCsv(cgYears, cgCols1),
    "One row per tax year, 2022-2025: DOR net capital gains tax payments and payment counts (as of June 9, 2026), the change from the prior year, and the original fiscal note's estimate for the fiscal year in which that tax year's payments arrive."),
  artifact("cg-concentration", "curatedTable", "wa_cg_payment_concentration.csv", "text/csv", toCsv(cgConc, cgCols2),
    "One row per period (tax years 2022 and 2023, fiscal years 2023 and 2024): DOR's total net payments, the top 10 payments and their reported share, and the top decile of payments, from DOR's decile table as of January 6, 2025."),
];
const cgSources = [
  { id: "dor-ty2025-collections", name: "Washington Department of Revenue, Capital Gains Tax - Tax Year 2025 Initial Collections",
    url: "https://dor.wa.gov/about/statistics-reports/capital-gains-tax-tax-year-2025-initial-collections", retrievedAt: "2026-10-21",
    role: "Net payments and number of net payments by tax year, 2022-2025, as of June 9, 2026; return and extension counts for tax year 2025.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; the page is linked, not hosted.",
    reliability: gov("The Department of Revenue administers and collects the tax and publishes these totals from its own payment records. Totals change as refunds and late payments post, so each figure is tied to its as-of date.") },
  { id: "dor-decile-table", name: "Washington Department of Revenue, Research & Fiscal Analysis, Capital gains tax payments by decile (as of January 6, 2025)",
    url: "https://dor.wa.gov/sites/default/files/2025-02/CapGainsPmtsDecile.pdf", retrievedAt: "2026-10-21",
    role: "Net payments by decile of payment size, and the total of the top 10 payments with DOR's stated share, for tax years 2022 and 2023 and fiscal years 2023 and 2024.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; the PDF is linked, not hosted.",
    reliability: gov("DOR's research office compiled the table from payment records. Its tax-year totals ($780.4M for 2022, $416.7M for 2023) are earlier counts than DOR's June 2026 figures and differ because of later refunds and payments.") },
  { id: "ofm-fn-5096", name: "OFM Fiscal Note Package 63363, SSB 5096 (2021), capital gains tax, as passed",
    url: "https://fnspublic.ofm.wa.gov/FNSPublicSearch/GetPDF?packageID=63363", retrievedAt: "2026-10-21",
    role: "The Department of Revenue's pre-enactment revenue estimate: FY2023 $442.0M, FY2024 $469.0M, FY2025 $505.0M, FY2026 $519.0M; assumptions on taxpayers (about 7,000) and compliance.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; the PDF is linked, not hosted.",
    reliability: gov("Official fiscal note prepared by DOR for the Legislature before the vote. It grows 2018 capital gains with CBO realization forecasts and assumes no behavioral response other than charitable giving.") },
  { id: "ofm-fn-5813", name: "OFM Fiscal Note Package 75010, ESSB 5813 (2025), 2.9% capital gains surcharge over $1 million",
    url: "https://fnspublic.ofm.wa.gov/FNSPublicSearch/GetPDF?packageID=75010", retrievedAt: "2026-10-21",
    role: "Estimate for the added 2.9% rate on gains over $1 million from tax year 2025: about $151.0M in FY2026 (Education Legacy Trust Account capital gains portion), about 900 taxpayers.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; the PDF is linked, not hosted.",
    reliability: gov("Official fiscal note prepared by DOR before the 2025 vote; the surcharge applied retroactively to January 1, 2025.") },
];
const cgAnalyses = [
  { id: "collections-by-tax-year", title: "Capital gains tax net collections by tax year, 2022-2025",
    question: "How much did Washington's capital gains tax collect for each tax year, and how much did it change from one year to the next?",
    operation: "netCollectionsMillion and changeFromPriorYearPct by tax year from the cg-collections table.",
    parameters: { table: "cg-collections" }, scope: "Washington capital gains excise tax (RCW 82.87), all payers, net of refunds as of June 9, 2026",
    resultShape: "timeSeries",
    structuredResult: cgYears.map(r => ({ taxYear: String(r.taxYear), netCollectionsMillion: r.netCollectionsMillion, changeFromPriorYearPct: r.changeFromPriorYearPct })),
    displayResult: "Net collections: $782.0M (tax year 2022), $394.7M (2023, down 49.5%), $583.8M (2024, up 47.9%) and about $1.5B (2025, up about 157%, including the new 2.9% rate on gains over $1 million).",
    methodologicalNotes: ["Change is computed from DOR's rounded totals; the 2025 total is DOR's rounded $1.5 billion, so its change is approximate."],
    limitations: ["Tax year 2025 is an initial count: extended returns were due October 15, 2026, and estimated payments may be revised.",
      "Tax year 2025 includes the 2.9% surcharge on gains over $1 million (ESSB 5813), so it is not directly comparable with earlier years.",
      "DOR's totals for each year change as refunds and amendments post; earlier releases gave different figures (for example $840.3M for 2022 in June 2025)."],
    executedAt: EXEC },
  { id: "payment-concentration", title: "Share of capital gains tax collected from the 10 largest payments and the top decile",
    question: "How much of Washington's capital gains tax revenue came from its largest payments?",
    operation: "top10SharePctReported as stated by DOR; topDecileSharePct = topDecilePayments / totalNetPayments, from the cg-concentration table.",
    parameters: { table: "cg-concentration" }, scope: "DOR decile table as of January 6, 2025",
    resultShape: "table",
    structuredResult: cgConc.map(r => ({ period: r.period, totalNetMillion: r1(r.totalNetPayments / 1e6), top10Million: r2(r.top10Payments / 1e6), top10SharePct: r.top10SharePctReported, topDecileCount: r.topDecileCount, topDecileSharePct: r.topDecileSharePct })),
    displayResult: "Tax year 2022: the 10 largest payments were $394.3M, 51% of $780.4M, and the top tenth of payments (341) supplied 85.6%. Tax year 2023: the 10 largest were $129.1M, 31% of $416.7M; the top tenth supplied 73.5%.",
    methodologicalNotes: ["DOR counts payments, not people: one taxpayer can make more than one payment, and a joint return is one payment.", "The top-10 share is DOR's stated, rounded percentage; the top-decile share is our calculation from the same table."],
    limitations: ["The decile table is an earlier count than DOR's June 2026 totals ($780.4M vs $782.0M for 2022; $416.7M vs $394.7M for 2023).", "DOR has not published a decile table for tax years 2024 or 2025."],
    executedAt: EXEC },
  { id: "top10-share-of-drop", title: "How much of the 2022-to-2023 drop came from the 10 largest payments",
    question: "What share of the fall in capital gains tax collections from tax year 2022 to 2023 is accounted for by the 10 largest payments?",
    operation: "(top10 TY2022 - top10 TY2023) / (total TY2022 - total TY2023), from the cg-concentration table.",
    parameters: { table: "cg-concentration", periods: ["Tax year 2022", "Tax year 2023"] }, scope: "DOR decile table as of January 6, 2025",
    resultShape: "scalar",
    structuredResult: { top10DropMillion: r2((394300000 - 129120000) / 1e6), totalDropMillion: r2((780401874 - 416681025) / 1e6), top10ShareOfDropPct: r1(100 * (394300000 - 129120000) / (780401874 - 416681025)) },
    displayResult: "The 10 largest payments fell by $265.2M, about 73% of the $363.7M total drop from tax year 2022 to 2023 (our calculation from DOR's decile table).",
    methodologicalNotes: ["The 10 largest payments in each year need not come from the same taxpayers."],
    limitations: ["Uses DOR's January 2025 decile counts, not the later June 2026 totals.", "Shows where the drop was concentrated, not why: market declines, timing of sales, and taxpayers leaving the state cannot be separated with this table."],
    executedAt: EXEC },
  { id: "actual-vs-fiscal-note", title: "Capital gains tax collections compared with the 2021 fiscal note",
    question: "How did each tax year's collections compare with the Department of Revenue's pre-enactment estimate?",
    operation: "netCollectionsMillion / fiscalNoteMillion, matching each tax year to the fiscal year in which its April payments arrive.",
    parameters: { table: "cg-collections" }, scope: "Tax years 2022-2025 against fiscal note package 63363 (FY2023-FY2026)",
    resultShape: "table",
    structuredResult: cgYears.map(r => ({ taxYear: String(r.taxYear), fiscalNoteFY: r.fiscalNoteFY, fiscalNoteMillion: r.fiscalNoteMillion, netCollectionsMillion: r.netCollectionsMillion, actualToBaseEstimateRatio: r.actualToBaseEstimateRatio })),
    displayResult: "Actual collections were 1.77 times the fiscal note's estimate for tax year 2022, 0.84 times for 2023 and 1.16 times for 2024. Tax year 2025 ($1.5B) was 2.9 times the base estimate ($519M), or about 2.2 times once the separately estimated 2.9% surcharge (about $151M) is added.",
    methodologicalNotes: ["Fiscal note figures are by state fiscal year (July-June); a tax year's payments arrive mostly in April of the following fiscal year."],
    limitations: ["Fiscal-year and tax-year totals are not identical: some payments and refunds fall in other fiscal years.", "Tax year 2025 includes a rate increase enacted after the 2021 estimate."],
    executedAt: EXEC },
];
const cgForecasts = cgYears.slice(0, 3).map(r => ({
  id: `fn-5096-${r.fiscalNoteFY.toLowerCase()}`, forecaster: "Washington Department of Revenue (fiscal note for SSB 5096)", madeOn: "2021-06-22",
  metric: `Capital gains tax revenue, ${r.fiscalNoteFY} (tax year ${r.taxYear} payments)`, unit: "million US dollars",
  forecastValue: r.fiscalNoteMillion, horizon: `${r.fiscalNoteFY} (tax year ${r.taxYear})`, sourceId: "ofm-fn-5096",
  outcome: { analysisId: "collections-by-tax-year", observedValue: r.netCollectionsMillion, observedPeriod: `Tax year ${r.taxYear}, net payments as of June 9, 2026` },
  note: "The fiscal note estimates fiscal-year cash; DOR reports net payments by tax year. Most of a tax year's payments arrive in April of the next fiscal year, so the two are close but not identical." }));
const cgDataset = {
  key: "wa-capital-gains-collections", status: "Final", rulesVersion: "2.23",
  title: "Washington capital gains tax: collections by tax year and concentration of payments, 2022-2025",
  description: "Net collections of Washington's capital gains excise tax for tax years 2022-2025, the share paid by the 10 largest payments and the top tenth of payments, and the Department of Revenue's pre-enactment estimates, compiled from DOR and OFM documents.",
  selectionRationale: "Washington's capital gains tax is the state's closest existing example of a tax on high earners' investment income. DOR's tax-year totals are the official collection figures, and its decile table is the only official breakdown of how concentrated the payments are. The 2021 fiscal note is the estimate lawmakers voted on.",
  topics: ["Taxes", "Washington state budget", "Washington I-645 (2026)"], geography: ["Washington"],
  temporalCoverage: "Tax years 2022-2025 (collections); tax years 2022-2023 and fiscal years 2023-2024 (concentration)",
  unitOfObservation: "Tax year (collections); tax year or fiscal year (concentration)", updateCadence: "When DOR publishes new collection totals",
  curator: "FactHub Research Team", reproducibilityStatus: "Reproducible",
  reproducibilityExplanation: "Every value is typed from a DOR or OFM document named in the field's sourceIds. The build script (build_datasets.js, Node.js) holds the inputs and recomputes every derived value and saved analysis.",
  limitations: [
    "DOR revises its totals as refunds and late payments post, so each figure is tied to an as-of date; the decile table (January 2025) and the collection totals (June 2026) are different snapshots.",
    "Tax year 2025 is an initial count and includes the 2.9% surcharge on gains over $1 million that took effect that year.",
    "DOR counts payments, not individual taxpayers.",
    "Fiscal-note estimates are by fiscal year and collections by tax year, so comparisons are approximate.",
  ],
  sources: cgSources, artifacts: cgArts,
  tables: [
    { id: "cg-collections", title: "Capital gains tax collections by tax year", description: "Net capital gains tax payments by tax year with the original fiscal note estimate for comparison.", artifactId: "cg-collections", unitOfObservation: "Tax year",
      fields: [
        fld("taxYear", "integer", "Tax year in which the gains were realized; payments are due the following April.", null, false, "observed", null, ["dor-ty2025-collections"]),
        fld("netCollectionsMillion", "number", "Net payments: payments received minus refunds issued, as of the asOf date.", "million US dollars (nominal)", false, "observed", "As reported by DOR; tax year 2025 reported as $1.5 billion (rounded).", ["dor-ty2025-collections"]),
        fld("netPayments", "integer", "Number of net payments.", "payments", false, "observed", null, ["dor-ty2025-collections"]),
        fld("asOf", "date", "Date of DOR's count.", null, false, "observed", null, ["dor-ty2025-collections"]),
        fld("status", "string", "Whether the total is final-to-date or an initial count.", null, false, "observed", null, ["dor-ty2025-collections"]),
        fld("changeFromPriorYearPct", "number", "Percent change in net collections from the prior tax year.", "percent", true, "derived", "100 x (this year / prior year - 1), one decimal; blank for 2022.", ["dor-ty2025-collections"]),
        fld("fiscalNoteFY", "string", "State fiscal year in the 2021 fiscal note matched to this tax year.", null, false, "derived", "Tax year + 1.", ["ofm-fn-5096"]),
        fld("fiscalNoteMillion", "number", "DOR's pre-enactment estimate for that fiscal year (base 7% tax only).", "million US dollars (nominal)", false, "estimated", null, ["ofm-fn-5096"]),
        fld("surchargeNoteMillion", "number", "Separate estimate for the 2.9% surcharge (tax year 2025 only).", "million US dollars (nominal)", true, "estimated", "Education Legacy Trust Account capital gains portion in FY2026 per the ESSB 5813 fiscal note.", ["ofm-fn-5813"]),
        fld("actualToBaseEstimateRatio", "number", "netCollectionsMillion / fiscalNoteMillion.", "ratio", false, "derived", "Rounded to 2 decimals.", ["dor-ty2025-collections", "ofm-fn-5096"]),
      ] },
    { id: "cg-concentration", title: "Concentration of capital gains tax payments", description: "Total net payments, the 10 largest payments, and the top decile of payments, by period.", artifactId: "cg-concentration", unitOfObservation: "Tax year or fiscal year",
      fields: [
        fld("period", "string", "Tax year or fiscal year covered.", null, false, "observed", null, ["dor-decile-table"]),
        fld("periodType", "string", "'taxYear' or 'fiscalYear'.", null, false, "observed", null, ["dor-decile-table"]),
        fld("asOf", "date", "Date DOR compiled the table.", null, false, "observed", null, ["dor-decile-table"]),
        fld("netPaymentsCount", "integer", "Number of net payments (All accounts).", "payments", false, "observed", null, ["dor-decile-table"]),
        fld("totalNetPayments", "integer", "Total net payments (All accounts).", "US dollars (nominal)", false, "observed", null, ["dor-decile-table"]),
        fld("top10Payments", "integer", "Total of the 10 largest payments, as stated in DOR's note 1 (rounded by DOR).", "US dollars (nominal)", false, "observed", null, ["dor-decile-table"]),
        fld("top10SharePctReported", "integer", "DOR's stated share of the total from the 10 largest payments.", "percent", false, "observed", null, ["dor-decile-table"]),
        fld("topDecileCount", "integer", "Number of payments in the top decile.", "payments", false, "observed", null, ["dor-decile-table"]),
        fld("topDecilePayments", "integer", "Net payments in the top decile.", "US dollars (nominal)", false, "observed", null, ["dor-decile-table"]),
        fld("topDecileSharePct", "number", "Top-decile share of total net payments.", "percent", false, "derived", "100 x topDecilePayments / totalNetPayments, one decimal.", ["dor-decile-table"]),
      ] },
  ],
  events: [
    { id: "surcharge-2025", eventType: "policy", date: "2025-01-01", label: "2.9% surcharge on gains over $1M", description: "ESSB 5813 added a 2.9% rate on Washington capital gains over $1 million, effective for tax year 2025 (payments due April 2026).", sourceIds: ["ofm-fn-5813"] },
  ],
  analyses: cgAnalyses, forecasts: cgForecasts,
  changelog: [{ date: TODAY, summary: "Initial version." }],
};

// =====================================================================================
// 2. wa-millionaire-income-irs
// =====================================================================================
const irs = readCsv(path.join(dataDir, "wa_irs_1m_plus_history.csv"));
const src = readCsv(path.join(dataDir, "wa_revenue_by_source_history.csv")).sort((a, b) => a.fiscalYear - b.fiscalYear);
const backcast = readCsv(path.join(dataDir, "wa_millionaires_tax_backcast.csv"));
const crash = readCsv(path.join(dataDir, "wa_crash_scenarios.csv"));
const vol = JSON.parse(fs.readFileSync(path.join(dataDir, "volatility_results.json"), "utf8")).volatility;
const revArt = src.map(r => ({ fiscalYear: r.fiscalYear, retailSalesMillion: r.retailSalesMillion, boMillion: r.boMillion, statePropertyMillion: r.statePropertyMillion, totalGfsMillion: r.totalGfsMillion, source: r.source }));
const bcCols = Object.keys(backcast[0]).filter(k => k !== "label");
const crCols = Object.keys(crash[0]).filter(k => k !== "label");
const irsArts = [
  artifact("irs-high-income", "curatedTable", "wa_irs_1m_plus_history.csv", "text/csv", fs.readFileSync(path.join(dataDir, "wa_irs_1m_plus_history.csv")),
    "IRS Statistics of Income: Washington (Historic Table 2) and U.S. (Table 1.4 and Historic Table 2) returns, AGI, wages, net capital gain, pass-through and business income for the top AGI class, 1997-2023. WA 2002-2009 rows are the $200,000-or-more class (no $1M class published)."),
  artifact("wa-revenue-by-source", "curatedTable", "wa_revenue_by_source_history.csv", "text/csv", toCsv(revArt, ["fiscalYear", "retailSalesMillion", "boMillion", "statePropertyMillion", "totalGfsMillion", "source"]),
    "Washington state general fund retail sales, B&O and state property tax collections (DOR Tax Statistics Table 3, GAAP) and total general fund-state collections (ERFC, cash), fiscal years 1998-2025, with the source of each row."),
  artifact("backcast", "derivedTable", "wa_millionaires_tax_backcast.csv", "text/csv", toCsv(backcast, bcCols),
    "FactHub illustration, not a forecast: what a 9.9% tax on income above $1M would have raised from the IRS $1M+ class each year (gross, static, nominal $1M threshold), with and without net capital gains."),
  artifact("crash-scenarios", "derivedTable", "wa_crash_scenarios.csv", "text/csv", toCsv(crash, crCols),
    "FactHub illustration, not a forecast: historical one- and two-year drops in the excess-over-$1M base applied to two official baselines for the millionaires tax."),
  artifact("volatility-script", "transformationManifest", "volatility.js", "text/javascript", fs.readFileSync(path.join(dataDir, "volatility.js")),
    "Node.js script that computes the back-cast, crash scenarios and volatility statistics from the two curated tables. Run: node volatility.js in a folder holding both CSVs."),
];
// Analysis: YoY series for the chart
const waTop = irs.filter(r => r.geography === "WA" && r.topClass === "1M+");
const agiByYear = Object.fromEntries(waTop.map(r => [r.taxYear, r.agi1mPlusBillion]));
const yoyRows = [];
for (let y = 1998; y <= 2025; y++) {
  const a = agiByYear[y], p = agiByYear[y - 1];
  yoyRows.push({ year: y, series: "WA $1M+ AGI (tax year)", changePct: (a != null && p != null) ? pct(a, p) : null });
}
for (let i = 1; i < src.length; i++) {
  const r = src[i], p = src[i - 1];
  yoyRows.push({ year: r.fiscalYear, series: "WA retail sales tax (fiscal year)", changePct: pct(r.retailSalesMillion, p.retailSalesMillion) });
  yoyRows.push({ year: r.fiscalYear, series: "WA general fund revenue (fiscal year)", changePct: r.fiscalYear === 2005 ? null : pct(r.totalGfsMillion, p.totalGfsMillion) });
}
yoyRows.sort((a, b) => a.year - b.year || a.series.localeCompare(b.series));
const statRow = (label, s, basis) => ({ series: label, basis, years: s.n, sdChangePct: s.sdYoYPct, worstChangePct: s.minYoYPct, bestChangePct: s.maxYoYPct, yearsDown: s.yearsDown });
const irsSources = [
  { id: "irs-soi-ht2", name: "IRS Statistics of Income, Historic Table 2: Individual income and tax data by state and size of adjusted gross income (Washington files, 1997-2023)",
    url: "https://www.irs.gov/statistics/soi-tax-stats-historic-table-2", retrievedAt: "2026-10-21",
    role: "Washington returns, AGI, wages, net capital gain, partnership/S-corporation and business income by AGI class; the $1M-or-more class for 1997-2001 and 2010-2023, $200,000-or-more for 2002-2009. Each row's file is in the sourceFile column.",
    licenseName: "Public domain (U.S. government work)", licenseUrl: null, redistribution: "Public domain; values restated, files linked.",
    reliability: gov("Compiled by the IRS from individual income tax returns. The 1997-2001 state files are unedited counts of returns processed in each calendar year; later files are tax-year tabulations with counts rounded to the nearest 10 from 2015.") },
  { id: "irs-soi-table14", name: "IRS Statistics of Income, Table 1.4: All returns, sources of income, by size of adjusted gross income (U.S., 1998-2023)",
    url: "https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income", retrievedAt: "2026-10-21",
    role: "U.S. returns, AGI, wages, net capital gain and pass-through income for AGI of $1M or more, used to show the 2002-2009 years for which no Washington $1M class exists.",
    licenseName: "Public domain (U.S. government work)", licenseUrl: null, redistribution: "Public domain; values restated, files linked.",
    reliability: gov("National sample-based estimates from the IRS individual complete report. Net capital gain and pass-through are FactHub sums of the gain and loss columns, matching the Historic Table 2 U.S. totals within about 1% in 2010-2023.") },
  { id: "dor-tax-statistics", name: "Washington Department of Revenue, Tax Statistics, Table 3: Washington State General Fund Revenues (fiscal years 1998-2024)",
    url: "https://dor.wa.gov/about/statistics-reports/tax-statistics", retrievedAt: "2026-10-21",
    role: "Retail sales, business and occupation and state property tax revenue to the general fund by fiscal year (GAAP basis, from the state's annual financial report). Each year's file is in the source column.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; files linked.",
    reliability: gov("DOR's annual statistical report restating general fund revenue from the state's audited financial statements. FY2002 and FY2025 come from ERFC cash tables because DOR's tables for those years were not available.") },
  { id: "erfc-revenue-review-2026-09", name: "Washington Economic and Revenue Forecast Council, Revenue Review, September 25, 2026",
    url: "https://erfc.wa.gov/sites/default/files/2026-09/rev20260925.pdf", retrievedAt: "2026-10-21",
    role: "Total general fund-state collections by fiscal year, FY2005-FY2025 (current definition, cash), and the September 2026 forecast of the individual income tax used as a baseline in the crash scenarios.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; the PDF is linked, not hosted.",
    reliability: gov("ERFC is the state's official, bipartisan revenue forecasting body. Its forecast is the basis for the budget; it published no separate risk analysis of the new income tax.") },
  { id: "erfc-june-2010", name: "Washington Economic and Revenue Forecast Council, Economic and Revenue Forecast, June 2010 (Table 3.15)",
    url: "https://web.archive.org/web/20100708001207/http://www.erfc.wa.gov/publications/documents/jun10pub.pdf", retrievedAt: "2026-10-21",
    role: "Total general fund-state collections (GF-S plus related fund, cash) for fiscal years 1998-2004.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; archived copy linked.",
    reliability: gov("ERFC historical table under the definition used in 2010; it differs from the current definition, so FY1998-2004 totals are not spliced to FY2005 onward for year-over-year change.") },
  { id: "ofm-fn-6346", name: "OFM Fiscal Note Package 77653, ESSB 6346 (2026), tax on income over $1 million",
    url: "https://fnspublic.ofm.wa.gov/FNSPublicSearch/GetPDF?packageID=77653", retrievedAt: "2026-10-21",
    role: "The Department of Revenue's estimate of the income tax (FY2030 $3.732B gross), used as the second baseline in the crash scenarios.",
    licenseName: "Public record of a Washington state agency", licenseUrl: null, redistribution: "Figures restated with attribution; the PDF is linked, not hosted.",
    reliability: gov("Official fiscal note for the enacted bill. It is a static estimate: it states that affected taxpayers do not reduce taxable income.") },
];
const irsAnalyses = [
  { id: "agi-vs-revenue-change", title: "Year-over-year change: Washington $1M+ AGI vs. state sales tax and general fund revenue",
    question: "How much more does the income of Washington's $1M+ filers swing from year to year than the state's main existing revenue?",
    operation: "Percent change from the prior year in agi1mPlusBillion (WA, $1M+ class), retailSalesMillion and totalGfsMillion.",
    parameters: { tables: ["irs-high-income", "wa-revenue-by-source"] }, scope: "Washington; tax years for AGI, state fiscal years for revenue, 1998-2025",
    resultShape: "timeSeries", structuredResult: yoyRows,
    displayResult: "Washington $1M+ AGI fell 44.6% in 2001 and 34.0% in 2022 and rose by more than half in 1998, 1999, 2012 and 2021. The general fund's worst year was -9.6% (FY2009) and retail sales tax's -12.5% (FY2009).",
    methodologicalNotes: ["AGI is by tax year (calendar year); revenue is by state fiscal year (July-June). Plotted on the same year axis for comparison of magnitude, not timing.", "No Washington $1M+ class exists for 2002-2009, so those changes (and 2010's) are missing, not zero.", "General fund revenue for FY2005 is missing because ERFC's series changes definition between FY2004 and FY2005."],
    limitations: ["1997-2001 IRS state files count returns processed in each calendar year, a different product from the 2010+ tax-year tables.", "The $1M threshold is nominal, so more returns cross it over time from inflation alone.", "Sales tax is DOR GAAP-basis revenue; total general fund is ERFC cash collections."],
    executedAt: EXEC },
  { id: "volatility-summary", title: "Volatility of the $1M+ income base compared with Washington's existing taxes",
    question: "How large are typical year-to-year swings in each series?",
    operation: "Sample standard deviation of year-over-year percent changes over all available years, plus the worst and best year and the number of declines (volatility.js).",
    parameters: { script: "volatility.js" }, scope: "Washington, all available years per series",
    resultShape: "table",
    structuredResult: [
      statRow("Illustrative 9.9% tax on WA income over $1M (gross)", vol.waIllustrativeTax, "IRS tax years 1998-2001, 2011-2023"),
      statRow("Illustrative tax, excluding net capital gains", vol.waIllustrativeTaxExNetCg, "IRS tax years 1998-2001, 2011-2023"),
      statRow("WA $1M+ AGI", vol.waAgi1mPlus, "IRS tax years 1998-2001, 2011-2023"),
      statRow("WA retail sales tax", vol.retailSales, "Fiscal years 1999-2025"),
      statRow("WA B&O tax", vol.bo, "Fiscal years 1999-2025"),
      statRow("WA total general fund revenue", vol.totalGfs, "Fiscal years 1999-2025 except 2005"),
    ],
    displayResult: `Typical yearly swing (standard deviation of changes): $1M+ AGI ${vol.waAgi1mPlus.sdYoYPct}%; illustrative tax ${vol.waIllustrativeTax.sdYoYPct}% (${vol.waIllustrativeTaxExNetCg.sdYoYPct}% without capital gains); retail sales tax ${vol.retailSales.sdYoYPct}%; B&O ${vol.bo.sdYoYPct}%; total general fund ${vol.totalGfs.sdYoYPct}%.`,
    methodologicalNotes: ["State property tax is omitted from the table because its swings come from definitional changes (FY2010 reclassification, FY2018-21 McCleary levy swap); the levy is limited to 1% growth and did not fall in the recession."],
    limitations: ["The $1M+ series has 17 annual changes with an eight-year gap that includes the 2007-2009 recession; the revenue series have 26-27.", "Revenue series include the effects of tax-law changes (for example the 2010 B&O increase), which add to their measured volatility."],
    executedAt: EXEC },
  { id: "illustrative-backcast", title: "Illustrative back-cast: a 9.9% tax on Washington income over $1M, 1997-2023",
    question: "What would a 9.9% tax on income above $1M have raised from Washington's $1M+ filers each year, and how much would it have swung?",
    operation: "illustrativeTaxBillion = 0.099 x (agi1mPlusBillion - returns1mPlus x $1M); illustrativeTaxExNetCgBillion removes net capital gain from the excess first.",
    parameters: { table: "backcast", rate: 0.099, threshold: 1000000 }, scope: "Washington $1M+ AGI class, tax years with a $1M class",
    resultShape: "timeSeries",
    structuredResult: backcast.filter(r => r.geography === "WA").map(r => ({ year: r.taxYear, illustrativeTaxBillion: r.illustrativeTaxBillion, illustrativeTaxExNetCgBillion: r.illustrativeTaxExNetCgBillion, changePct: r.illustrativeTaxYoYPct })),
    displayResult: "Illustration only: about $1.57B (2000) to $0.80B (2001, -49%); $7.04B (2021) to $4.40B (2022, -38%) to $3.69B (2023). Without net capital gains: -42% in 2001, -15% in 2022, +15% in 2023.",
    methodologicalNotes: ["ESSB 6346 removes long-term capital gains from its base and adds back only gains taxed by the capital gains tax, with a credit for that tax (secs. 302 and 205). The variant without net capital gains is therefore closer to the income tax alone; it also removes short-term gains, which stay in the base, so it understates the tax's base somewhat."],
    limitations: ["Not a forecast. Ignores compliance, the 4.71% enforcement adjustment, the shared $1M deduction for couples, inflation indexing and any behavioral response.", "Returns are not households; nominal $1M threshold in every year."],
    executedAt: EXEC },
  { id: "crash-scenarios", title: "Illustrative crash scenarios for the millionaires tax",
    question: "If the $1M+ income base fell as it did in 2001, 2008-09 or 2022, how much revenue would the tax lose a year against official baselines?",
    operation: "Annual hit = baseline x historical % drop in the excess-over-$1M base (gross and excluding net capital gains). Baselines: ERFC September 2026 FY2030-31 average ($4.72B) and fiscal note FY2030 ($3.732B).",
    parameters: { table: "crash-scenarios" }, scope: "Illustration for the first full years of the tax",
    resultShape: "table",
    structuredResult: crash.map(r => ({ scenario: r.scenario, baseline: r.baseline, baselineAnnualBillion: r.baselineAnnualBillion, grossDropPct: r.excessBaseDropPct, grossAnnualHitBillion: r.annualHitBillion, exNetCgDropPct: r.exNetCgBaseDropPct, exNetCgAnnualHitBillion: r.exNetCgAnnualHitBillion })),
    displayResult: "Illustration only. Income tax alone (excluding capital gains): about $0.5-0.7B a year in a 2022-style year, $1.6-2.0B in a 2001-style year, $0.8-1.0B a year over a 2007-09-style downturn. Counting all income over $1M, which is closer to total state revenue from top earners with the capital gains tax: $1.2-2.4B.",
    methodologicalNotes: ["2001 and 2022 scenarios use Washington data; 2000-02 and 2007-09 use U.S. $1M+ data because no Washington $1M class exists for those years."],
    limitations: ["Not a forecast. A future downturn may be larger or smaller and may hit wages, options or gains differently.", "Baselines are themselves estimates; ERFC raised its first-year estimate 30% between June and September 2026."],
    executedAt: EXEC },
];
const irsDataset = {
  key: "wa-millionaire-income-irs", status: "Final", rulesVersion: "2.23",
  title: "Washington's $1 million-plus earners: IRS income data 1997-2023, revenue volatility and an illustrative back-cast",
  description: "IRS Statistics of Income data on Washington returns with AGI of $1 million or more (with U.S. data for the years Washington's class is missing), Washington's general fund revenue by source, and FactHub's illustrative back-cast and crash scenarios for a 9.9% tax on income above $1 million.",
  selectionRationale: "ESSB 6346 taxes household income above $1 million. The IRS state tables are the only official, long-run record of how much income Washington's $1M+ filers report and what kind. DOR and ERFC revenue tables give the comparison with the taxes Washington already relies on.",
  topics: ["Taxes", "Washington state budget", "Washington I-645 (2026)"], geography: ["Washington", "United States"],
  temporalCoverage: "Tax years 1997-2023 (IRS; WA $1M class 1997-2001 and 2010-2023); fiscal years 1998-2025 (revenue)",
  unitOfObservation: "Tax year and geography (IRS); state fiscal year (revenue)", updateCadence: "Annual, when IRS releases a new year of state data",
  curator: "FactHub Research Team", reproducibilityStatus: "Reproducible",
  reproducibilityExplanation: "The curated tables hold values read from IRS, DOR and ERFC files named in each row. volatility.js recomputes the back-cast, crash scenarios and volatility statistics; build_datasets.js assembles the saved analyses.",
  limitations: [
    "IRS publishes no Washington $1M+ class for 2002-2009; those rows give the $200,000-or-more class, and U.S. $1M+ data stands in for the shape of the 2001-02 and 2007-09 downturns.",
    "The 1997-2001 Washington IRS files are a different product (returns processed in a calendar year, unedited) from the 2010+ tax-year tables.",
    "Returns are not households, and the $1M class is in nominal dollars.",
    "The back-cast and crash scenarios are FactHub illustrations, not forecasts, and ignore behavioral responses, compliance and the law's capital-gains credit except as noted.",
    "Revenue by source mixes DOR GAAP figures with ERFC cash figures for two years, and total general fund revenue changes ERFC definition at FY2005.",
  ],
  sources: irsSources, artifacts: irsArts,
  tables: [
    { id: "irs-high-income", title: "IRS returns in the top AGI class: Washington and U.S.", description: "Returns, AGI and income components for the top AGI class by tax year and geography.", artifactId: "irs-high-income", unitOfObservation: "Tax year x geography",
      fields: [
        fld("taxYear", "integer", "Tax year.", null, false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("geography", "string", "WA (Historic Table 2, Washington), US (Table 1.4) or US_HT2 (Historic Table 2, United States).", null, false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("topClass", "string", "Top AGI class in the file: '1M+' or '200k+' (WA 2002-2009).", null, false, "observed", null, ["irs-soi-ht2"]),
        fld("allReturns", "integer", "All returns in the geography.", "returns", true, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("allAgiBillion", "number", "AGI of all returns.", "billion US dollars (nominal)", true, "observed", "Source thousands / 1e6.", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("returns1mPlus", "integer", "Returns in the top class.", "returns", false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("agi1mPlusBillion", "number", "AGI of the top class.", "billion US dollars (nominal)", false, "observed", "Source thousands / 1e6.", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("wages1mPlusBillion", "number", "Salaries and wages of the top class.", "billion US dollars (nominal)", true, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("netCapitalGain1mPlusBillion", "number", "Net capital gain (less loss) in AGI of the top class.", "billion US dollars (nominal)", true, "observed", "US Table 1.4: Schedule D gain minus loss plus capital gain distributions.", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("partnershipSCorp1mPlusBillion", "number", "Partnership and S-corporation net income (less loss); from 2009.", "billion US dollars (nominal)", true, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("businessIncome1mPlusBillion", "number", "Business or profession net income (less loss).", "billion US dollars (nominal)", true, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("sourceFile", "string", "Exact IRS file for the row.", null, false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
      ] },
    { id: "wa-revenue-by-source", title: "Washington general fund revenue by source", description: "Retail sales, B&O and state property tax revenue and total general fund-state collections by fiscal year.", artifactId: "wa-revenue-by-source", unitOfObservation: "State fiscal year (July-June)",
      fields: [
        fld("fiscalYear", "integer", "State fiscal year ending June 30.", null, false, "observed", null, ["dor-tax-statistics"]),
        fld("retailSalesMillion", "number", "Retail sales tax to the general fund.", "million US dollars (nominal)", false, "observed", null, ["dor-tax-statistics", "erfc-revenue-review-2026-09"]),
        fld("boMillion", "number", "Business and occupation tax to the general fund.", "million US dollars (nominal)", false, "observed", null, ["dor-tax-statistics", "erfc-revenue-review-2026-09"]),
        fld("statePropertyMillion", "number", "State property tax levy to the general fund.", "million US dollars (nominal)", false, "observed", null, ["dor-tax-statistics", "erfc-revenue-review-2026-09"]),
        fld("totalGfsMillion", "number", "Total general fund-state collections (ERFC, cash).", "million US dollars (nominal)", false, "observed", "FY1998-2004 from ERFC June 2010 (GF-S plus related fund); FY2005-2025 from ERFC September 2026 (current definition).", ["erfc-june-2010", "erfc-revenue-review-2026-09"]),
        fld("source", "string", "Document and URL for each row's values.", null, false, "observed", null, ["dor-tax-statistics", "erfc-june-2010", "erfc-revenue-review-2026-09"]),
      ] },
    { id: "backcast", title: "Illustrative back-cast of a 9.9% tax on income over $1M", description: "FactHub illustration, not a forecast.", artifactId: "backcast", unitOfObservation: "Tax year x geography",
      fields: [
        fld("taxYear", "integer", "Tax year.", null, false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("geography", "string", "WA or US.", null, false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("returns1mPlus", "integer", "Returns with AGI of $1M or more.", "returns", false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("agi1mPlusBillion", "number", "AGI of $1M+ returns.", "billion US dollars (nominal)", false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("netCapitalGain1mPlusBillion", "number", "Net capital gain of $1M+ returns.", "billion US dollars (nominal)", true, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("excessOver1mBillion", "number", "AGI above $1M per return, summed.", "billion US dollars (nominal)", false, "derived", "agi1mPlusBillion - returns1mPlus x $1M.", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("illustrativeTaxBillion", "number", "9.9% of the excess.", "billion US dollars (nominal)", false, "estimated", "0.099 x excessOver1mBillion; gross and static.", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("illustrativeTaxExNetCgBillion", "number", "9.9% of the excess after removing net capital gain.", "billion US dollars (nominal)", true, "estimated", "0.099 x max(0, excessOver1mBillion - netCapitalGain1mPlusBillion).", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("agiYoYPct", "number", "Change in AGI from the prior year.", "percent", true, "derived", "Blank when the prior year has no $1M class.", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("netCgYoYPct", "number", "Change in net capital gain from the prior year.", "percent", true, "derived", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("illustrativeTaxYoYPct", "number", "Change in the illustrative tax from the prior year.", "percent", true, "derived", null, ["irs-soi-ht2", "irs-soi-table14"]),
      ] },
    { id: "crash-scenarios", title: "Illustrative crash scenarios", description: "FactHub illustration, not a forecast.", artifactId: "crash-scenarios", unitOfObservation: "Scenario x baseline",
      fields: [
        fld("scenario", "string", "Historical episode and years used.", null, false, "observed", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("baseline", "string", "Official revenue estimate the drop is applied to.", null, false, "observed", null, ["erfc-revenue-review-2026-09", "ofm-fn-6346"]),
        fld("excessBaseDropPct", "number", "Drop in the excess-over-$1M base over the episode.", "percent", false, "derived", "1 - illustrativeTax(end) / illustrativeTax(start).", ["irs-soi-ht2", "irs-soi-table14"]),
        fld("baselineAnnualBillion", "number", "Baseline annual revenue.", "billion US dollars (nominal)", false, "estimated", "ERFC Sept 2026 2029-31 biennium $9.435B / 2; fiscal note FY2030 $3.732B.", ["erfc-revenue-review-2026-09", "ofm-fn-6346"]),
        fld("revenueAfterBillion", "number", "Baseline x (1 - drop).", "billion US dollars (nominal)", false, "estimated", null, ["erfc-revenue-review-2026-09", "ofm-fn-6346"]),
        fld("annualHitBillion", "number", "Baseline x drop (gross base).", "billion US dollars (nominal)", false, "estimated", null, ["erfc-revenue-review-2026-09", "ofm-fn-6346"]),
        fld("exNetCgBaseDropPct", "number", "Drop in the base excluding net capital gains.", "percent", false, "derived", null, ["irs-soi-ht2", "irs-soi-table14"]),
        fld("exNetCgAnnualHitBillion", "number", "Baseline x drop excluding net capital gains.", "billion US dollars (nominal)", false, "estimated", null, ["erfc-revenue-review-2026-09", "ofm-fn-6346"]),
      ] },
  ],
  analyses: irsAnalyses,
  changelog: [{ date: TODAY, summary: "Initial version." }],
};

const buildArt = artifact("build-script", "transformationManifest", "build_datasets.js", "text/javascript", fs.readFileSync(__filename),
  "Node.js script holding every typed input value and computing the CSV tables and saved analyses for this dataset. Run: node build_datasets.js.");
cgArts.push(buildArt); irsArts.push(buildArt);
fs.writeFileSync(path.join(here, "dataset-wa-capital-gains-collections.json"), JSON.stringify(cgDataset, null, 1));
fs.writeFileSync(path.join(here, "dataset-wa-millionaire-income-irs.json"), JSON.stringify(irsDataset, null, 1));
for (const a of [...cgArts, ...irsArts]) console.log(a.id, a.byteLength, a.contentDigest);
console.log(JSON.stringify(cgAnalyses[2].structuredResult), JSON.stringify(irsAnalyses[1].structuredResult.map(r => [r.series, r.sdChangePct])));

// ---- Compact request form: large arrays move to a staged JSON artifact and are referenced with $artifact ----
// (Inline requests of ~25 KB timed out through the connector; the server expands references to identical content.)
function refForm(ds, partsId, fileName) {
  const keys = ["limitations", "sources", "tables", "events", "analyses", "forecasts"].filter(k => ds[k]);
  const parts = Object.fromEntries(keys.map(k => [k, ds[k]]));
  const art = artifact(partsId, "documentation", fileName, "application/json", JSON.stringify(parts, null, 1),
    "The dataset's limitations, sources, tables, events, saved analyses and forecasts as JSON, as stored in this version.");
  const out = { ...ds, artifacts: [...ds.artifacts, art] };
  for (const k of keys) out[k] = { $artifact: partsId, pointer: "/" + k };
  return { out, art };
}
for (const [ds, f] of [[cgDataset, "cg"], [irsDataset, "irs"]]) {
  const { out, art } = refForm(ds, "dataset-parts", `dataset-parts-${f}.json`);
  fs.writeFileSync(path.join(here, `request-${ds.key}.json`), JSON.stringify(out));
  console.log("parts", ds.key, art.byteLength, art.contentDigest, "request bytes", JSON.stringify(out).length);
}