// Downloads every raw input for the Washington apple-share dataset into <raw_dir>.
// Usage: node fetch_sources.js <raw_dir>
// Node.js 20 or later (global fetch), no packages.
//
// USDA NASS Quick Stats requires an API key for its /api endpoint, so this script uses the
// public Quick Stats web interface instead: it POSTs the same form fields the web page sends
// to /uuid/encode, then downloads the "Spreadsheet" (CSV) export for that query from
// /data/spreadsheet/<uuid>.csv. The query is recorded next to each CSV.
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const raw = process.argv[2];
if (!raw) { console.error('usage: node fetch_sources.js <raw_dir>'); process.exit(1); }
fs.mkdirSync(raw, { recursive: true });

const QS = 'https://quickstats.nass.usda.gov';
const QUERIES = {
  // Annual survey estimates (commercial orchards): production, utilized production, value, bearing acres.
  'quickstats_apples_state_national.csv': [
    ['source_desc', 'SURVEY'], ['sector_desc', 'CROPS'], ['group_desc', 'FRUIT & TREE NUTS'], ['commodity_desc', 'APPLES'],
    ['statisticcat_desc', 'PRODUCTION'], ['statisticcat_desc', 'AREA BEARING'],
    ['agg_level_desc', 'STATE'], ['agg_level_desc', 'NATIONAL'], ['freq_desc', 'ANNUAL'], ['reference_period_desc', 'YEAR']],
  // Census of Agriculture: apple acreage on all farms, every state.
  'quickstats_census_apples.csv': [
    ['source_desc', 'CENSUS'], ['sector_desc', 'CROPS'], ['group_desc', 'FRUIT & TREE NUTS'], ['commodity_desc', 'APPLES'],
    ['agg_level_desc', 'STATE'], ['agg_level_desc', 'NATIONAL'], ['freq_desc', 'ANNUAL'], ['reference_period_desc', 'YEAR']],
};

// NASS Noncitrus Fruits and Nuts annual summaries (plain-text editions) from USDA ESMIS, and the
// ERS Fruit and Tree Nuts Outlook used as an external cross-check.
const ESMIS = 'https://esmis.nal.usda.gov/sites/default/release-files/';
const FILES = {
  'NoncFruiNu-07-08-2003_Annual_Summary.txt': ESMIS + 'zs25x846c/7w62fb85j/hh63sz388/NoncFruiNu-07-08-2003_Annual_Summary.txt',
  'NoncFruiNu-07-06-2006_final.txt': ESMIS + 'zs25x846c/zk51vk31m/9019s512f/NoncFruiNu-07-06-2006_final.txt',
  'NoncFruiNu-07-08-2008_revision.txt': ESMIS + 'zs25x846c/h128nh386/ft848t234/NoncFruiNu-07-08-2008_revision.txt',
  'ncit0526.txt': ESMIS + '795891/ncit0526.txt',
  'fts-386.pdf': 'https://ers.usda.gov/media/29687/fts-386.pdf',
};

const sha = (b) => 'sha256:' + crypto.createHash('sha256').update(b).digest('hex');

(async () => {
  const manifest = { retrievedAt: new Date().toISOString().slice(0, 10), files: [] };
  for (const [file, fields] of Object.entries(QUERIES)) {
    const body = new URLSearchParams(fields).toString();
    const opts = { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' }, body };
    const count = await (await fetch(QS + '/row_counts', opts)).json();
    if (count.rowcnt > 50000) throw new Error(file + ': query too large for the web export (' + count.rowcnt + ' rows)');
    const uuid = (await (await fetch(QS + '/uuid/encode', opts)).json());
    const buf = Buffer.from(await (await fetch(`${QS}/data/spreadsheet/${uuid}.csv`)).arrayBuffer());
    fs.writeFileSync(path.join(raw, file), buf);
    manifest.files.push({ file, byteLength: buf.length, contentDigest: sha(buf), url: `${QS}/results/${uuid}`, download: `${QS}/data/spreadsheet/${uuid}.csv`, query: fields.map(([k, v]) => `${k}=${v}`).join('&'), rows: count.rowcnt });
    console.log(file, count.rowcnt, 'rows', uuid);
  }
  for (const [file, url] of Object.entries(FILES)) {
    const res = await fetch(url, { headers: { 'user-agent': 'Mozilla/5.0' } });
    if (!res.ok) throw new Error(url + ' ' + res.status);
    const buf = Buffer.from(await res.arrayBuffer());
    fs.writeFileSync(path.join(raw, file), buf);
    manifest.files.push({ file, byteLength: buf.length, contentDigest: sha(buf), url });
    console.log(file, buf.length);
  }
  fs.writeFileSync(path.join(raw, 'fetch-manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
})().catch((e) => { console.error(e); process.exit(1); });