// Build CoC panel (HUD PIT + ACS 1-year) and fit cross-sectional OLS of homelessness rate on rent.
// Usage: node build_and_analyze.js <dataDir>
// Inputs:  <dataDir>/raw/pit2025_csv/<year>.csv  (sheets of HUD 2007-2025-PIT-Counts-by-CoC.xlsb, exported via Excel)
//          <dataDir>/raw/acs/acs1_<year>_<table>_<county|place>.json (data.census.gov table API)
//          <dataDir>/coc_geography_map.csv
// Outputs: <dataDir>/coc_homelessness_rent_panel.csv, <dataDir>/regression_results.json
const fs = require("fs");
const path = require("path");
const DATA = path.resolve(process.argv[2] || path.join(__dirname, ".."));
const PIT_DIR = path.join(DATA, "raw", "pit2025_csv");
const ACS_DIR = path.join(DATA, "raw", "acs");
const PIT_YEARS = [2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025];
const ACS_YEARS = [2015, 2016, 2017, 2018, 2019, 2021, 2022, 2023, 2024];

function parseCSV(t) {
  t = t.replace(/^/, "");
  const rows = []; let r = [], f = "", q = false;
  for (let i = 0; i < t.length; i++) {
    const c = t[i];
    if (q) { if (c === '"') { if (t[i + 1] === '"') { f += '"'; i++; } else q = false; } else f += c; }
    else if (c === '"') q = true;
    else if (c === ",") { r.push(f); f = ""; }
    else if (c === "\n") { r.push(f.replace(/\r$/, "")); rows.push(r); r = []; f = ""; }
    else f += c;
  }
  if (f !== "" || r.length) { r.push(f); rows.push(r); }
  return rows;
}
const num = (s) => { if (s == null) return null; s = String(s).replace(/,/g, "").trim(); if (s === "" || s === ".") return null; const v = Number(s); return isFinite(v) ? v : null; };

// ---- mapping
const mapRows = parseCSV(fs.readFileSync(path.join(DATA, "coc_geography_map.csv"), "utf8")).filter(r => r.length > 1);
const mh = mapRows[0];
const cocs = mapRows.slice(1).map(r => {
  const o = Object.fromEntries(mh.map((h, i) => [h, r[i] || ""]));
  const g = (s) => s ? s.split(";").map(x => { const [lvl, code] = x.split(":"); return { lvl, code }; }) : [];
  return { code: o.coc_code, aliases: o.code_aliases ? o.code_aliases.split(";") : [], short: o.short_name, add: g(o.add_geos), sub: g(o.subtract_geos), note: o.note };
});

// ---- PIT
const pit = {}; // pit[year][code] = {...}
for (const y of PIT_YEARS) {
  const rows = parseCSV(fs.readFileSync(path.join(PIT_DIR, `${y}.csv`), "utf8"));
  const h = rows[0];
  const ix = (n) => { const i = h.indexOf(n); if (i < 0) throw new Error(`${y}: no column ${n}`); return i; };
  const iCat = h.indexOf("CoC Category");
  const iT = ix("Overall Homeless"), iS = ix("Sheltered Total Homeless"), iU = ix("Unsheltered Homeless"), iC = ix("Count Types");
  pit[y] = {};
  for (const r of rows.slice(1)) {
    if (!/^[A-Z]{2}-\d/.test(r[0] || "")) continue;
    pit[y][r[0]] = { category: iCat >= 0 ? r[iCat] : "", name: r[1], countType: r[iC], total: num(r[iT]), sheltered: num(r[iS]), unsheltered: num(r[iU]) };
  }
}

// ---- ACS
const acs = {}; // acs[year][lvl][geoid] = {pop, rent, renterOcc, forRent, rentedNotOcc}
for (const y of ACS_YEARS) {
  acs[y] = { county: {}, place: {} };
  for (const lvl of ["county", "place"]) {
    const prefix = lvl === "county" ? "0500000US" : "1600000US";
    const get = (t) => JSON.parse(fs.readFileSync(path.join(ACS_DIR, `acs1_${y}_${t}_${lvl}.json`), "utf8")).response.data;
    const tables = { B01003: ["B01003_001E", "pop"], B25064: ["B25064_001E", "rent"], B25003: ["B25003_003E", "renterOcc"] };
    const put = (id, k, v) => { const code = id.replace(prefix, ""); (acs[y][lvl][code] ||= {})[k] = v; };
    for (const [t, [col, key]] of Object.entries(tables)) {
      const d = get(t); const ci = d[0].indexOf(col), gi = d[0].indexOf("GEO_ID"), ni = d[0].indexOf("NAME");
      for (const r of d.slice(1)) { put(r[gi], key, num(r[ci])); put(r[gi], "name", r[ni]); }
    }
    const d = get("B25004"); const gi = d[0].indexOf("GEO_ID");
    const c2 = d[0].indexOf("B25004_002E"), c3 = d[0].indexOf("B25004_003E");
    for (const r of d.slice(1)) { put(r[gi], "forRent", num(r[c2])); put(r[gi], "rentedNotOcc", num(r[c3])); }
  }
}

function acsForCoc(coc, y) {
  if (!acs[y]) return null;
  let pop = 0, rentW = 0, renters = 0, forRent = 0, rno = 0, vacMissing = false, missing = [];
  for (const g of coc.add) {
    const v = acs[y][g.lvl][g.code];
    if (!v || v.pop == null || v.rent == null) { missing.push(`${g.lvl}:${g.code}`); continue; }
    pop += v.pop; rentW += v.rent * v.renterOcc; renters += v.renterOcc; if (v.forRent == null || v.rentedNotOcc == null) vacMissing = true; else { forRent += v.forRent; rno += v.rentedNotOcc; }
  }
  for (const g of coc.sub) {
    const v = acs[y][g.lvl][g.code];
    if (!v || v.pop == null) { missing.push(`-${g.lvl}:${g.code}`); continue; }
    pop -= v.pop; // rent/vacancy keep whole-county values
  }
  if (missing.length) return { missing };
  return { pop, rent: rentW / renters, vacancy: vacMissing ? null : 100 * forRent / (renters + forRent + rno) };
}

// ---- OLS (small dense solver)
function ols(X, y) {
  const n = X.length, k = X[0].length;
  const XtX = Array.from({ length: k }, () => Array(k).fill(0)), Xty = Array(k).fill(0);
  for (let i = 0; i < n; i++) for (let a = 0; a < k; a++) { Xty[a] += X[i][a] * y[i]; for (let b = 0; b < k; b++) XtX[a][b] += X[i][a] * X[i][b]; }
  // invert XtX (Gauss-Jordan)
  const A = XtX.map((r, i) => [...r, ...Array.from({ length: k }, (_, j) => (i === j ? 1 : 0))]);
  for (let c = 0; c < k; c++) {
    let p = c; for (let r = c + 1; r < k; r++) if (Math.abs(A[r][c]) > Math.abs(A[p][c])) p = r;
    [A[c], A[p]] = [A[p], A[c]];
    const d = A[c][c]; for (let j = 0; j < 2 * k; j++) A[c][j] /= d;
    for (let r = 0; r < k; r++) if (r !== c) { const f = A[r][c]; for (let j = 0; j < 2 * k; j++) A[r][j] -= f * A[c][j]; }
  }
  const inv = A.map(r => r.slice(k));
  const beta = inv.map(r => r.reduce((s, v, j) => s + v * Xty[j], 0));
  const yhat = X.map(r => r.reduce((s, v, j) => s + v * beta[j], 0));
  const ybar = y.reduce((s, v) => s + v, 0) / n;
  const sse = y.reduce((s, v, i) => s + (v - yhat[i]) ** 2, 0), sst = y.reduce((s, v) => s + (v - ybar) ** 2, 0);
  const s2 = sse / (n - k);
  const se = inv.map((r, i) => Math.sqrt(s2 * r[i]));
  return { beta, se, r2: 1 - sse / sst, n, rmse: Math.sqrt(s2), predict: (x) => x.reduce((s, v, j) => s + v * beta[j], 0) };
}

// HUD's 2025 CoC Category (e.g. "Major City CoC") applied to all years
const category2025 = (coc) => { const c = [coc.code, ...coc.aliases].find(c => pit[2025][c]); return c ? pit[2025][c].category : ""; };
// ---- build panel
const panel = [];
const problems = [];
for (const coc of cocs) {
  for (const y of PIT_YEARS) {
    const code = [coc.code, ...coc.aliases].find(c => pit[y][c]);
    if (!code) { problems.push(`${coc.code} missing in PIT ${y}`); continue; }
    const p = pit[y][code];
    const acsYear = acs[y] ? y : (y === 2025 ? 2024 : null); // 2025 ACS 1-yr not yet released; 2020 ACS 1-yr not standard
    const a = acsYear ? acsForCoc(coc, acsYear) : null;
    if (a && a.missing) { problems.push(`${coc.code} ${y}: ACS missing ${a.missing.join(",")}`); }
    const ok = a && !a.missing;
    const shelteredOnly = /sheltered-only/i.test(p.countType);
    panel.push({
      coc_code: coc.code, coc_name: p.name, short_name: coc.short, hud_category_2025: category2025(coc), year: y,
      count_type: p.countType.replace(/\*$/, ""), sheltered_only_count: shelteredOnly ? 1 : 0,
      total: p.total, sheltered: p.sheltered, unsheltered: p.unsheltered,
      acs_year: ok ? acsYear : "",
      population: ok ? a.pop : null,
      rate_per_10k: ok ? (1e4 * p.total) / a.pop : null,
      sheltered_rate: ok ? (1e4 * p.sheltered) / a.pop : null,
      unsheltered_rate: ok ? (1e4 * p.unsheltered) / a.pop : null,
      median_rent: ok ? a.rent : null, vacancy: ok ? a.vacancy : null,
    });
  }
}

// ---- regressions per year
const results = {};
const models = {
  total_rent: { y: "rate_per_10k", x: ["median_rent"] },
  total_rent_vac: { y: "rate_per_10k", x: ["median_rent", "vacancy"] },
  unsh_rent: { y: "unsheltered_rate", x: ["median_rent"] },
  unsh_rent_vac: { y: "unsheltered_rate", x: ["median_rent", "vacancy"] },
  sh_rent: { y: "sheltered_rate", x: ["median_rent"] },
  log_total_rent: { y: "rate_per_10k", x: ["median_rent"], log: true },
};
for (const y of PIT_YEARS) {
  results[y] = {};
  for (const sample of ["full_counts", "all_rows", "full_counts_excl_NYC", "full_counts_major_city"]) {
    const rowsAll = panel.filter(r => r.year === y && r.median_rent != null &&
      (sample === "all_rows" || !r.sheltered_only_count) && (sample !== "full_counts_excl_NYC" || r.coc_code !== "NY-600") && (sample !== "full_counts_major_city" || r.hud_category_2025 === "Major City CoC"));
    const yearRows = panel.filter(r => r.year === y && r.median_rent != null);
    if (rowsAll.length < 10) continue;
    results[y][sample] = {};
    for (const [m, spec] of Object.entries(models)) {
      const rows = rowsAll.filter(r => spec.x.every(v => r[v] != null));
      const X = rows.map(r => [1, ...spec.x.map(v => (v === "median_rent" ? r[v] / 100 : r[v]))]);
      const Y = rows.map(r => (spec.log ? Math.log(r[spec.y]) : r[spec.y]));
      const f = ols(X, Y);
      const resid = rows.map((r, i) => {
        const pr = f.predict(X[i]);
        const predicted = spec.log ? Math.exp(pr) : pr;
        return { coc_code: r.coc_code, short_name: r.short_name, actual: r[spec.y], predicted, residual: r[spec.y] - predicted, ratio: r[spec.y] / predicted };
      }).sort((a, b) => b.residual - a.residual);
      // rank of WA-500 among residuals
      const kingIdx = resid.findIndex(r => r.coc_code === "WA-500");
      results[y][sample][m] = {
        n: f.n, r2: f.r2, rmse: f.rmse,
        coef: Object.fromEntries(["intercept", ...spec.x.map(v => (v === "median_rent" ? "median_rent_per_$100" : v))].map((k, i) => [k, { b: f.beta[i], se: f.se[i] }])),
        king: kingIdx >= 0 ? { ...resid[kingIdx], rank_of_n: kingIdx + 1 } : null,
        residuals: resid,
      };
      // write predictions back to panel for headline columns (sample "all")
      // headline columns: model fit on full-count rows, applied to every row of that year (out-of-sample for sheltered-only rows)
      if (sample === "full_counts") for (const r of yearRows.filter(r => spec.x.every(v => r[v] != null))) {
        const xr = [1, ...spec.x.map(v => (v === "median_rent" ? r[v] / 100 : r[v]))];
        const p = f.predict(xr); const predicted = spec.log ? Math.exp(p) : p;
        const rr = { predicted, residual: r[spec.y] - predicted };
        if (m === "total_rent") { r.predicted_rate = rr.predicted; r.residual = rr.residual; }
        if (m === "total_rent_vac") { r.predicted_rate_rent_vac = rr.predicted; r.residual_rent_vac = rr.residual; }
        if (m === "unsh_rent") { r.predicted_unsheltered_rate = rr.predicted; r.unsheltered_residual = rr.residual; }
      }
    }
  }
}

// ---- write outputs
const cols = ["coc_code", "coc_name", "short_name", "hud_category_2025", "year", "count_type", "sheltered_only_count", "total", "sheltered", "unsheltered",
  "acs_year", "population", "rate_per_10k", "sheltered_rate", "unsheltered_rate", "median_rent", "vacancy",
  "predicted_rate", "residual", "predicted_rate_rent_vac", "residual_rent_vac", "predicted_unsheltered_rate", "unsheltered_residual"];
const fmt = (v, c) => {
  if (v == null || v === "") return "";
  if (typeof v === "number") return ["median_rent"].includes(c) ? v.toFixed(0) : Number.isInteger(v) ? String(v) : v.toFixed(2);
  return /[",]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
};
panel.sort((a, b) => a.coc_code.localeCompare(b.coc_code) || a.year - b.year);
fs.writeFileSync(path.join(DATA, "coc_homelessness_rent_panel.csv"), [cols.join(","), ...panel.map(r => cols.map(c => fmt(r[c], c)).join(","))].join("\n") + "\n");
fs.writeFileSync(path.join(DATA, "regression_results.json"), JSON.stringify(results, null, 1));
console.log("rows", panel.length, "CoCs", cocs.length);
if (problems.length) console.log("PROBLEMS:\n" + problems.join("\n"));