"""Build the national-thanksgiving dataset tables from research-proclamations.md and states-1939.csv. Outputs CSVs + computed analyses JSON in this folder. Run: python -I build.py """ import csv, json, re, sys, datetime, hashlib, os root = sys.argv[1] here = os.path.dirname(os.path.abspath(__file__)) md = open(os.path.join(root, 'research-proclamations.md'), encoding='utf-8').read() APP = 'https://www.presidency.ucsb.edu/documents/' rows = [] def add(year, body, by, signed, obs, kind, url, quote, note=''): rows.append(dict(year=year, proclaimed_by=by, body=body, signed=signed, observance=obs, kind=kind, source_url=url, date_phrase=quote, note=note)) # Table A: Congress jcc = 'https://archive.org/details/journals-of-the-continental-congress-1774-1789-vol-' A = [ (1777, '1777-11-01', '1777-12-18', '9-oct-3-dec-31-1777', 'to set apart Thursday, the eighteenth day of December next, for solemn thanksgiving and praise', 'Saratoga victory'), (1778, '1778-11-17', '1778-12-30', '12-sept-2-dec-31-1778', 'to appoint Wednesday, the 30th day of December next, to be observed as a day of public thanksgiving and praise', 'A Wednesday'), (1779, '1779-10-20', '1779-12-09', '15-sept-2-dec-31-1779', 'to appoint Thursday, the 9th of December next, to be a day of public and solemn thanksgiving to Almighty God', ''), (1780, '1780-10-18', '1780-12-07', '18-sept-7-dec-29-1780', 'to set apart Thursday, the seventh day [of December next, to be observed as a day of publie thanksgiving and prayer', 'Quote is OCR text as printed'), (1781, '1781-10-26', '1781-12-13', '21-july-23-dec-31-1781', 'to set apart the 13th day of December next, to be religiously observed as a Day of Thanksgiving and Prayer', 'Yorktown victory; date fixed Sep 13, text adopted Oct 26'), (1782, '1782-10-11', '1782-11-28', '23-aug-12-dec-31-1782', 'the 28 day of November next, as a day of solemn thanksgiving to God for all his mercies', ''), (1783, '1783-10-18', '1783-12-11', '25-sept-1-dec-31-1783', 'to set apart the second Thursday in December next, as a day of public thanksgiving', 'Peace and independence'), (1784, '1784-08-03', '1784-10-19', '27-may-11-dec-24-1784', 'to set apart Tuesday, the 19th day of October next, as a day of public prayer and thanksgiving', 'Committee of the States; a Tuesday, Yorktown anniversary'), ] for y, s, o, vol, q, n in A: add(y, 'Congress', 'Continental/Confederation Congress', s, o, 'Congress', jcc + vol, q, n) B1 = [ (1789, 'George Washington', '1789-10-03', '1789-11-26', 'President, one-time', APP + 'proclamation-day-national-thanksgiving', 'I do recommend and assign Thursday, the 26th day of November next', 'Requested by Congress'), (1795, 'George Washington', '1795-01-01', '1795-02-19', 'President, one-time', APP + 'proclamation-6-day-public-thanksgiving', 'to set apart and observe Thursday, the 19th day of February next, as a day of public thanksgiving and prayer', ''), (1815, 'James Madison', '1815-03-04', '1815-04-13', 'President, one-time', APP + 'proclamation-20-recommending-day-public-thanksgiving-for-peace', 'I now recommend that the second Thursday in April next be set apart', 'Peace with Britain'), (1862, 'Abraham Lincoln', '1862-04-10', None, 'President, one-time', APP + 'proclamation-88-day-public-thanksgiving-for-victories-during-the-civil-war', 'at their next weekly assemblages in their accustomed places of public worship which shall occur after notice of this proclamation', 'No fixed date: next weekly church service'), (1863, 'Abraham Lincoln', '1863-07-15', '1863-08-06', 'President, one-time', APP + 'proclamation-103-day-thanksgiving-praise-and-prayer-august-6-1863', 'I do set apart Thursday, the 6th day of August next, to be observed as a day for national thanksgiving, praise, and prayer', 'Gettysburg and Vicksburg'), (1864, 'Abraham Lincoln', '1864-09-03', '1864-09-11', 'President, one-time', APP + 'proclamation-thanksgiving', 'It is therefore requested that on next Sunday, in all places of public worship in the United States, thanksgiving be offered', 'Mobile Bay and Atlanta'), (1889, 'Benjamin Harrison', '1889-04-04', '1889-04-30', 'President, one-time', APP + 'proclamation-289-centennial-george-washingtons-inauguration', 'do recommend that on Tuesday, April 30, at the hour of 9 o\'clock in the morning', 'An hour of prayer and thanksgiving for the centennial of Washington\'s inauguration'), (1898, 'William McKinley', '1898-07-06', None, 'President, one-time', APP + 'address-the-people-for-thanksgiving-and-prayer', 'upon next assembling for divine worship in their respective places of meeting, to offer thanksgiving to Almighty God', 'War victories; no fixed date'), ] for y, by, s, o, k, u, q, n in B1: add(y, 'President', by, s, o, k, u, q, n) # Table B2 parse months = {m: i for i, m in enumerate(['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'], 1)} def pdate(txt, year): m = re.search(r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w* (\d+), (\d{4})', txt) if not m: return None return datetime.date(int(m.group(3)), months[m.group(1)], int(m.group(2))).isoformat() sec = md.split('### B2.')[1].split('### Anomalies')[0] for line in sec.splitlines(): if not re.match(r'\| 1[89]\d\d \|', line): continue c = [x.strip() for x in line.strip('|').split('|')] year = int(c[0]); by = c[1] signed = pdate(c[2], year) if 'UNCONFIRMED' not in c[2] else None obs = pdate(c[3], year) url = re.search(r'\((https[^)]+)\)', c[5]).group(1) q = c[6].strip('"') q = q.split('" (the text')[0] add(year, 'President', by, signed, obs, 'President, yearly autumn', url, q, c[4]) rows.sort(key=lambda r: (r['year'], r['observance'] or '')) assert len([r for r in rows if r['kind']=='President, yearly autumn']) == 83 # derived: weekday, day, nth Thursday for r in rows: if r['observance']: d = datetime.date.fromisoformat(r['observance']) r['weekday'] = d.strftime('%A') first = d.replace(day=1) # count Thursdays if d.weekday() == 3: nth = (d.day - 1)//7 + 1 ndays = (datetime.date(d.year + (d.month==12), d.month % 12 + 1, 1) - datetime.timedelta(days=1)).day total = sum(1 for k in range(1, ndays+1) if datetime.date(d.year, d.month, k).weekday()==3) r['thursday_of_month'] = f'{nth} of {total}' r['is_last_thursday'] = (nth == total) else: r['thursday_of_month'] = None; r['is_last_thursday'] = None else: r['weekday'] = None; r['thursday_of_month'] = None; r['is_last_thursday'] = None cols = ['year','body','proclaimed_by','kind','signed','observance','weekday','thursday_of_month','is_last_thursday','date_phrase','source_url','note'] with open(os.path.join(here, 'national-thanksgivings.csv'), 'w', newline='', encoding='utf-8') as f: w = csv.DictWriter(f, fieldnames=cols); w.writeheader() for r in rows: w.writerow({k: r.get(k) for k in cols}) # states 1939 with governor party party = {} ft = open(os.path.join(root, 'states-1939-footnote.txt'), encoding='utf-8').read() for m in re.finditer(r'\(([DR])\) ([A-Za-z ]+?)\*?\n', ft): party[m.group(2).strip()] = m.group(1) st = list(csv.DictReader(open(os.path.join(root, 'states-1939.csv'), encoding='utf-8'))) for s in st: s['governor_party'] = party[s['state']] with open(os.path.join(here, 'states-1939.csv'), 'w', newline='', encoding='utf-8') as f: w = csv.DictWriter(f, fieldnames=['state','usps','fips','date_observed','governor_party']); w.writeheader(); w.writerows(st) # analyses by_year = {} for r in rows: by_year.setdefault((r['year'], r['kind']), 0); by_year[(r['year'], r['kind'])] += 1 timeline = [] for y in range(1777, 1946): for k in ['Congress', 'President, one-time', 'President, yearly autumn']: if (y, k) in by_year: timeline.append({'year': y, 'kind': k, 'days': by_year[(y, k)]}) pres_years_pre = sorted({r['year'] for r in rows if r['body']=='President' and r['year'] <= 1862}) summary = [ {'period': '1777-1784', 'who': 'Congress', 'years': 8, 'years_with_thanksgiving': len({r['year'] for r in rows if r['body']=='Congress'})}, {'period': '1785-1788', 'who': 'Congress', 'years': 4, 'years_with_thanksgiving': 0}, {'period': '1789-1862', 'who': 'Presidents', 'years': 74, 'years_with_thanksgiving': len(pres_years_pre)}, {'period': '1863-1945', 'who': 'Presidents', 'years': 83, 'years_with_thanksgiving': len({r['year'] for r in rows if r['kind']=='President, yearly autumn'})}, ] annual = [r for r in rows if r['kind']=='President, yearly autumn'] off = [{'year': r['year'], 'president': r['proclaimed_by'], 'observance': r['observance'], 'thursday_of_month': r['thursday_of_month']} for r in annual if not r['is_last_thursday']] last_count = sum(1 for r in annual if r['is_last_thursday'] and r['year'] <= 1938) counts39 = {} for s in st: counts39.setdefault(s['date_observed'], {'D':0,'R':0}); counts39[s['date_observed']][s['governor_party']] += 1 split39 = [{'date_observed': k, 'states': v['D']+v['R'], 'democratic_governors': v['D'], 'republican_governors': v['R']} for k, v in sorted(counts39.items())] mapr = [{'fips': s['fips'], 'state': s['state'], 'usps': s['usps'], 'date_observed': {'Nov 23':"FDR's date (Nov. 23)", 'Nov 30':'Traditional date (Nov. 30)', 'Both':'Both days'}[s['date_observed']], 'states': 1} for s in st] out = dict(timeline=timeline, summary=summary, pres_years_pre=pres_years_pre, off_dates=off, last_thursday_1863_1938=last_count, annual_1863_1938=sum(1 for r in annual if r['year']<=1938), split39=split39, map39=mapr) json.dump(out, open(os.path.join(here, 'computed.json'), 'w', encoding='utf-8'), indent=1) print(json.dumps({k: v for k, v in out.items() if k not in ('timeline','map39')}, indent=1)) print(len(rows), 'rows;', len(timeline), 'timeline rows')