phowell
/
canvasapp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed reg_data using reg_history

This commit is contained in:
Peter Howell 2025-08-16 14:48:22 -07:00
parent f15d103fb8
commit 1a0daf82b5
1 changed files with 31 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
pipelines.py
View File

@ -1222,7 +1222,7 @@ def process_reg_history():
output1.write(f" [{dt}] {msg}\n") output1.write(f" [{dt}] {msg}\n")
def recreate_reg_data(): def recreate_reg_data():
import csv
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
@ -1230,43 +1230,55 @@ def recreate_reg_data():
dt = datetime.strptime(row['datetime'], "%Y-%m-%dT%H-%M") dt = datetime.strptime(row['datetime'], "%Y-%m-%dT%H-%M")
crn = row['crn'] crn = row['crn']
enrolled = int(row['enrolled']) enrolled = int(row['enrolled'])
return dt, crn, enrolled return dt, row['datetime'], crn, enrolled
def reduce_latest_per_day(rows): def reduce_latest_per_day(rows):
latest = defaultdict(dict) # latest[crn][date_str] = (dt, enrolled) latest = defaultdict(dict) # latest[crn][date] = (dt, ts, enrolled)
latest_ts_by_date = {} # date → (dt, ts) for header naming
for row in rows: for row in rows:
dt, crn, enrolled = parse_row(row) dt, full_ts, crn, enrolled = parse_row(row)
day = dt.date().isoformat() date_str = dt.date().isoformat()
if day not in latest[crn] or dt > latest[crn][day][0]: ts_header = dt.strftime("%Y-%m-%dT%H") # <-- this is what we want
latest[crn][day] = (dt, enrolled)
return latest
def pivot_to_table(latest_data): # for each crn, per day, keep latest reading
all_dates = sorted({day for crn in latest_data for day in latest_data[crn]}) if date_str not in latest[crn] or dt > latest[crn][date_str][0]:
crns = sorted(latest_data) latest[crn][date_str] = (dt, ts_header, enrolled)
# also record latest timestamp per day for consistent column headers
if date_str not in latest_ts_by_date or dt > latest_ts_by_date[date_str][0]:
latest_ts_by_date[date_str] = (dt, ts_header)
return latest, [ts for _, ts in sorted(latest_ts_by_date.values())]
def pivot_table(latest, headers):
crns = sorted(latest)
table = [] table = []
for crn in crns: for crn in crns:
row = [crn] row = [crn]
for day in all_dates: for ts in headers:
val = latest_data[crn].get(day, (None, None))[1] date_str = ts[:10] # match on YYYY-MM-DD
row.append(str(val) if val is not None else "") val = latest[crn].get(date_str)
if val and val[1] == ts:
row.append(str(val[2]))
else:
row.append("")
table.append(row) table.append(row)
return ["crn"] + all_dates, table return ['crn'] + headers, table
with open("cache/reg_history_fa25.csv", newline='') as f: with open("cache/reg_history_fa25.csv", newline='') as f:
fieldnames = ['datetime', 'crn', 'course', 'teacher', 'max', 'enrolled', 'waitlistmax', 'waitlisted'] fieldnames = ['datetime', 'crn', 'course', 'teacher', 'max', 'enrolled', 'waitlistmax', 'waitlisted']
reader = csv.DictReader(f, fieldnames=fieldnames) reader = csv.DictReader(f, fieldnames=fieldnames)
rows = list(reader) rows = list(reader)
latest = reduce_latest_per_day(rows) latest, headers = reduce_latest_per_day(rows)
header, table = pivot_to_table(latest) header_row, table = pivot_table(latest, headers)
with open("cache/reg_data_fa25.csv", "w", newline='') as f: with open("cache/reg_data_fa25.csv", "w", newline='') as f:
writer = csv.writer(f) writer = csv.writer(f)
writer.writerow(header) writer.writerow(header_row)
writer.writerows(table) writer.writerows(table)