12_23 edits

courses.py

@@ -324,8 +324,8 @@ def getCourses(x=0):   # a dict
         
     for id in courselist:
         t = url + '/api/v1/courses/' + str(id) # + '?perpage=100'
-        t = fetch(t,1)
+        t = fetch(t,0)
-    print(t)
+    #print(t)
     return t
 
 
@@ -993,7 +993,7 @@ def enroll_bulk_students_bydept(course_id, depts, the_term="172", cautious=1):
         #print(r3.text)
 
 
-def enroll_gott_workshops_su23():
+def enroll_gott_workshops():
     r = requests.get("https://www.gavilan.edu/staff/tlc/db.php?a=signups")
     signups = json.loads(r.text)
 
@@ -1010,7 +1010,11 @@ def enroll_gott_workshops_su23():
         #'GOTT 5: Essentials of Blended Learning2023-06-25 17:00:00': 17987, 
         #'GOTT 5: Essentials of Blended Learning (HyFlex)2023-06-25 17:00:00': 17987,
         #'GOTT 1: Intro to Teaching Online with Canvas2023-05-29 17:00:00': 17985,
-        'GOTT 1: Intro to Teaching Online with Canvas2023-08-20 17:00:00': 17994
+        #'GOTT 1: Intro to Teaching Online with Canvas2023-08-20 17:00:00': 17994
+        'GOTT 1: Intro to Online Teaching2024-01-02 16:00:00': 19221,
+        'GOTT 2: Intro to Asynchronous Teaching and Learning2024-01-02 16:00:00': 19222,
+        'GOTT 5: Essentials of Blended Learning2024-01-02 16:00:00': 19223,
+        'GOTT 6: Intro to Live Online Teaching and Learning2024-01-14 16:00:00': 19224,
     }
     #print(json.dumps(signups,indent=4))
     #print(json.dumps(by_email,indent=4))
@@ -1025,7 +1029,9 @@ def enroll_gott_workshops_su23():
             'bt@gavilan.edu': 'btagg@gavilan.edu',
             'tagg.brian@yahoo.com': 'btagg@gavilan.edu',
             'tmiller.realestate@gmail.com': 'tmiller@gavilan.edu',
-
+            'gemayo70@yahoo.com': 'pclaros@gavilan.edu',
+            'csalvin@gmail.com': 'csalvin@gavilan.edu',
+            'efalvey@aol.com': 'efalvey@gavilan.edu',
             }
 
     for wkshp,su_list in signups.items():
@@ -1059,12 +1065,12 @@ def enroll_art_students_live():
     print("done.")
 
 def enroll_orientation_students():
-    ori_shell_id = "15924"    #   2023 orientation shell   #      2022: "9768"
+    ori_shell_id = "19094"    #  2024    #  ""    #   2023 orientation shell  15924   #      2022: "9768"
 
     users_in_ori_shell = set( \
             [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
 
-    for the_semester in ["202350", "202370"]:
+    for the_semester in ["202430"]:
         #the_semester = "202350"   # "202350"
 
         users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
@@ -1360,9 +1366,81 @@ def teacher_to_many_shells():
         print(f"enrolled user id: {usrid} as teacher in course {id}.")
 
 
-
+import os, pickle
 
 def create_sandboxes():
+    courses_to_sandbox = [ (19221, ' Sandbox GOTT1 WI24'), 
+                           (19222, ' Sandbox GOTT2 WI24'),
+                           (19223, ' Sandbox GOTT5 WI24'),
+                           #(19224, ' Sandbox GOTT6 WI24')
+                         ]
+    filepath = 'cache/sandbox_courses.pkl'
+
+    if os.path.exists(filepath):
+        with open(filepath, 'rb') as f:
+            sandbox_log = pickle.load(f)
+    else:
+        sandbox_log = []
+
+
+     
+
+    for crs_id, label in courses_to_sandbox:
+        crs_info = getCourses(crs_id)
+        c_name = crs_info['name']
+        print(f"Students in course {crs_id}: {c_name}" )
+        enrolled = course_enrollment(crs_id)
+        for eid,stu in enrolled.items():
+            if stu['role'] != 'StudentEnrollment': 
+                continue
+            u_name = stu['user']['short_name']
+            u_id = stu['user']['id']
+            initials = ''.join([ x[0] for x in u_name.split(" ") ])
+            print(f"    id: {stu['user_id']}  ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}")
+            coursename = f"{initials}{label}"
+            if coursename in sandbox_log:
+                print(f"    - Already created: {coursename}")
+            else:
+                print(f"    + Creating course: {coursename} for {u_name}, id: {u_id}")
+                u2 = url + "/api/v1/accounts/1/courses"
+                data = { 
+                    "course[name]": coursename,
+                    "course[code]": coursename,
+                    "course[term_id]": "8",
+                }
+                
+                # Create a course
+                r3 = requests.post(u2, headers=header, params=data)
+                new_course_response = json.loads(r3.text)
+                id = new_course_response['id']
+                print(f"      created course id {id}")
+
+                # Add teacher
+                u3 = url + f"/api/v1/courses/{id}/enrollments" 
+                data2 = { "enrollment[type]":"TeacherEnrollment", "enrollment[user_id]":u_id,
+                        "enrollment[enrollment_state]":"active" }
+                r4 = requests.post(u3, headers=header, params=data2)
+                print(f"      enrolled user id: {u_id} as teacher.")
+
+                # Desired settings
+                data = { 'course[is_public_to_auth_users]': True, 'course[event]': 'offer' } 
+                t = url + f"/api/v1/courses/{id}"
+                r3 = requests.put(t, headers=header, params=data)
+                result = json.loads(r3.text)
+                if 'name' in result:
+                    print(f"    > Name: {result['name']}")
+                if 'workflow_state' in result:
+                    print(f"    > State: {result['workflow_state']}")
+                if 'is_public_to_auth_users' in result:
+                    print(f"    > Public: {result['is_public_to_auth_users']}")
+                sandbox_log.append(coursename)
+
+                # Write log back out
+                with open(filepath, 'wb') as handle:
+                    pickle.dump(sandbox_log, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+    return 1
+
     # ('ED','82'), 
     sandboxes = [ ('JH','45324'), ('PK','38183'), ('GM','5167'), ('BS','19231'),
                   ('ST','303'), ('KW','5145')]
@@ -1380,7 +1458,7 @@ def create_sandboxes():
 
     sandboxes = [('HA','61620'), ('AS','61451'), ('MP', '11565'), ('AA','51276') ]
     sandboxes = [('JR','61062')]
-    report = []
+
     for (N,usrid) in sandboxes:
         coursename = f"{N} Sandbox SU23 (GOTT1)"
         coursecode = f"{N} SU23 Sandbox (GOTT1)"
@@ -1822,12 +1900,13 @@ def create_calendar_event():
         local = pytz.timezone("America/Los_Angeles")
         naive = datetime.strptime(date, "%Y-%m-%d")
         local_dt = local.localize(naive, is_dst=None)
-        utc_dt = local_dt.timezone(pytz.utc).isoformat()
+        utc_dt = local_dt.astimezone(pytz.utc).isoformat()
 
 
 
         params = {
             "calendar_event[context_code]": "course_15924",      # 2023 student orientation
+            "calendar_event[context_code]": "course_19094",      # 2024         orientation
             "calendar_event[title]": title,
             "calendar_event[description]": desc,
             "calendar_event[start_at]": utc_dt,              # DateTime
@@ -1948,7 +2027,7 @@ if __name__ == "__main__":
                 34: ['Create calendar event', create_calendar_event],
                 35: ['list all assignments', list_all_assignments],
 
-                40: ['Enroll GOTT Workshops', enroll_gott_workshops_su23], 
+                40: ['Enroll GOTT Workshops', enroll_gott_workshops], 
                 42: ['Add teacher to many shells', teacher_to_many_shells], 
                 43: ['Bulk unenroll from course', bulk_unenroll],
                 # 24: ['Add course evals to whole semester',instructor_list_to_activate_evals],   

localcache.py

@@ -1150,6 +1150,7 @@ def full_reload():
     #merge_requests()
 
     #make_views_summarys()
+    sched_to_db()
 
 def guess_dept(t):
     #print(t)
@@ -1679,11 +1680,12 @@ GROUP BY h.address;"""
 
 
 def all_sem_courses_teachers():
-    q = """SELECT c.id, c.canvasid AS course_cid, c.name, c.code, u.name, u.sortablename, u.canvasid AS user_cid, p.sis_user_id  FROM courses AS c
+    SEM = "202430"
+    q = f"""SELECT c.id, c.canvasid AS course_cid, c.name, c.code, u.name, u.sortablename, u.canvasid AS user_cid, p.sis_user_id  FROM courses AS c
 JOIN enrollment AS e ON e.course_id=c.id
 JOIN users AS u ON u.id=e.user_id
 JOIN pseudonym AS p ON p.user_id=u.id
-WHERE c.sis LIKE "202170-%" 
+WHERE c.sis LIKE "{SEM}-%" 
 AND NOT c.state="deleted"
 AND e."type"="TeacherEnrollment"
 ORDER BY u.sortablename;"""
@@ -1994,7 +1996,8 @@ def sched_to_db():
 ) ;
 '''
 
-    conn,cur = db('cache/canvas_data/data20231012.db')
+    #conn,cur = db('cache/canvas_data/data20231012.db')
+    conn,cur = db()
     print(table)
     cur.execute(d)
     cur.execute(table)
@@ -2005,7 +2008,7 @@ def sched_to_db():
     i = 0
 
     output = codecs.open('cache/schedule.sql','w','utf-8')
-    for year in ['16','17','18','19','20','21','22','23']:
+    for year in ['16','17','18','19','20','21','22','23','24']:
         for sem in ['sp','su','fa']:
             term = f"{sem}{year}"
             print(term)
@@ -2062,6 +2065,20 @@ ORDER BY u.sortablename, t.sis, c.code ;'''
     #    print(json.dumps(result,indent=2))
     return result
 
+def sem_schedule(sem):
+    q = f"""SELECT * FROM schedule WHERE sem='{sem}' ORDER BY code,crn;"""
+    return query_multiple(q)
+
+
+def course_mode(crn,sem):
+    q = f"""SELECT type FROM schedule WHERE crn='{crn}' AND sem='{sem}';"""
+    #print(q)
+    result = query_multiple(q)
+    if result:
+        return result[0]['type']
+    return ''
+
+
 if __name__ == "__main__":
 
     print ('')

pipelines.py

@@ -189,7 +189,7 @@ dean['HIST'] = 'nl'
 dean['HUM'] = 'nl'
 dean['JOUR'] = 'nl'
 dean['JPN'] = 'nl'
-dean['LIB'] = 'kn'
+dean['LIB'] = 'jn'
 dean['MATH'] = 'jn'
 dean['MCTV'] = 'nl'
 dean['MGMT'] = 'ss'

stats.py

@@ -1282,6 +1282,105 @@ def visualize_course_modes_multi_semester():
 
 
 
+# given a list of classes, report back about the student on one row of info
+def student_history_analysis(sh):
+    from functools import reduce
+    semesters_set = set()
+    num_sems = 0
+    num_course = len(sh)
+    num_units = 0
+    units_online = 0
+    units_inperson = 0
+    units_hybrid = 0
+    units_ol = 0
+    fa_23_units = 0
+    fa_23_online_units = 0
+    fa23_courses = 0
+    fa23_onlinecourses = 0
+    
+    #un_list = [ float(x['units'].split('-')[0].split('/')[0]) for x in sh ]
+    #num_units = reduce(lambda x,y: x+y, un_list)
+    for section in sh:
+        semesters_set.add(section['sis'])
+        units = float(section['units'].split('-')[0].split('/')[0])
+        num_units += units
+        if section['type'] == 'in-person': units_inperson += units
+        if section['type'] == 'online': units_online += units
+        if section['type'] == 'hybrid': units_hybrid += units
+        if section['type'] == 'online live': units_ol += units
+
+        if section['sis'] == '202370':
+            fa_23_units += units
+            fa23_courses += 1
+            if not section['type'] == 'in-person': 
+                fa_23_online_units += units
+                fa23_onlinecourses += 1
+
+    num_sems = len(semesters_set)
+    if num_units == 0:
+        pct_online = 0
+    else:
+        pct_online = round(100 * (units_online+units_hybrid+units_ol) / num_units, 1)
+
+    if fa_23_units == 0:
+        fa_23_pct_online = 0
+    else:
+        fa_23_pct_online = round(100 * (fa_23_online_units) / fa_23_units, 1)
+    
+    if fa23_courses == 0:
+        fa23_pct_course_online = 0
+    else:
+        fa23_pct_course_online = round(100 * (fa23_onlinecourses) / fa23_courses, 1)
+    summary = [units, num_course, f"\"{sh[0]['sortablename']}\",{sh[0]['canvasid']},{num_sems},{num_course},{num_units},{units_online},{units_inperson},{units_hybrid},{units_ol},{pct_online},{fa_23_units},{fa_23_online_units},{fa_23_pct_online},{fa23_courses},{fa23_onlinecourses},{fa23_pct_course_online}"]
+    return summary
+
+def report_student_stats():
+    from localcache import users_with_history, students_current_semester
+    from itertools import groupby
+    import plotly.graph_objects as go
+    import plotly.io as pio
+    import numpy as np
+
+    u = users_with_history()
+    this_sem = [x['canvasid'] for x in students_current_semester()]
+
+    df = pd.DataFrame(u)
+    filtered_df = df[df['canvasid'].isin(this_sem)]
+    filtered_df.to_csv('cache/student_history_current_students.csv',index=False)
+
+    oo = codecs.open('cache/student_units.txt','w','utf-8')
+    oo.write("name,id,num_sems,num_course,num_units,units_online,units_inperson,units_hybrid,units_ol,percent_online,fa23_units,fa23_onlineunits,fa23_pct_online,fa23_num_courses,fa23_num_onlinecourses,fa23_percent_online_course\n")
+    # Now group by that key
+    def kk(x): return x['canvasid']
+    grouped_dict = {key:list(group) for key, group in groupby(u, kk)}
+
+    shorter = []
+    percentages = []
+
+    for k,g in grouped_dict.items():
+        if k in this_sem:
+            h = student_history_analysis(g)
+            #oo.write(json.dumps(h[2],indent=2)+ "\n")
+            oo.write( str(h[2]) + "\n")
+            shorter.append(h)
+            p = h[2].split(',')[-1]
+            percentages.append(float(p))
+        else:
+            print(f"Skipping {k}")
+    #print(this_sem)
+    #oo.write('units,courses\n')
+    #shorter.sort(key=lambda x: x[0], reverse=True)
+    #for s in shorter: 
+    #    print(s[2])
+    #    #oo.write(f"{s[0]},{s[1]}\n")
+    #    #print('\n\n')
+
+    # Create a histogram
+    fig = go.Figure(data=[go.Histogram(x=percentages, xbins=dict(start=0,end=101, size=10))])
+
+    # Save the figure in an HTML file
+    pio.write_html(fig, 'cache/student_pct_onlinecourse.html')
+
 
 
 if __name__ == "__main__":
@@ -1303,6 +1402,7 @@ if __name__ == "__main__":
                 16: ['LSTM model sections', lstm_model_sections],
                 17: ['rearrange section data to yearly form', sections_grouped_by_year_mode],
                 30: ['visualize course modes multi semester', visualize_course_modes_multi_semester],
+                31: ['Report on student stats', report_student_stats],
               }
     print ('')
 

users.py

@@ -2012,9 +2012,9 @@ def nlp_sample2():
 
 def one_course_enrol():
 
-    users = '96 18771 2693 5863 327'.split()
+    users = '25730'.split()
-    course = '11015'
+    course = '16459'
-    the_type = 'TeacherEnrollment'                          # 'StudentEnrollment'
+    the_type = 'StudentEnrollment'   # 'TeacherEnrollment'                          # 
     u = url + '/api/v1/courses/%s/enrollments' % course
     
     for user in users:
@@ -2228,91 +2228,334 @@ def compare_db_tables():
     for e in common_emails:
         out.write(f"update `conf_users` set `p2id`='{by_email_conf[e]['id']}' where lower(`email`)='{e}';\n")
 
-# given a list of classes, report back about the student on one row of info
+
-def student_history_analysis(sh):
+def training_find_goos():
-    from functools import reduce
+    from localcache import all_sem_courses_teachers
-    semesters_set = set()
+    from localcache import course_mode
-    num_sems = 0
+    from localcache import sem_schedule
-    num_course = len(sh)
+    from pipelines import dean
-    num_units = 0
+    from openpyxl import Workbook, load_workbook
-    units_online = 0
+    from openpyxl.chart import BarChart, Series, Reference
-    units_inperson = 0
+    from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
-    units_hybrid = 0
+    wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
-    units_ol = 0
+    print(wb.sheetnames)
-    fa_23_units = 0
+
-    fa_23_online_units = 0
+    all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())
-    fa23_courses = 0
+
-    fa23_onlinecourses = 0
+    records = {}
+    sheets = ['GOTT1', 'GOTT2', 'GOTT4', 'GOTT5', 'GOTT6', 'HUM.STEM', 'POCR Reviewed','BOOT CAMP','TITLE V GOTT ACADEMY', 'Other Certifications']
+    for sname in sheets:
+        print(f"{sname}:")
+        sheet = wb[sname]
+        records[sname] = {}
+        for row in sheet.iter_rows():
+            try:
+                name = row[0].value
+                if row[0].value == 'G Number': continue
+                lastname = name.split(' ')[-1].lower()
+                goo = row[1].value
+                if not goo:
+                    print(f"   Missing ID: {row[0].value}")
+                    for t in all_teachers:
+                        if re.search(lastname, t['name'].lower()):
+                            print(f"      {t['name']}: {t['sis_user_id']}")
+            except Exception as e:
+                pass
+        print()
+
+def cross_ref_training():
+    from localcache import all_sem_courses_teachers
+    from localcache import course_mode
+    from localcache import sem_schedule
+    from pipelines import dean
+    from openpyxl import Workbook, load_workbook
+    from openpyxl.chart import BarChart, Series, Reference
+    from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
+    wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
+    print(wb.sheetnames)
+
+    all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())
+
+    records = {}
+    sheets = ['GOTT1', 'GOTT2', 'GOTT4', 'GOTT5', 'GOTT6', 'HUM.STEM', 'POCR Reviewed', 'SU21 Workshop', 'BOOT CAMP', 'GOTT ABC', 'TITLE V GOTT ACADEMY', 'Other Certifications']
+    for sname in sheets:
+        print(f"{sname}:")
+        sheet = wb[sname]
+        records[sname] = {}
+        for row in sheet.iter_rows():
+            if row[0].value == 'G Number': continue
+            goo = row[1].value
+            rowvals = [str(v.value) for v in row]
+            records[sname][goo] = rowvals
+            print("  " + " ".join(rowvals))
+        print()
+    #print(json.dumps(records,indent=2))
+    teachers = defaultdict(list)
+
+    teachers_bydept = defaultdict(set)
+    alldepts = set()
+
+
+    courses = all_sem_courses_teachers()
+    for c in courses:
+        goo = c[7]
+        crn = c[2].split(' ')[-1].split('/')[0]
+        name = c[2]
+        teacher = c[4]
+        ctype = course_mode(crn,'sp24')
+        dept1 = re.search(r'([A-Z]+)(\d+)',c[2].split(' ')[0]).group(1)
+        alldepts.add(dept1)
+        d = list(c)
+        d.append(ctype)
+        if not ctype:
+            print(f"not finding mode for {name}")
+            continue
+        if ctype=='in-person': continue
+        teachers[teacher].append(d)
+        teachers_bydept[dept1].add(teacher)
+
+    alldepts = list(alldepts)
+    alldepts.sort()
+
+    sheet = wb.create_sheet("Spring 2024 Summary")
+    r = 1
+
+    deptfont = Font(bold=True)
+    flagfont = PatternFill("solid", fgColor="00FFFFCC")
+
+    for thedean in ['et','nl','ss','jn']:
+        sheet.cell(row=r, column=1).value = thedean
+        sheet.cell(row=r, column=1).font = deptfont
+        r += 2
+
+        for D in alldepts:
+            if not D in dean:
+                print(f"MISSING DEAN for dept: {D}")
+            if dean[D] == thedean:
+                if len(teachers_bydept[D]) == 0: continue
+                print(f"\n------------\n{D}")
+                sheet.cell(row=r, column=1).value = D
+                sheet.cell(row=r, column=1).font = deptfont
+                r += 1
+
+                for t in teachers_bydept[D]:
+                    if t == 'STAFF STAFF': continue
+                    completed = 0
+                    waived = 0
+                    sects = teachers[t]
+                    goo = sects[0][7]
+                    print(t)
+                    sheet.cell(row=r, column=1).value = f"{t}"
+                    sheet.cell(row=r, column=2).value = f"{goo}"
+                    r += 1
+                    if goo in records['GOTT1']:
+                        sheet.cell(row=r, column=2).value = f"✓ GOTT 1 Trained"
+                        completed =1
+                        r += 1
+                    if goo in records['Other Certifications']:
+                        sheet.cell(row=r, column=2).value = f"✓ GOTT Waived - Outside Training"
+                        completed = 1
+                        waived = 1
+                        r += 1
+                    if goo in records['GOTT2']:
+                        sheet.cell(row=r, column=2).value = f"✓ GOTT 2 Trained"
+                        completed = 1
+                        waived = 1
+                        r += 1
+                    if goo in records['POCR Reviewed']:
+                        sheet.cell(row=r, column=2).value = f"✓ POCR Reviewed"
+                        completed = 1
+                        waived = 1
+                        r += 1
+                    if goo in records['TITLE V GOTT ACADEMY']:
+                        sheet.cell(row=r, column=2).value = f"✓ TITLE V GOTT ACADEMY 2014"
+                        completed = 1
+                        waived = 1
+                        r += 1
+                    if not completed:
+                        if waived:
+                            sheet.cell(row=r, column=2).value = f"✓ GOTT 1 Waived"
+                        else:
+                            sheet.cell(row=r, column=2).value = f"- MISSING GOTT 1"
+                            sheet.cell(row=r-1,column=1).fill = flagfont
+                        r += 1
+                    if goo in records['GOTT4']:
+                        sheet.cell(row=r, column=2).value = f"✓ GOTT 4 Trained"
+                        r += 1
+                    if goo in records['GOTT5']:
+                        sheet.cell(row=r, column=2).value = f"✓ GOTT 5 Trained"
+                        r += 1
+                    if goo in records['GOTT6']:
+                        sheet.cell(row=r, column=2).value = f"✓ GOTT 6 Trained"
+                        r += 1
+                    if goo in records['SU21 Workshop']:
+                        sheet.cell(row=r, column=2).value = f"✓ SU21 Workshop"
+                        r += 1
+                    if goo in records['HUM.STEM']:
+                        sheet.cell(row=r, column=2).value = f"✓ Humanizing Stem"
+                        r += 1
+                    if goo in records['BOOT CAMP']:
+                        sheet.cell(row=r, column=2).value = f"✓ Boot Camp Self Paced"
+                        r += 1
+                    if goo in records['GOTT ABC']:
+                        sheet.cell(row=r, column=2).value = f"✓ {records['GOTT ABC'][goo][2]} Self Paced"
+                        r += 1
+                    for s in sects:
+                        sheet.cell(row=r, column=2).value = f"{s[8]}"
+                        sheet.cell(row=r, column=3).value = f"{s[2]}"
+                        r += 1
     
-    #un_list = [ float(x['units'].split('-')[0].split('/')[0]) for x in sh ]
+    #for c in sheet.columns:
-    #num_units = reduce(lambda x,y: x+y, un_list)
+    #    print(c)
-    for section in sh:
+    #    print(f"{c} width: {sheet.column_dimensions[c].value}")
-        semesters_set.add(section['sis'])
+    sheet.column_dimensions['A'].width = 20
-        units = float(section['units'].split('-')[0].split('/')[0])
+    sheet.column_dimensions['B'].width = 30
-        num_units += units
+    sheet.column_dimensions['C'].width = 75
-        if section['type'] == 'in-person': units_inperson += units
+    wb.save("C:/Users/peter/Downloads/GOTT_Completion_masterlist 102023_summarized.xlsx")
-        if section['type'] == 'online': units_online += units
-        if section['type'] == 'hybrid': units_hybrid += units
-        if section['type'] == 'online live': units_ol += units
 
-        if section['sis'] == '202370':
+def cross_ref_training_withcsv():
-            fa_23_units += units
+    from localcache import all_sem_courses_teachers
-            fa23_courses += 1
+    from localcache import course_mode
-            if not section['type'] == 'in-person': 
+    from localcache import sem_schedule
-                fa_23_online_units += units
-                fa23_onlinecourses += 1
 
-    num_sems = len(semesters_set)
+    gott1 = {}
-    if num_units == 0:
+    g1read = csv.reader(codecs.open('cache/GOTT_1.csv','r','utf-8'))
-        pct_online = 0
+    i = 0
-    else:
+    for row in g1read:
-        pct_online = round(100 * (units_online+units_hybrid+units_ol) / num_units, 1)
+        if i == 0:
-
+            headers = row
-    if fa_23_units == 0:
+            #print(f"Headers: {headers}")
-        fa_23_pct_online = 0
+            i += 1
-    else:
-        fa_23_pct_online = round(100 * (fa_23_online_units) / fa_23_units, 1)
-    
-    if fa23_courses == 0:
-        fa23_pct_course_online = 0
-    else:
-        fa23_pct_course_online = round(100 * (fa23_onlinecourses) / fa23_courses, 1)
-    summary = [units, num_course, f"\"{sh[0]['sortablename']}\",{sh[0]['canvasid']},{num_sems},{num_course},{num_units},{units_online},{units_inperson},{units_hybrid},{units_ol},{pct_online},{fa_23_units},{fa_23_online_units},{fa_23_pct_online},{fa23_courses},{fa23_onlinecourses},{fa23_pct_course_online}"]
-    return summary
-
-def report_student_stats():
-    from localcache import users_with_history, students_current_semester
-    from itertools import groupby
-    u = users_with_history()
-    this_sem = [x['canvasid'] for x in students_current_semester()]
-
-    df = pd.DataFrame(u)
-    filtered_df = df[df['canvasid'].isin(this_sem)]
-    filtered_df.to_csv('cache/student_history_current_students.csv',index=False)
-
-    oo = codecs.open('cache/student_units.txt','w','utf-8')
-    oo.write("name,id,num_sems,num_course,num_units,units_online,units_inperson,units_hybrid,units_ol,percent_online,fa23_units,fa23_onlineunits,fa23_pct_online,fa23_num_courses,fa23_num_onlinecourses,fa23_percent_online_course\n")
-    # Now group by that key
-    def kk(x): return x['canvasid']
-    grouped_dict = {key:list(group) for key, group in groupby(u, kk)}
-
-    shorter = []
-
-    for k,g in grouped_dict.items():
-        if k in this_sem:
-            h = student_history_analysis(g)
-            #oo.write(json.dumps(h[2],indent=2)+ "\n")
-            oo.write( str(h[2]) + "\n")
-            shorter.append(h)
         else:
-            print(f"Skipping {k}")
+            goo = row[1]
-    #print(this_sem)
+            gott1[goo] = row
-    #oo.write('units,courses\n')
+
-    #shorter.sort(key=lambda x: x[0], reverse=True)
+    gott2 = {}
-    #for s in shorter: 
+    g2read = csv.reader(codecs.open('cache/GOTT_2.csv','r','utf-8'))
-    #    print(s[2])
+    i = 0
-    #    #oo.write(f"{s[0]},{s[1]}\n")
+    for row in g2read:
-    #    #print('\n\n')
+        if i == 0:
+            headers = row
+            #print(f"Headers: {headers}")
+            i += 1
+        else:
+            goo = row[1]
+            gott2[goo] = row
+
+    gott4 = {}
+    g4read = csv.reader(codecs.open('cache/GOTT_4.csv','r','utf-8'))
+    i = 0
+    for row in g4read:
+        if i == 0:
+            headers = row
+            #print(f"Headers: {headers}")
+            i += 1
+        else:
+            goo = row[1]
+            gott4[goo] = row
+
+    gott5 = {}
+    g5read = csv.reader(codecs.open('cache/GOTT_5.csv','r','utf-8'))
+    i = 0
+    for row in g5read:
+        if i == 0:
+            headers = row
+            #print(f"Headers: {headers}")
+            i += 1
+        else:
+            goo = row[1]
+            gott5[goo] = row
+
+    gott6 = {}
+    g6read = csv.reader(codecs.open('cache/GOTT_6.csv','r','utf-8'))
+    i = 0
+    for row in g6read:
+        if i == 0:
+            headers = row
+            #print(f"Headers: {headers}")
+            i += 1
+        else:
+            goo = row[1]
+            gott6[goo] = row
+
+    gott9 = {}
+    g9read = csv.reader(codecs.open('cache/GOTT_others.csv','r','utf-8'))
+    i = 0
+    for row in g9read:
+        if i == 0:
+            headers = row
+            #print(f"Headers: {headers}")
+            i += 1
+        else:
+            goo = row[1]
+            gott9[goo] = row
+
+
+    #report = codecs.open('cache/training_report.csv','w','utf-8')
+    #out = csv.writer(report)
+    #out.writerow(['crn','course','mode','teacher','goo','training','training_date'])
+
+    #sections = sem_schedule('sp24')
+    teachers = defaultdict(list)
+
+    teachers_bydept = defaultdict(set)
+    alldepts = set()
+
+
+    courses = all_sem_courses_teachers()
+    for c in courses:
+        goo = c[7]
+        crn = c[2].split(' ')[-1].split('/')[0]
+        name = c[2]
+        teacher = c[4]
+        ctype = course_mode(crn,'sp24')
+
+        dept1 = re.search(r'([A-Z]+)(\d+)',c[2].split(' ')[0]).group(1)
+        alldepts.add(dept1)
+        d = list(c)
+        d.append(ctype)
+        if ctype=='in-person': continue
+        teachers[teacher].append(d)
+
+        teachers_bydept[dept1].add(teacher)
+
+
+        #print(f"{crn} {ctype} {name}  ")
+        #if goo in gott1:
+        #    out.writerow([c[1], c[2], ctype, c[4], goo, "GOTT 1", "/".join(gott1[goo][2:])])
+        #
+        #else:
+        #    out.writerow([c[1], c[2], ctype, c[4], goo, "GOTT 1 MISSING", ''])
+    
+    alldepts = list(alldepts)
+    alldepts.sort()
+
+    for D in alldepts:
+        print(f"\n------------\n{D}")
+        for t in teachers_bydept[D]:
+            if t == 'STAFF STAFF': continue
+            sects = teachers[t]
+            print(t)
+            goo = sects[0][7]
+            if goo in gott1:
+                print("   + GOTT 1 Trained")
+            else:
+                print("   - MISSING GOTT 1")
+            if goo in gott2:
+                print("   + GOTT 2 Trained")
+            if goo in gott4:
+                print("   + GOTT 4 Trained")
+            if goo in gott5:
+                print("   + GOTT 5 Trained")
+            if goo in gott6:
+                print("   + GOTT 6 Trained")
+            if goo in gott9:
+                print("   + GOTT Waived - Outside Training")
+            for s in sects:
+                print(f"   {s[8]}   {s[2]}")
+            print()
+
+
 
 
 if __name__ == "__main__":
@@ -2340,7 +2583,8 @@ if __name__ == "__main__":
                 22: ['Sync personnel and conference user databases', user_db_sync], 
                 23: ['Find non-gnumbers', find_no_goo ],
                 24: ['compare user tables', compare_db_tables],
-                25: ['Report on student stats', report_student_stats],
+                25: ['cross ref training', cross_ref_training],
+                26: ['find goo numbers in training spreadsheet', training_find_goos],
                 #3: ['Main index, 1 year, teachers and their classes', getAllTeachersInTerm],
                 #5: ['Match names in schedule & ilearn', match_usernames],
                 #6: ['Create Dept\'s ZTC list', create_ztc_list],