semester start updates

content.py

@@ -1,6 +1,7 @@
 
 
 #saved_titles = json.loads( codecs.open('cache/saved_youtube_titles.json','r','utf-8').read() )
+from calendar import FRIDAY
 import requests, codecs, os, re, json, sys, pypandoc
 import webbrowser, bs4, trafilatura, pickle, tomd, checker 
 import html2markdown as h2m
@@ -614,6 +615,102 @@ def update_page():
     fixed_page = checker.safe_html(mypage['body'])
     upload_page(course_num,chosen_url,fixed_page)
 
+# given dict of file info (from files api), construct an img tag that works in a page
+#def file_to_img_tag(f, alt, course, soup):
+#    #tag = f"<img id=\"\" src=\"https://ilearn.gavilan.edu/courses/{course}/files/{f['id']}/preview\" alt=\"{f['filename']}\" "
+#    #tag += f"data-api-endpoint=\"https://ilearn.gavilan.edu/api/v1/courses/{course}/files/{f['id']}\" data-api-returntype=\"File\" />"
+#    return T
+
+
+def html_file_to_page(filename, course, tags):
+    
+    try:
+        soup = bs4.BeautifulSoup(codecs.open(filename,'r', 'utf-8').read(), 'html.parser')
+    except Exception as e:
+        print(f"Exception on {filename}: {e}")
+        return
+    img_tags = soup.find_all('img')
+    
+    result = {'title': soup.title.text if soup.title else ''}
+    result['title'].strip()
+    
+    for img in img_tags:
+        src = img['src']
+        try:
+            alt = img['alt']
+        except:
+            alt = src
+        orig_filename = os.path.basename(src)
+        if orig_filename in tags:
+            T = soup.new_tag(name='img', src=f"https://ilearn.gavilan.edu/courses/{course}/files/{tags[orig_filename]['id']}/preview")
+            T['id'] = tags[orig_filename]['id']
+            T['alt'] = alt
+            T['data-api-endpoint'] = f"https://ilearn.gavilan.edu/api/v1/courses/{course}/files/{tags[orig_filename]['id']}"
+            T['data-api-returntype'] = "File"            
+            img.replace_with(T)
+            print( f"   replaced image: {src}   alt: {alt}")
+        else:
+            print( f"   couldn't find replacement image: {src}   alt: {alt}")
+    outfile = codecs.open(filename+"_mod.html", 'w', 'utf-8')
+    outfile.write( soup.prettify() )
+    outfile.close()
+    result['body'] = ''.join(map(str, soup.body.contents)) if soup.body else ''
+    return result
+
+def create_new_page(course_id, title, body):
+    print(f"Creating page: {title}, length: {len(body)}")
+    request = f"{url}/api/v1/courses/{course_id}/pages"
+    print(request)
+    data = { 'wiki_page[title]': title, 'wiki_page[body]': body }
+    r3 = requests.post(request, headers=header, data=data)
+    try:
+        result = json.loads(r3.text)
+        print( f"    + ok: {result['url']}")
+    except:
+        print("    - problem creating page?")
+
+# Given a folder full of html pages and their linked images, create Canvas PAGES of them
+def make_pages_from_folder(folder='cache/csis6/', course = '20558'):
+    if 0:
+        request = f"{url}/api/v1/courses/{course}/files"
+        print("Fetching course files")
+        files = fetch(request)
+    
+        tempfile = codecs.open('cache/csis6filelist.json','w','utf-8')
+        tempfile.write(json.dumps(files))
+        tempfile.close()
+        
+    if 1:
+        files = json.loads( codecs.open('cache/csis6filelist.json', 'r', 'utf-8').read())
+    
+
+
+    course_files = {f['filename']: f for f in files}
+    tags = {}
+    for f in files:
+        if f['filename'].lower().endswith('.jpg') or f['filename'].lower().endswith('.png'):
+            tags[f['filename']] = f
+
+
+    contents = os.listdir(folder)
+    contents = ['welcome.html','welcome2.html', 'welcome3.html']
+    print(contents)
+    for f in contents:
+        m = re.search(r'^(.*)\.(html?)$', f)
+        if m:
+            print(f"html file: {m.group(1)}, extension: {m.group(2)}")
+            newpage = html_file_to_page(folder+f, course, tags)
+            create_new_page(course, newpage['title'], newpage['body'])
+        else:
+            m = re.search(r'^(.*)\.(.*)$', f)
+            if m:
+                print(f"other file: {m.group(1)}, extension: {m.group(2)}")
+            else:
+                print(f"unknown file: {f}")
+
+
+
+
 # Given course, page url, and new content, upload the new revision of a page
 def upload_page(course_num,pageurl,new_content):
     print("Repaired page:\n\n")
@@ -1385,6 +1482,7 @@ if __name__ == "__main__":
                15: ['test priority', test_priority], 
                16: ['test embed', test_embed], 
                17: ['repair ezproxy links', repairy_ezproxy_links],
+               18: ['create pages from html files', make_pages_from_folder],
               }
     
     if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):

courses.py

@@ -4,11 +4,11 @@ from datetime import datetime
 import pytz
 from dateutil import parser
 from datetime import datetime
-from util import print_table, int_or_zero, float_or_zero
+from util import print_table, int_or_zero, float_or_zero, dept_from_name, num_from_name
 from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem
 from pipelines import sems
-from localcache import db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
+from localcache import course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
-from localcache2 import users_new_this_semester
+from localcache2 import db, users_new_this_semester, users_new_this_2x_semester, course_from_id, user_ids_in_shell
 from collections import defaultdict
 
 
@@ -545,8 +545,8 @@ def all_equal2(iterator):
 177     2023 Winter
 """
 def semester_cross_lister():
-    sem = "sp24"
+    sem = "fa24"
-    term = 181
+    term = 184    # fa24=184
     xlist_filename = f"cache/{sem}_crosslist.csv"
     checkfile = codecs.open('cache/xlist_check.html','w','utf-8')
     checkfile.write('<html><body><table>\n')
@@ -558,7 +558,7 @@ def semester_cross_lister():
     crn_to_canvasname = {}
     crn_to_canvascode = {}
     
-    get_fresh = 0
+    get_fresh = 1
     c = getCoursesInTerm(term,get_fresh,0)    
     
     for C in c:
@@ -629,13 +629,74 @@ def semester_cross_lister():
             xlist_ii(target_section[3],host_id,new_name,new_code)
             #pass
 
+def do_manual_xlist():
+    infile = [ x.strip() for x in open('cache/fa24_manual_crosslist.txt','r').readlines() ]
+    for L in infile:
+        print(L)
+        paraL,host = L.split(' -> ')
+        para_list = paraL.split(',')
+        print(host)
+        print(para_list)
+        xlist(host, para_list)
+
+# Crosslist given 2 ids, computing the new name and code
+def xlist(host_id, parasite_list):
+    host_info = course_from_id(host_id)
+    host_info['crn'] = host_info['sis_source_id'][7:]
+    host_info['dept'] = dept_from_name( host_info['course_code'] )
+    host_info['num'] = num_from_name(host_info['course_code'] )
+    host_info['bare_name'] = ' '.join(host_info['name'].split(' ')[1:-1])    # name without course code or crn
+
+    sem = host_info['course_code'].split(' ')[1]
+    
+    para_info_list = [ course_from_id(x) for x in parasite_list ]
+    for p in para_info_list:
+        p['crn'] = p['sis_source_id'][7:]
+        p['dept'] = dept_from_name(p['course_code'] )
+        p['num'] = num_from_name(p['course_code'] )
+        p['bare_name'] = ' '.join(p['name'].split(' ')[1:-1])    # name without course code or crn
+    all = para_info_list.copy()
+    all.append(host_info)
+
+    # determine new name and code
+    sects = [ z['crn'] for z in all ]
+    sects.sort()
+    nic = numbers_in_common(sects)
+    new_sec = combined_name(nic,sects)
+
+    # same dept?
+    depts_list = [ z['dept'] for z in all ]
+    nums_list = list(set([ z['num'] for z in all ]))
+    if all_equal2(depts_list):
+        depts = depts_list[0] 
+        nums_list.sort()
+        nums = '/'.join(nums_list)
+    else:
+        depts = list(set(depts_list))
+        depts.sort()
+        depts = '/'.join(depts )
+        nums = all[0]['num']
+    
+    new_name = f"{depts}{nums} {all[0]['bare_name']} {new_sec}"
+    #new_name = by_group[y][0][4][0:-5] + new_sec
+    new_code = f"{depts}{nums} {sem.upper()} {new_sec}"
+    #new_code = by_group[y][0][5][0:-5] + new_sec
+    print(f"New name: {new_name}")
+    print(f"New code: {new_code}")
+    print(sects)
+
+    for target_section in para_info_list:
+        xlist_ii(target_section['id'],host_id,new_name,new_code)
+
+
+
 # Perform an actual cross-list, given 2 id numbers, new name and code
 def xlist_ii(parasite_id,host_id,new_name,new_code):
     print("Parasite id: ",parasite_id,"  Host id: ", host_id)
     print("New name: ", new_name)
     print("New code: ", new_code)
     xyz = 'y'
-    #xyz = input("Perform cross list? Enter for yes, n for no: ")
+    #xyz = input("Perform cross list? Enter y for yes, n for no: ")
     if xyz != 'n':
         uu = url + '/api/v1/courses/%s/sections' % parasite_id
         c_sect = fetch(uu)
@@ -843,7 +904,7 @@ def eslCrosslister():
         if not c:
             print("Didn't catch: "+ str(combos[i]))
             
-def xlist(parasite='', host=''):        # section id     ,    new course id 
+def xlist_iii(parasite='', host=''):        # section id     ,    new course id 
 
     host = host or input("ID number of the HOSTING COURSE? ")
     if not parasite:
@@ -894,7 +955,7 @@ def enroll_id_list_to_shell(id_list, shell_id, v=0):
 
     for j in enroll_us:
         try:
-            q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+            q = "SELECT name,id FROM canvas.users u WHERE u.id=%s" % j
             cursor.execute(q)
             s = cursor.fetchall()
             if s:
@@ -1057,8 +1118,13 @@ def enroll_bulk_students_bydept(course_id, depts, the_term="172", cautious=1):
 
 
 def enroll_gott_workshops():
-    r = requests.get("https://www.gavilan.edu/staff/tlc/db.php?a=signups")
+    # stupid gav tls broken
-    signups = json.loads(r.text)
+    # r = requests.get("https://www.gavilan.edu/staff/tlc/db.php?a=signups")
+    # signups = json.loads(r.text)
+    
+    signups = json.loads(codecs.open('cache/signups.json','r','utf-8').read())
+    
+    all_staff = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())
 
     # update w/ users.py #1
     all_staff = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())
@@ -1074,10 +1140,16 @@ def enroll_gott_workshops():
         #'GOTT 5: Essentials of Blended Learning (HyFlex)2023-06-25 17:00:00': 17987,
         #'GOTT 1: Intro to Teaching Online with Canvas2023-05-29 17:00:00': 17985,
         #'GOTT 1: Intro to Teaching Online with Canvas2023-08-20 17:00:00': 17994
-        'GOTT 1: Intro to Online Teaching2024-01-02 16:00:00': 19278,
+        #'GOTT 1: Intro to Online Teaching2024-01-02 16:00:00': 19278,
-        'GOTT 2: Intro to Asynchronous Teaching and Learning2024-01-02 16:00:00': 19222,
+        #'GOTT 2: Intro to Asynchronous Teaching and Learning2024-01-02 16:00:00': 19222,
-        'GOTT 5: Essentials of Blended Learning2024-01-02 16:00:00': 19223,
+        #'GOTT 5: Essentials of Blended Learning2024-01-02 16:00:00': 19223,
-        'GOTT 6: Intro to Live Online Teaching and Learning2024-01-14 16:00:00': 19224,
+        #'GOTT 6: Intro to Live Online Teaching and Learning2024-01-14 16:00:00': 19224,
+        '5/28-6/9 GOTT 1: Intro to Teaching Online 2024-05-28 12:00:00': 20567,
+        '5/28-6/21 GOTT 2: Introduction to Asynchronous Teaching and Design2024-05-28 12:00:00': 20575,
+        'GOTT 4: Assessment in Digital Learning2024-06-02 17:00:00': 20600,  # 6/2
+        '6/10-6/23 GOTT 5: Essentials of Blended Learning, Hyflex2024-06-10 12:00:00': 20568,
+        '6/17-6/30 GOTT 6 Introduction to Live Online Teaching and Learning2024-06-17 12:00:00': 20569,
+        'GOTT 1 Intro to Teaching Online AUG242024-07-29 12:00:00': 20603,  # 7/29
     }
     #print(json.dumps(signups,indent=4))
     #print(json.dumps(by_email,indent=4))
@@ -1095,6 +1167,7 @@ def enroll_gott_workshops():
             'gemayo70@yahoo.com': 'pclaros@gavilan.edu',
             'csalvin@gmail.com': 'csalvin@gavilan.edu',
             'efalvey@aol.com': 'efalvey@gavilan.edu',
+            'lorrmay36@mac.com': 'llevy@gavilan.edu',
             }
 
     for wkshp,su_list in signups.items():
@@ -1128,50 +1201,59 @@ def enroll_art_students_live():
     print("done.")
 
 def enroll_orientation_students():
+    
+    # For testing purposes
+    DO_IT = 1
+    
     import localcache2
     ori_shell_id = "19094"    #  2024    #  ""    #   2023 orientation shell  15924   #      2022: "9768"
 
     print("Getting users in orientation shell")
-    users_in_ori_shell = set( \
+    #users_in_ori_shell = set( \
-            [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
+    #        [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])     # api fetch
 
-    for the_semester in ["202430"]:
+    users_in_ori_shell = list(user_ids_in_shell(ori_shell_id))
-        users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
+    
-        print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
+    # single semester
-        print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
+    # users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
+    
+    # double semester (SU + FA)
+    users_to_enroll = users_new_this_2x_semester("202450", "202470")                        #####   USES LOCAL DB
 
-        enroll_us = users_to_enroll.difference(users_in_ori_shell)
+    #print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
+    #print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
 
-        print("\n\nTO ENROLL %s\n" % str(enroll_us))
+    enroll_us = users_to_enroll.difference(users_in_ori_shell)
-        print("%i new users to enroll.\n" % len(enroll_us))
 
-        eee = 0
+    #print("\n\nTO ENROLL %s\n" % str(enroll_us))
-        uuu = 0
+    print(f"{len(enroll_us)} new students to enroll in Orientation shell." )
+
+    eee = 0
+    uuu = 0
         
-        (connection,cursor) = localcache2.db()
+    (connection,cursor) = localcache2.db()
 
-        for j in enroll_us:
+    for j in enroll_us:
-            s = ""
+        s = ""
-            try:
+        try:
-                q = "SELECT name,id FROM canvas.users WHERE id=%s" % j
+            q = "SELECT name,id FROM canvas.users WHERE id=%s" % j
-                #print(q)
+            cursor.execute(q)
-                cursor.execute(q)
+            s = cursor.fetchall()
-                s = cursor.fetchall()
+            if s:
-                if s:
+                s = s[0]
-                    s = s[0]
+                print(" + Enrolling: %s" % s[0])
-                    print(" + Enrolling: %s" % s[0])
+            t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
-                t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
+            data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
-                data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
+                    'enrollment[enrollment_state]': 'active' }
-                        'enrollment[enrollment_state]': 'active' }
+            #print(t)
-                #print(t)
+            #print(data)
-                #print(data)
+            if DO_IT:
-                if 1:
+                r3 = requests.post(t, headers=header, params=data)
-                    r3 = requests.post(t, headers=header, params=data)
+                eee += 1
-                    eee += 1
+                #print(r3.text)
-                    #print(r3.text)
+                time.sleep(0.250)
-                    time.sleep(0.250)
+        except Exception as e:
-            except Exception as e:
+            print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
-                print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
     # return (eee,uuu)
 
 def enroll_o_s_students():
@@ -1325,11 +1407,11 @@ def xlist_cwe():
     # cwe192 get put into another shell
 
 
-    this_sem_190_id = 18424   # they get 190s and 290s
+    this_sem_190_id = 20187   # they get 190s and 290s
-    this_sem_192_id = 18519   # they get 192s
+    this_sem_192_id = 19687   # they get 192s
-    this_sem_term = 181
+    this_sem_term = 184
     
-    get_fresh = 0
+    get_fresh = 1
     sem_courses = getCoursesInTerm(this_sem_term, get_fresh, 0)
 
     for search_string in ['CWE190','WTRM290']:
@@ -1435,12 +1517,15 @@ def teacher_to_many_shells():
 import os, pickle
 
 def create_sandboxes():
-    courses_to_sandbox = [ (19278, ' Sandbox GOTT1 WI24'), 
+    courses_to_sandbox = [ (20567, ' Sandbox GOTT1 SU24'), 
-                           (19222, ' Sandbox GOTT2 WI24'),
+                           (20575, ' Sandbox GOTT2 SU24'),
+                           (20600, ' Sandbox GOTT4 SU24'),
                            (19223, ' Sandbox GOTT5 WI24'),
                            #(19224, ' Sandbox GOTT6 WI24')
                          ]
     filepath = 'cache/sandbox_courses.pkl'
+    
+    report = codecs.open('cache/sandbox_report.txt','a','utf-8')
 
     if os.path.exists(filepath):
         with open(filepath, 'rb') as f:
@@ -1448,14 +1533,12 @@ def create_sandboxes():
     else:
         sandbox_log = []
 
-
-     
-
     for crs_id, label in courses_to_sandbox:
         crs_info = getCourses(crs_id)
-        print(json.dumps(crs_info,indent=2))
+        # print(json.dumps(crs_info,indent=2))
         c_name = crs_info['name']
-        print(f"Students in course {crs_id}: {c_name}" )
+        print(f"\nStudents in course {crs_id}: {c_name}" )
+        report.write(f"\nCourse: {c_name}\n" )
         enrolled = course_enrollment(crs_id)
         for eid,stu in enrolled.items():
             if stu['role'] != 'StudentEnrollment': 
@@ -1464,6 +1547,7 @@ def create_sandboxes():
             u_id = stu['user']['id']
             initials = ''.join([ x[0] for x in u_name.split(" ") ])
             print(f"    id: {stu['user_id']}  ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}")
+            report.write(f"    id: {stu['user_id']}  ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}")
             coursename = f"{initials}{label}"
             if coursename in sandbox_log:
                 print(f"    - Already created: {coursename}")
@@ -1481,6 +1565,7 @@ def create_sandboxes():
                 new_course_response = json.loads(r3.text)
                 id = new_course_response['id']
                 print(f"      created course id {id}")
+                report.write(f"    link: https://ilearn.gavilan.edu/courses/{id}   id: {stu['user_id']}  ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}\n")
 
                 # Add teacher
                 u3 = url + f"/api/v1/courses/{id}/enrollments" 
@@ -2142,11 +2227,11 @@ if __name__ == "__main__":
                 9: ['Simple list of course data, search by sis_id', course_search_by_sis],
                 10: ['Overview of a term', course_term_summary],
                 20: ['process the semester overview output (10)', course_term_summary_2],
-                35: ['Check all courses & their sections in semester', all_semester_course_sanity_check],
+                55: ['Check all courses & their sections in semester', all_semester_course_sanity_check],
 
                 11: ['Enroll ORIENTATION and STEM student shells after catching up database.', enroll_o_s_students],
                 12: ['Enroll stem students', enroll_stem_students_live],
-                13: ['Enroll orientation students (refresh local db)', enroll_orientation_students],
+                13: ['Enroll orientation students (refresh local db first)', enroll_orientation_students],
                 14: ['Enroll ART students', enroll_art_students_live],
 
                 22: ['Get a course info by id',getCourses],
@@ -2162,7 +2247,8 @@ if __name__ == "__main__":
                 31: ['Fine tune term dates and winter session', course_by_depts_terms],
                 #32: ['Cross-list classes', xlist ],
                 #33: ['Cross list helper', eslCrosslister],
-                34: ['Cross list a semester from file', semester_cross_lister],
+                34: ['Cross list a semester from argos export file', semester_cross_lister],
+                35: ['Cross list from manually created file', do_manual_xlist],
                 36: ['Quick course list', quick_sem_course_list ],
                 37: ['Cross list CWE courses', xlist_cwe], 
                 38: ['Create calendar event', create_calendar_event],

localcache2.py

@@ -124,7 +124,14 @@ def all_gav_employees():
 
 '''
 
-
+def course_from_id(id):
+    q = f"SELECT * FROM canvas.courses c WHERE c.id={id}"
+    (connection,cursor) = db()
+    cursor.execute(q, None)  # execute query with optional parameters
+    row = cursor.fetchone()  # fetch a single row
+    if row:
+        # convert row to dict using column names as keys
+        return dict(zip([desc[0] for desc in cursor.description], row))
 
 def teachers_by_term(TERM = "202430"):
     q = f"""SELECT c.id, c.name, c.course_code, c.sis_source_id, c.created_at, c.start_at, c.workflow_state, e.last_attended_at, 
@@ -170,6 +177,15 @@ ORDER BY c.sis_source_id, wp.title;"""
     return all
 
 
+def user_ids_in_shell(shellid):
+    q = f"""select e.user_id from canvas.enrollments e
+where e.course_id = {shellid} and e.type='StudentEnrollment' and e.workflow_state='active';"""
+    (connection,cursor) = db()
+    cursor.execute(q)
+    students = {str(row[0]) for row in cursor.fetchall()}
+    print(f"{len(students)} students currently in shell id {shellid}.")
+    return students
+
 
 def users_new_this_semester(sem=''):
     if not len(sem):
@@ -202,7 +218,7 @@ ORDER BY num DESC, u.sortable_name""" % (where1,where2)
 
     (connection,cursor) = db()
     cursor.execute(q)
-    #s = cursor.fetchall()
+    #s = cursor.fetchall()     # TODO see below
     #if s:
     for u in cursor:
         users_to_enroll.add(str(u[0]))
@@ -211,8 +227,46 @@ ORDER BY num DESC, u.sortable_name""" % (where1,where2)
     print(users_to_enroll)
     return users_to_enroll
 
-def all_sem_courses_teachers():
+
-    SEM = "202430"
+# when registrations opens for SUMMER+FALL, get new students from both semesters combined.
+def users_new_this_2x_semester(sem1='', sem2=''):
+    if not len(sem1) or (not len(sem2)):
+        print("Need 2 semesters")
+        return 0
+
+    where1 = f"(c.sis_source_id LIKE '{sem1}-%%' or c.sis_source_id LIKE '{sem2}-%%')"
+    where2 = f"(c.sis_source_id NOT LIKE '{sem1}-%%' and c.sis_source_id NOT LIKE '{sem2}-%%')"
+
+    q = """SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e
+JOIN canvas.users AS u ON e.user_id=u.id
+JOIN canvas.courses AS c ON e.course_id=c.id
+WHERE %s
+AND e.workflow_state='active'
+AND e.type='StudentEnrollment'
+AND u.id NOT IN (
+	SELECT u.id FROM canvas.enrollments AS e
+	JOIN canvas.users AS u ON e.user_id=u.id
+	JOIN canvas.courses AS c ON e.course_id=c.id
+	WHERE %s
+	AND e.workflow_state='active'
+	AND e.type='StudentEnrollment'
+	GROUP BY u.id
+)
+GROUP BY u.id
+ORDER BY num DESC, u.sortable_name""" % (where1,where2)
+
+    (connection,cursor) = db()
+    cursor.execute(q)
+    users_to_enroll = {str(row[0]) for row in cursor.fetchall()}
+    print("%i new users this semester." % len(users_to_enroll))
+    return users_to_enroll
+
+
+
+
+
+
+def all_sem_courses_teachers(SEM="202450"):
     q = f"""SELECT c.id, c.name, c.course_code, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id  FROM canvas.courses AS c
 JOIN canvas.enrollments AS e ON e.course_id=c.id
 JOIN canvas.users AS u ON u.id=e.user_id
@@ -231,6 +285,27 @@ ORDER BY u.sortable_name;"""
 
 
 
+
+def all_2x_sem_courses_teachers(sem1, sem2):
+    q = f"""SELECT c.id, c.name, c.course_code, s.type, s.crn, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id  FROM canvas.courses AS c
+JOIN canvas.enrollments AS e ON e.course_id=c.id
+JOIN canvas.users AS u ON u.id=e.user_id
+JOIN canvas.pseudonyms AS p ON p.user_id=u.id
+JOIN canvas.schedule as s on c.id=s.canvascourse 
+WHERE (c.sis_source_id LIKE '{sem1}-%' or c.sis_source_id LIKE '{sem2}-%')
+AND NOT c.workflow_state='deleted'
+AND e.type='TeacherEnrollment'
+ORDER BY u.sortable_name;"""
+    (connection,cursor) = db()
+    cursor.execute(q)
+    courses = cursor.fetchall()
+    print(courses)
+    return courses
+
+
+
+
+
 def create_schedule_table_if_not_exists():
     conn,cur = db()
     

tasks.py

@@ -1136,14 +1136,14 @@ def print_a_calendar():
     generate_custom_calendar(year, l_semesters)
 
 
-
+# task list calendar for a semester
 def word_calendar():
     from docx import Document
     from docx.shared import Inches
     import datetime
 
     # Define the start date of semester
-    start_date = datetime.date(2024, 1, 29)
+    start_date = datetime.date(2024, 7, 1)
 
     # Prepare a list of 18 weeks beginning from the start date
     dates = [start_date + datetime.timedelta(weeks=x) for x in range(18)]
@@ -1168,6 +1168,40 @@ def word_calendar():
     # Save the document
     doc.save('cache/tasks_schedule.docx')
 
+# more general purpose
+def word_calendar_v2():
+    from docx import Document
+    from docx.shared import Inches
+    import datetime
+
+    # Define the start date of semester
+    start_date = datetime.date(2024, 7, 1)
+
+    # Prepare a list of 18 weeks beginning from the start date
+    dates = [start_date + datetime.timedelta(weeks=x) for x in range(40)]
+
+    # Initialize an instance of a word document
+    doc = Document()
+    table = doc.add_table(rows=1, cols=3)
+
+    # Set the headers
+    hdr_cells = table.rows[0].cells
+    hdr_cells[0].text = 'Week of'
+    hdr_cells[1].text = 'Events'
+    hdr_cells[2].text = 'Notes'
+
+    # Iterate through the list of dates
+    for i, date in enumerate(dates):
+        end_date = date + datetime.timedelta(days=6)  # Calculate the end date
+        cells = table.add_row().cells
+        #cells[0].text = str(i+1)
+        cells[0].text = f"{date.strftime('%B %d')} - {end_date.strftime('%B %d')}"
+        cells[1].text = ''
+        cells[2].text = ''
+
+    # Save the document
+    doc.save('cache/weekly_calendar.docx')
+
 
 # TODO some weird hour offset issue w/ these activities
 
@@ -1271,8 +1305,9 @@ if __name__ == "__main__":
                 11: ['list auth', list_auth],
                 12: ['update auth', update_auth],
                 13: ['print a calendar', print_a_calendar],
-                14: ['create a week calendar in word', word_calendar],
+                14: ['create a week calendar in word (semester)', word_calendar],
-                15: ['create GOTT certificates', certificates_gott_build],
+                15: ['create a week calendar in word (general purpose)', word_calendar_v2],
+                16: ['create GOTT certificates', certificates_gott_build],
                 20: ['build_quiz', build_quiz],
                 21: ['certificates_gott_build, certificates_gott_build']
               }

useful queries.sql

@@ -16,7 +16,68 @@ INSERT INTO canvas.schedule (canvascourse, crn, code, units, teacher, start,"end
 VALUES (9651, '40724', 'SPAN 1A', '5.000', 'David G  Perez', '1-27', '5-22', 'in-person', 'HU 105', 'Gilroy', 'Midday', 30, 26, '202030');
 
 
+-- courses in a semester, good for finding sections to merge
+select c.id, c.course_code, c.sis_source_id, e.role_id, s.start, s.type, u.sortable_name from canvas.courses c
+join canvas.enrollments e on c.id=e.course_id
+join canvas.users u on u.id=e.user_id
+full outer join canvas.schedule s on c.id=s.canvascourse
+where c.sis_source_id like '202370-%' and e.role_id=4
+order by c.course_code, u.sortable_name;
 
+-- courses in a semester, good for finding sections to merge, no users
+select c.id, c.course_code, c.sis_source_id,  s.start, s.type from canvas.courses c
+full outer join canvas.schedule s on c.id=s.canvascourse
+where c.sis_source_id like '202370-%' 
+order by c.course_code;
+
+
+select c.id, c.course_code, c.sis_source_id, e.role_id, u.sortable_name from canvas.courses c
+join canvas.enrollments e on c.id=e.course_id
+join canvas.users u on u.id=e.user_id
+where c.sis_source_id like '202470-%'
+order by c.course_code, u.sortable_name;
+
+
+
+-- all teachers in (2) semester
+SELECT distinct c.id, c.name, c.course_code, s.type, s.crn, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id  FROM canvas.courses AS c
+JOIN canvas.enrollments AS e ON e.course_id=c.id
+JOIN canvas.users AS u ON u.id=e.user_id
+JOIN canvas.pseudonyms AS p ON p.user_id=u.id
+JOIN canvas.schedule as s on c.id=s.canvascourse 
+WHERE (c.sis_source_id LIKE '202450-%' or c.sis_source_id LIKE '202470-%')
+AND NOT c.workflow_state='deleted' AND e.type='TeacherEnrollment'
+ORDER BY u.sortable_name;
+
+
+
+
+
+-- teachers of online/onlinelive/hybrid in a semester and their emails 
+select distinct u.sortable_name, LOWER(cc.path) as email from canvas.courses c
+join canvas.enrollments e on c.id=e.course_id
+join canvas.users u on u.id=e.user_id
+join canvas.communication_channels cc on u.id=cc.user_id
+full outer join canvas.schedule s on c.id=s.canvascourse
+where (s.type='online' or s.type='hybrid' or s.type='online line')
+	and c.sis_source_id like '202450-%' 
+	and cc.path_type='email' 
+	and not cc.path like '%noemail%'
+	and not cc.path='sstaff@gavilan.edu'
+order by u.sortable_name;
+
+
+-- for outlook
+select string_agg(distinct LOWER(cc.path), '; ') from canvas.courses c
+join canvas.enrollments e on c.id=e.course_id
+join canvas.users u on u.id=e.user_id
+join canvas.communication_channels cc on u.id=cc.user_id
+full outer join canvas.schedule s on c.id=s.canvascourse
+where (s.type='online' or s.type='hybrid' or s.type='online line')
+	and c.sis_source_id like '202450-%' 
+	and cc.path_type='email' 
+	and not cc.path like '%noemail%'
+	and not cc.path='sstaff@gavilan.edu';
 
 --
 --
@@ -92,4 +153,53 @@ join canvas.enrollments e on u.id=e.user_id
 join canvas.courses c on e.course_id=c.id
 join canvas.schedule s on c.id=s.canvascourse
 where s.sem='202430' and e.workflow_state='active' and e.type='StudentEnrollment'
-order by u.sortable_name, c.course_code;
+order by u.sortable_name, c.course_code;
+
+
+
+-- users in orientation shell
+
+select e.user_id from canvas.enrollments e
+where e.course_id = 19094 and e.type='StudentEnrollment' and e.workflow_state='active';
+
+
+-- new users this (single) semester
+SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e
+JOIN canvas.users AS u ON e.user_id=u.id
+JOIN canvas.courses AS c ON e.course_id=c.id
+WHERE c.sis_source_id LIKE '202450-%%'
+AND e.workflow_state='active'
+AND e.type='StudentEnrollment'
+AND u.id NOT IN (
+	SELECT u.id FROM canvas.enrollments AS e
+	JOIN canvas.users AS u ON e.user_id=u.id
+	JOIN canvas.courses AS c ON e.course_id=c.id
+	WHERE c.sis_source_id NOT LIKE '202450-%%'
+	AND e.workflow_state='active'
+	AND e.type='StudentEnrollment'
+	GROUP BY u.id
+)
+GROUP BY u.id
+ORDER BY num DESC, u.sortable_name
+
+
+
+-- new users this (summer+fall) semester
+SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e
+JOIN canvas.users AS u ON e.user_id=u.id
+JOIN canvas.courses AS c ON e.course_id=c.id
+WHERE (c.sis_source_id LIKE '202450-%%' or c.sis_source_id LIKE '202470-%%')
+AND e.workflow_state='active'
+AND e.type='StudentEnrollment'
+AND u.id NOT IN (
+	SELECT u.id FROM canvas.enrollments AS e
+	JOIN canvas.users AS u ON e.user_id=u.id
+	JOIN canvas.courses AS c ON e.course_id=c.id
+	WHERE (c.sis_source_id NOT LIKE '202450-%%' and c.sis_source_id NOT LIKE '202470-%%')
+	AND e.workflow_state='active'
+	AND e.type='StudentEnrollment'
+	GROUP BY u.id
+)
+GROUP BY u.id
+ORDER BY num DESC, u.sortable_name
+

users.py

@@ -11,7 +11,10 @@ from collections import defaultdict
 from pipelines import fetch, fetch_stream, getSemesterSchedule, header, url, FetchError, put_file
 from courses import course_enrollment, users_in_semester
 from localcache import users_this_semester_db, unwanted_req_paths, timeblock_24hr_from_dt, dt_from_24hr_timeblock
-from localcache import teachers_courses_semester
+from localcache import teachers_courses_semester, course_mode, sem_schedule
+from localcache2 import all_2x_sem_courses_teachers, all_sem_courses_teachers
+from pipelines import dean, dean_names
+
 from util import dept_from_name, most_common_item
 from os.path import exists, getmtime
 
@@ -2234,10 +2237,6 @@ def compare_db_tables():
 
 
 def training_find_goos():
-    from localcache2 import all_sem_courses_teachers
-    from localcache import course_mode
-    from localcache import sem_schedule
-    from pipelines import dean
     from openpyxl import Workbook, load_workbook
     from openpyxl.chart import BarChart, Series, Reference
     from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
@@ -2268,19 +2267,21 @@ def training_find_goos():
         print()
 
 def cross_ref_training():
-    from localcache2 import all_sem_courses_teachers
-    from localcache import course_mode
-    from localcache import sem_schedule
-    from pipelines import dean, dean_names
     from openpyxl import Workbook, load_workbook
     from openpyxl.chart import BarChart, Series, Reference
     from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
-    wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
+    wb = load_workbook("C:/Users/phowell/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
     print(wb.sheetnames)
 
     # report for email
     report = codecs.open('cache/gott_report.txt','w','utf-8')
 
+    # update local list of teachers from ilearn?
+    RELOAD_TEACHERS = 0
+    if RELOAD_TEACHERS:
+        teacherRolesUpdateCache()
+    
+    # TODO inefficient but just read it again
     all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())
 
     records = {}
@@ -2303,18 +2304,18 @@ def cross_ref_training():
     alldepts = set()
 
 
-    courses = all_sem_courses_teachers()
+    courses = all_2x_sem_courses_teachers('202450', '202470')        # all_sem_courses_teachers()   
     for c in courses:
         print(c)
-        goo = c[6]
+        goo = c[8]
-        crn = c[2].split(' ')[-1].split('/')[0]
+        crn = c[4]
         name = c[2]
-        teacher = c[4]
+        teacher = c[5]
-        ctype = course_mode(crn,'sp24')
+        ctype = c[3]
         dept1 = re.search(r'([A-Z]+)(\d+)',c[2].split(' ')[0]).group(1)
         alldepts.add(dept1)
         d = list(c)
-        d.append(ctype)
+        #d.append(ctype)
         if not ctype:
             print(f"not finding mode for {name}")
             continue
@@ -2325,7 +2326,7 @@ def cross_ref_training():
     alldepts = list(alldepts)
     alldepts.sort()
 
-    sheet = wb.create_sheet("Spring 2024 Summary")
+    sheet = wb.create_sheet("New Summary")
     r = 1
 
     deptfont = Font(bold=True)
@@ -2354,7 +2355,7 @@ def cross_ref_training():
                     waived = 0
                     sects = teachers[t]
                     print(f"Sections for {t}: {sects}")
-                    goo = sects[0][6]
+                    goo = sects[0][8]
                     print(t)
                     sheet.cell(row=r, column=1).value = f"{t}"
                     sheet.cell(row=r, column=2).value = f"{goo}"
@@ -2425,12 +2426,12 @@ def cross_ref_training():
                         print(f"   GOTT abc self paced")
                         r += 1
                     for s in sects:
-                        sheet.cell(row=r, column=2).value = f"{s[7]}"
+                        sheet.cell(row=r, column=2).value = f"{s[3]}"
                         sheet.cell(row=r, column=3).value = f"{s[1]}"
                         r += 1
 
                         if (not completed) and (not waived):
-                            report.write(f"\t\t{s[7]}\t{s[1]}\n")
+                            report.write(f"\t\t{s[3]}\t{s[1]}\n")
                     if (not completed) and (not waived):
                         report.write(f"\n")
     
@@ -2441,7 +2442,7 @@ def cross_ref_training():
     sheet.column_dimensions['B'].width = 30
     sheet.column_dimensions['C'].width = 75
     formatted_date = datetime.datetime.now().strftime('%Y%m%d')
-    wb.save(f"C:/Users/peter/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx")
+    wb.save(f"C:/Users/phowell/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx")
 
 def cross_ref_training_withcsv():
     from localcache2 import all_sem_courses_teachers

util.py

@@ -105,6 +105,13 @@ def dept_from_name(n):
     print(("Couldn't find dept from: " + n))
     return ''
     
+# ENGL250 returns 250    
+def num_from_name(n):
+    m = re.search('^([a-zA-Z]+)\s?([\d\/]+[A-Z]?)',n)
+    if m: return m.group(2)
+    print(("Couldn't find num from: " + n))
+    return ''
+    
 def most_common_item(li):
     d = defaultdict(int)
     for x in li: