diff --git a/content.py b/content.py index 2f43590..dceb58b 100644 --- a/content.py +++ b/content.py @@ -1,6 +1,7 @@ #saved_titles = json.loads( codecs.open('cache/saved_youtube_titles.json','r','utf-8').read() ) +from calendar import FRIDAY import requests, codecs, os, re, json, sys, pypandoc import webbrowser, bs4, trafilatura, pickle, tomd, checker import html2markdown as h2m @@ -614,6 +615,102 @@ def update_page(): fixed_page = checker.safe_html(mypage['body']) upload_page(course_num,chosen_url,fixed_page) +# given dict of file info (from files api), construct an img tag that works in a page +#def file_to_img_tag(f, alt, course, soup): +# #tag = f"\"{f['filename']}\"" +# return T + + +def html_file_to_page(filename, course, tags): + + try: + soup = bs4.BeautifulSoup(codecs.open(filename,'r', 'utf-8').read(), 'html.parser') + except Exception as e: + print(f"Exception on {filename}: {e}") + return + img_tags = soup.find_all('img') + + result = {'title': soup.title.text if soup.title else ''} + result['title'].strip() + + for img in img_tags: + src = img['src'] + try: + alt = img['alt'] + except: + alt = src + orig_filename = os.path.basename(src) + if orig_filename in tags: + T = soup.new_tag(name='img', src=f"https://ilearn.gavilan.edu/courses/{course}/files/{tags[orig_filename]['id']}/preview") + T['id'] = tags[orig_filename]['id'] + T['alt'] = alt + T['data-api-endpoint'] = f"https://ilearn.gavilan.edu/api/v1/courses/{course}/files/{tags[orig_filename]['id']}" + T['data-api-returntype'] = "File" + img.replace_with(T) + print( f" replaced image: {src} alt: {alt}") + else: + print( f" couldn't find replacement image: {src} alt: {alt}") + outfile = codecs.open(filename+"_mod.html", 'w', 'utf-8') + outfile.write( soup.prettify() ) + outfile.close() + result['body'] = ''.join(map(str, soup.body.contents)) if soup.body else '' + return result + +def create_new_page(course_id, title, body): + print(f"Creating page: {title}, length: {len(body)}") + request = f"{url}/api/v1/courses/{course_id}/pages" + print(request) + data = { 'wiki_page[title]': title, 'wiki_page[body]': body } + r3 = requests.post(request, headers=header, data=data) + try: + result = json.loads(r3.text) + print( f" + ok: {result['url']}") + except: + print(" - problem creating page?") + +# Given a folder full of html pages and their linked images, create Canvas PAGES of them +def make_pages_from_folder(folder='cache/csis6/', course = '20558'): + if 0: + request = f"{url}/api/v1/courses/{course}/files" + print("Fetching course files") + files = fetch(request) + + tempfile = codecs.open('cache/csis6filelist.json','w','utf-8') + tempfile.write(json.dumps(files)) + tempfile.close() + + if 1: + files = json.loads( codecs.open('cache/csis6filelist.json', 'r', 'utf-8').read()) + + + + course_files = {f['filename']: f for f in files} + tags = {} + for f in files: + if f['filename'].lower().endswith('.jpg') or f['filename'].lower().endswith('.png'): + tags[f['filename']] = f + + + contents = os.listdir(folder) + contents = ['welcome.html','welcome2.html', 'welcome3.html'] + print(contents) + for f in contents: + m = re.search(r'^(.*)\.(html?)$', f) + if m: + print(f"html file: {m.group(1)}, extension: {m.group(2)}") + newpage = html_file_to_page(folder+f, course, tags) + create_new_page(course, newpage['title'], newpage['body']) + else: + m = re.search(r'^(.*)\.(.*)$', f) + if m: + print(f"other file: {m.group(1)}, extension: {m.group(2)}") + else: + print(f"unknown file: {f}") + + + + # Given course, page url, and new content, upload the new revision of a page def upload_page(course_num,pageurl,new_content): print("Repaired page:\n\n") @@ -1385,6 +1482,7 @@ if __name__ == "__main__": 15: ['test priority', test_priority], 16: ['test embed', test_embed], 17: ['repair ezproxy links', repairy_ezproxy_links], + 18: ['create pages from html files', make_pages_from_folder], } if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]): diff --git a/courses.py b/courses.py index 9c3c99d..6bce300 100644 --- a/courses.py +++ b/courses.py @@ -4,11 +4,11 @@ from datetime import datetime import pytz from dateutil import parser from datetime import datetime -from util import print_table, int_or_zero, float_or_zero +from util import print_table, int_or_zero, float_or_zero, dept_from_name, num_from_name from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem from pipelines import sems -from localcache import db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload -from localcache2 import users_new_this_semester +from localcache import course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload +from localcache2 import db, users_new_this_semester, users_new_this_2x_semester, course_from_id, user_ids_in_shell from collections import defaultdict @@ -545,8 +545,8 @@ def all_equal2(iterator): 177 2023 Winter """ def semester_cross_lister(): - sem = "sp24" - term = 181 + sem = "fa24" + term = 184 # fa24=184 xlist_filename = f"cache/{sem}_crosslist.csv" checkfile = codecs.open('cache/xlist_check.html','w','utf-8') checkfile.write('\n') @@ -558,7 +558,7 @@ def semester_cross_lister(): crn_to_canvasname = {} crn_to_canvascode = {} - get_fresh = 0 + get_fresh = 1 c = getCoursesInTerm(term,get_fresh,0) for C in c: @@ -629,13 +629,74 @@ def semester_cross_lister(): xlist_ii(target_section[3],host_id,new_name,new_code) #pass +def do_manual_xlist(): + infile = [ x.strip() for x in open('cache/fa24_manual_crosslist.txt','r').readlines() ] + for L in infile: + print(L) + paraL,host = L.split(' -> ') + para_list = paraL.split(',') + print(host) + print(para_list) + xlist(host, para_list) + +# Crosslist given 2 ids, computing the new name and code +def xlist(host_id, parasite_list): + host_info = course_from_id(host_id) + host_info['crn'] = host_info['sis_source_id'][7:] + host_info['dept'] = dept_from_name( host_info['course_code'] ) + host_info['num'] = num_from_name(host_info['course_code'] ) + host_info['bare_name'] = ' '.join(host_info['name'].split(' ')[1:-1]) # name without course code or crn + + sem = host_info['course_code'].split(' ')[1] + + para_info_list = [ course_from_id(x) for x in parasite_list ] + for p in para_info_list: + p['crn'] = p['sis_source_id'][7:] + p['dept'] = dept_from_name(p['course_code'] ) + p['num'] = num_from_name(p['course_code'] ) + p['bare_name'] = ' '.join(p['name'].split(' ')[1:-1]) # name without course code or crn + all = para_info_list.copy() + all.append(host_info) + + # determine new name and code + sects = [ z['crn'] for z in all ] + sects.sort() + nic = numbers_in_common(sects) + new_sec = combined_name(nic,sects) + + # same dept? + depts_list = [ z['dept'] for z in all ] + nums_list = list(set([ z['num'] for z in all ])) + if all_equal2(depts_list): + depts = depts_list[0] + nums_list.sort() + nums = '/'.join(nums_list) + else: + depts = list(set(depts_list)) + depts.sort() + depts = '/'.join(depts ) + nums = all[0]['num'] + + new_name = f"{depts}{nums} {all[0]['bare_name']} {new_sec}" + #new_name = by_group[y][0][4][0:-5] + new_sec + new_code = f"{depts}{nums} {sem.upper()} {new_sec}" + #new_code = by_group[y][0][5][0:-5] + new_sec + print(f"New name: {new_name}") + print(f"New code: {new_code}") + print(sects) + + for target_section in para_info_list: + xlist_ii(target_section['id'],host_id,new_name,new_code) + + + # Perform an actual cross-list, given 2 id numbers, new name and code def xlist_ii(parasite_id,host_id,new_name,new_code): print("Parasite id: ",parasite_id," Host id: ", host_id) print("New name: ", new_name) print("New code: ", new_code) xyz = 'y' - #xyz = input("Perform cross list? Enter for yes, n for no: ") + #xyz = input("Perform cross list? Enter y for yes, n for no: ") if xyz != 'n': uu = url + '/api/v1/courses/%s/sections' % parasite_id c_sect = fetch(uu) @@ -843,7 +904,7 @@ def eslCrosslister(): if not c: print("Didn't catch: "+ str(combos[i])) -def xlist(parasite='', host=''): # section id , new course id +def xlist_iii(parasite='', host=''): # section id , new course id host = host or input("ID number of the HOSTING COURSE? ") if not parasite: @@ -894,7 +955,7 @@ def enroll_id_list_to_shell(id_list, shell_id, v=0): for j in enroll_us: try: - q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j + q = "SELECT name,id FROM canvas.users u WHERE u.id=%s" % j cursor.execute(q) s = cursor.fetchall() if s: @@ -1057,8 +1118,13 @@ def enroll_bulk_students_bydept(course_id, depts, the_term="172", cautious=1): def enroll_gott_workshops(): - r = requests.get("https://www.gavilan.edu/staff/tlc/db.php?a=signups") - signups = json.loads(r.text) + # stupid gav tls broken + # r = requests.get("https://www.gavilan.edu/staff/tlc/db.php?a=signups") + # signups = json.loads(r.text) + + signups = json.loads(codecs.open('cache/signups.json','r','utf-8').read()) + + all_staff = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) # update w/ users.py #1 all_staff = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) @@ -1074,10 +1140,16 @@ def enroll_gott_workshops(): #'GOTT 5: Essentials of Blended Learning (HyFlex)2023-06-25 17:00:00': 17987, #'GOTT 1: Intro to Teaching Online with Canvas2023-05-29 17:00:00': 17985, #'GOTT 1: Intro to Teaching Online with Canvas2023-08-20 17:00:00': 17994 - 'GOTT 1: Intro to Online Teaching2024-01-02 16:00:00': 19278, - 'GOTT 2: Intro to Asynchronous Teaching and Learning2024-01-02 16:00:00': 19222, - 'GOTT 5: Essentials of Blended Learning2024-01-02 16:00:00': 19223, - 'GOTT 6: Intro to Live Online Teaching and Learning2024-01-14 16:00:00': 19224, + #'GOTT 1: Intro to Online Teaching2024-01-02 16:00:00': 19278, + #'GOTT 2: Intro to Asynchronous Teaching and Learning2024-01-02 16:00:00': 19222, + #'GOTT 5: Essentials of Blended Learning2024-01-02 16:00:00': 19223, + #'GOTT 6: Intro to Live Online Teaching and Learning2024-01-14 16:00:00': 19224, + '5/28-6/9 GOTT 1: Intro to Teaching Online 2024-05-28 12:00:00': 20567, + '5/28-6/21 GOTT 2: Introduction to Asynchronous Teaching and Design2024-05-28 12:00:00': 20575, + 'GOTT 4: Assessment in Digital Learning2024-06-02 17:00:00': 20600, # 6/2 + '6/10-6/23 GOTT 5: Essentials of Blended Learning, Hyflex2024-06-10 12:00:00': 20568, + '6/17-6/30 GOTT 6 Introduction to Live Online Teaching and Learning2024-06-17 12:00:00': 20569, + 'GOTT 1 Intro to Teaching Online AUG242024-07-29 12:00:00': 20603, # 7/29 } #print(json.dumps(signups,indent=4)) #print(json.dumps(by_email,indent=4)) @@ -1095,6 +1167,7 @@ def enroll_gott_workshops(): 'gemayo70@yahoo.com': 'pclaros@gavilan.edu', 'csalvin@gmail.com': 'csalvin@gavilan.edu', 'efalvey@aol.com': 'efalvey@gavilan.edu', + 'lorrmay36@mac.com': 'llevy@gavilan.edu', } for wkshp,su_list in signups.items(): @@ -1128,50 +1201,59 @@ def enroll_art_students_live(): print("done.") def enroll_orientation_students(): + + # For testing purposes + DO_IT = 1 + import localcache2 ori_shell_id = "19094" # 2024 # "" # 2023 orientation shell 15924 # 2022: "9768" print("Getting users in orientation shell") - users_in_ori_shell = set( \ - [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ]) + #users_in_ori_shell = set( \ + # [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ]) # api fetch - for the_semester in ["202430"]: - users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB - print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll)) - print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell)) + users_in_ori_shell = list(user_ids_in_shell(ori_shell_id)) + + # single semester + # users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB + + # double semester (SU + FA) + users_to_enroll = users_new_this_2x_semester("202450", "202470") ##### USES LOCAL DB - enroll_us = users_to_enroll.difference(users_in_ori_shell) + #print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll)) + #print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell)) - print("\n\nTO ENROLL %s\n" % str(enroll_us)) - print("%i new users to enroll.\n" % len(enroll_us)) + enroll_us = users_to_enroll.difference(users_in_ori_shell) - eee = 0 - uuu = 0 + #print("\n\nTO ENROLL %s\n" % str(enroll_us)) + print(f"{len(enroll_us)} new students to enroll in Orientation shell." ) + + eee = 0 + uuu = 0 - (connection,cursor) = localcache2.db() + (connection,cursor) = localcache2.db() - for j in enroll_us: - s = "" - try: - q = "SELECT name,id FROM canvas.users WHERE id=%s" % j - #print(q) - cursor.execute(q) - s = cursor.fetchall() - if s: - s = s[0] - print(" + Enrolling: %s" % s[0]) - t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id - data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment', - 'enrollment[enrollment_state]': 'active' } - #print(t) - #print(data) - if 1: - r3 = requests.post(t, headers=header, params=data) - eee += 1 - #print(r3.text) - time.sleep(0.250) - except Exception as e: - print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e))) + for j in enroll_us: + s = "" + try: + q = "SELECT name,id FROM canvas.users WHERE id=%s" % j + cursor.execute(q) + s = cursor.fetchall() + if s: + s = s[0] + print(" + Enrolling: %s" % s[0]) + t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id + data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment', + 'enrollment[enrollment_state]': 'active' } + #print(t) + #print(data) + if DO_IT: + r3 = requests.post(t, headers=header, params=data) + eee += 1 + #print(r3.text) + time.sleep(0.250) + except Exception as e: + print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e))) # return (eee,uuu) def enroll_o_s_students(): @@ -1325,11 +1407,11 @@ def xlist_cwe(): # cwe192 get put into another shell - this_sem_190_id = 18424 # they get 190s and 290s - this_sem_192_id = 18519 # they get 192s - this_sem_term = 181 + this_sem_190_id = 20187 # they get 190s and 290s + this_sem_192_id = 19687 # they get 192s + this_sem_term = 184 - get_fresh = 0 + get_fresh = 1 sem_courses = getCoursesInTerm(this_sem_term, get_fresh, 0) for search_string in ['CWE190','WTRM290']: @@ -1435,12 +1517,15 @@ def teacher_to_many_shells(): import os, pickle def create_sandboxes(): - courses_to_sandbox = [ (19278, ' Sandbox GOTT1 WI24'), - (19222, ' Sandbox GOTT2 WI24'), + courses_to_sandbox = [ (20567, ' Sandbox GOTT1 SU24'), + (20575, ' Sandbox GOTT2 SU24'), + (20600, ' Sandbox GOTT4 SU24'), (19223, ' Sandbox GOTT5 WI24'), #(19224, ' Sandbox GOTT6 WI24') ] filepath = 'cache/sandbox_courses.pkl' + + report = codecs.open('cache/sandbox_report.txt','a','utf-8') if os.path.exists(filepath): with open(filepath, 'rb') as f: @@ -1448,14 +1533,12 @@ def create_sandboxes(): else: sandbox_log = [] - - - for crs_id, label in courses_to_sandbox: crs_info = getCourses(crs_id) - print(json.dumps(crs_info,indent=2)) + # print(json.dumps(crs_info,indent=2)) c_name = crs_info['name'] - print(f"Students in course {crs_id}: {c_name}" ) + print(f"\nStudents in course {crs_id}: {c_name}" ) + report.write(f"\nCourse: {c_name}\n" ) enrolled = course_enrollment(crs_id) for eid,stu in enrolled.items(): if stu['role'] != 'StudentEnrollment': @@ -1464,6 +1547,7 @@ def create_sandboxes(): u_id = stu['user']['id'] initials = ''.join([ x[0] for x in u_name.split(" ") ]) print(f" id: {stu['user_id']} ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}") + report.write(f" id: {stu['user_id']} ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}") coursename = f"{initials}{label}" if coursename in sandbox_log: print(f" - Already created: {coursename}") @@ -1481,6 +1565,7 @@ def create_sandboxes(): new_course_response = json.loads(r3.text) id = new_course_response['id'] print(f" created course id {id}") + report.write(f" link: https://ilearn.gavilan.edu/courses/{id} id: {stu['user_id']} ititials: {initials} name: {stu['user']['short_name']} role: {stu['role']}\n") # Add teacher u3 = url + f"/api/v1/courses/{id}/enrollments" @@ -2142,11 +2227,11 @@ if __name__ == "__main__": 9: ['Simple list of course data, search by sis_id', course_search_by_sis], 10: ['Overview of a term', course_term_summary], 20: ['process the semester overview output (10)', course_term_summary_2], - 35: ['Check all courses & their sections in semester', all_semester_course_sanity_check], + 55: ['Check all courses & their sections in semester', all_semester_course_sanity_check], 11: ['Enroll ORIENTATION and STEM student shells after catching up database.', enroll_o_s_students], 12: ['Enroll stem students', enroll_stem_students_live], - 13: ['Enroll orientation students (refresh local db)', enroll_orientation_students], + 13: ['Enroll orientation students (refresh local db first)', enroll_orientation_students], 14: ['Enroll ART students', enroll_art_students_live], 22: ['Get a course info by id',getCourses], @@ -2162,7 +2247,8 @@ if __name__ == "__main__": 31: ['Fine tune term dates and winter session', course_by_depts_terms], #32: ['Cross-list classes', xlist ], #33: ['Cross list helper', eslCrosslister], - 34: ['Cross list a semester from file', semester_cross_lister], + 34: ['Cross list a semester from argos export file', semester_cross_lister], + 35: ['Cross list from manually created file', do_manual_xlist], 36: ['Quick course list', quick_sem_course_list ], 37: ['Cross list CWE courses', xlist_cwe], 38: ['Create calendar event', create_calendar_event], diff --git a/localcache2.py b/localcache2.py index 0507821..b726df7 100644 --- a/localcache2.py +++ b/localcache2.py @@ -124,7 +124,14 @@ def all_gav_employees(): ''' - +def course_from_id(id): + q = f"SELECT * FROM canvas.courses c WHERE c.id={id}" + (connection,cursor) = db() + cursor.execute(q, None) # execute query with optional parameters + row = cursor.fetchone() # fetch a single row + if row: + # convert row to dict using column names as keys + return dict(zip([desc[0] for desc in cursor.description], row)) def teachers_by_term(TERM = "202430"): q = f"""SELECT c.id, c.name, c.course_code, c.sis_source_id, c.created_at, c.start_at, c.workflow_state, e.last_attended_at, @@ -170,6 +177,15 @@ ORDER BY c.sis_source_id, wp.title;""" return all +def user_ids_in_shell(shellid): + q = f"""select e.user_id from canvas.enrollments e +where e.course_id = {shellid} and e.type='StudentEnrollment' and e.workflow_state='active';""" + (connection,cursor) = db() + cursor.execute(q) + students = {str(row[0]) for row in cursor.fetchall()} + print(f"{len(students)} students currently in shell id {shellid}.") + return students + def users_new_this_semester(sem=''): if not len(sem): @@ -202,7 +218,7 @@ ORDER BY num DESC, u.sortable_name""" % (where1,where2) (connection,cursor) = db() cursor.execute(q) - #s = cursor.fetchall() + #s = cursor.fetchall() # TODO see below #if s: for u in cursor: users_to_enroll.add(str(u[0])) @@ -211,8 +227,46 @@ ORDER BY num DESC, u.sortable_name""" % (where1,where2) print(users_to_enroll) return users_to_enroll -def all_sem_courses_teachers(): - SEM = "202430" + +# when registrations opens for SUMMER+FALL, get new students from both semesters combined. +def users_new_this_2x_semester(sem1='', sem2=''): + if not len(sem1) or (not len(sem2)): + print("Need 2 semesters") + return 0 + + where1 = f"(c.sis_source_id LIKE '{sem1}-%%' or c.sis_source_id LIKE '{sem2}-%%')" + where2 = f"(c.sis_source_id NOT LIKE '{sem1}-%%' and c.sis_source_id NOT LIKE '{sem2}-%%')" + + q = """SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e +JOIN canvas.users AS u ON e.user_id=u.id +JOIN canvas.courses AS c ON e.course_id=c.id +WHERE %s +AND e.workflow_state='active' +AND e.type='StudentEnrollment' +AND u.id NOT IN ( + SELECT u.id FROM canvas.enrollments AS e + JOIN canvas.users AS u ON e.user_id=u.id + JOIN canvas.courses AS c ON e.course_id=c.id + WHERE %s + AND e.workflow_state='active' + AND e.type='StudentEnrollment' + GROUP BY u.id +) +GROUP BY u.id +ORDER BY num DESC, u.sortable_name""" % (where1,where2) + + (connection,cursor) = db() + cursor.execute(q) + users_to_enroll = {str(row[0]) for row in cursor.fetchall()} + print("%i new users this semester." % len(users_to_enroll)) + return users_to_enroll + + + + + + +def all_sem_courses_teachers(SEM="202450"): q = f"""SELECT c.id, c.name, c.course_code, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id FROM canvas.courses AS c JOIN canvas.enrollments AS e ON e.course_id=c.id JOIN canvas.users AS u ON u.id=e.user_id @@ -231,6 +285,27 @@ ORDER BY u.sortable_name;""" + +def all_2x_sem_courses_teachers(sem1, sem2): + q = f"""SELECT c.id, c.name, c.course_code, s.type, s.crn, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id FROM canvas.courses AS c +JOIN canvas.enrollments AS e ON e.course_id=c.id +JOIN canvas.users AS u ON u.id=e.user_id +JOIN canvas.pseudonyms AS p ON p.user_id=u.id +JOIN canvas.schedule as s on c.id=s.canvascourse +WHERE (c.sis_source_id LIKE '{sem1}-%' or c.sis_source_id LIKE '{sem2}-%') +AND NOT c.workflow_state='deleted' +AND e.type='TeacherEnrollment' +ORDER BY u.sortable_name;""" + (connection,cursor) = db() + cursor.execute(q) + courses = cursor.fetchall() + print(courses) + return courses + + + + + def create_schedule_table_if_not_exists(): conn,cur = db() diff --git a/tasks.py b/tasks.py index 4841613..42f16f3 100644 --- a/tasks.py +++ b/tasks.py @@ -1136,14 +1136,14 @@ def print_a_calendar(): generate_custom_calendar(year, l_semesters) - +# task list calendar for a semester def word_calendar(): from docx import Document from docx.shared import Inches import datetime # Define the start date of semester - start_date = datetime.date(2024, 1, 29) + start_date = datetime.date(2024, 7, 1) # Prepare a list of 18 weeks beginning from the start date dates = [start_date + datetime.timedelta(weeks=x) for x in range(18)] @@ -1168,6 +1168,40 @@ def word_calendar(): # Save the document doc.save('cache/tasks_schedule.docx') +# more general purpose +def word_calendar_v2(): + from docx import Document + from docx.shared import Inches + import datetime + + # Define the start date of semester + start_date = datetime.date(2024, 7, 1) + + # Prepare a list of 18 weeks beginning from the start date + dates = [start_date + datetime.timedelta(weeks=x) for x in range(40)] + + # Initialize an instance of a word document + doc = Document() + table = doc.add_table(rows=1, cols=3) + + # Set the headers + hdr_cells = table.rows[0].cells + hdr_cells[0].text = 'Week of' + hdr_cells[1].text = 'Events' + hdr_cells[2].text = 'Notes' + + # Iterate through the list of dates + for i, date in enumerate(dates): + end_date = date + datetime.timedelta(days=6) # Calculate the end date + cells = table.add_row().cells + #cells[0].text = str(i+1) + cells[0].text = f"{date.strftime('%B %d')} - {end_date.strftime('%B %d')}" + cells[1].text = '' + cells[2].text = '' + + # Save the document + doc.save('cache/weekly_calendar.docx') + # TODO some weird hour offset issue w/ these activities @@ -1271,8 +1305,9 @@ if __name__ == "__main__": 11: ['list auth', list_auth], 12: ['update auth', update_auth], 13: ['print a calendar', print_a_calendar], - 14: ['create a week calendar in word', word_calendar], - 15: ['create GOTT certificates', certificates_gott_build], + 14: ['create a week calendar in word (semester)', word_calendar], + 15: ['create a week calendar in word (general purpose)', word_calendar_v2], + 16: ['create GOTT certificates', certificates_gott_build], 20: ['build_quiz', build_quiz], 21: ['certificates_gott_build, certificates_gott_build'] } diff --git a/useful queries.sql b/useful queries.sql index b15c516..70e72b9 100644 --- a/useful queries.sql +++ b/useful queries.sql @@ -16,7 +16,68 @@ INSERT INTO canvas.schedule (canvascourse, crn, code, units, teacher, start,"end VALUES (9651, '40724', 'SPAN 1A', '5.000', 'David G Perez', '1-27', '5-22', 'in-person', 'HU 105', 'Gilroy', 'Midday', 30, 26, '202030'); +-- courses in a semester, good for finding sections to merge +select c.id, c.course_code, c.sis_source_id, e.role_id, s.start, s.type, u.sortable_name from canvas.courses c +join canvas.enrollments e on c.id=e.course_id +join canvas.users u on u.id=e.user_id +full outer join canvas.schedule s on c.id=s.canvascourse +where c.sis_source_id like '202370-%' and e.role_id=4 +order by c.course_code, u.sortable_name; +-- courses in a semester, good for finding sections to merge, no users +select c.id, c.course_code, c.sis_source_id, s.start, s.type from canvas.courses c +full outer join canvas.schedule s on c.id=s.canvascourse +where c.sis_source_id like '202370-%' +order by c.course_code; + + +select c.id, c.course_code, c.sis_source_id, e.role_id, u.sortable_name from canvas.courses c +join canvas.enrollments e on c.id=e.course_id +join canvas.users u on u.id=e.user_id +where c.sis_source_id like '202470-%' +order by c.course_code, u.sortable_name; + + + +-- all teachers in (2) semester +SELECT distinct c.id, c.name, c.course_code, s.type, s.crn, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id FROM canvas.courses AS c +JOIN canvas.enrollments AS e ON e.course_id=c.id +JOIN canvas.users AS u ON u.id=e.user_id +JOIN canvas.pseudonyms AS p ON p.user_id=u.id +JOIN canvas.schedule as s on c.id=s.canvascourse +WHERE (c.sis_source_id LIKE '202450-%' or c.sis_source_id LIKE '202470-%') +AND NOT c.workflow_state='deleted' AND e.type='TeacherEnrollment' +ORDER BY u.sortable_name; + + + + + +-- teachers of online/onlinelive/hybrid in a semester and their emails +select distinct u.sortable_name, LOWER(cc.path) as email from canvas.courses c +join canvas.enrollments e on c.id=e.course_id +join canvas.users u on u.id=e.user_id +join canvas.communication_channels cc on u.id=cc.user_id +full outer join canvas.schedule s on c.id=s.canvascourse +where (s.type='online' or s.type='hybrid' or s.type='online line') + and c.sis_source_id like '202450-%' + and cc.path_type='email' + and not cc.path like '%noemail%' + and not cc.path='sstaff@gavilan.edu' +order by u.sortable_name; + + +-- for outlook +select string_agg(distinct LOWER(cc.path), '; ') from canvas.courses c +join canvas.enrollments e on c.id=e.course_id +join canvas.users u on u.id=e.user_id +join canvas.communication_channels cc on u.id=cc.user_id +full outer join canvas.schedule s on c.id=s.canvascourse +where (s.type='online' or s.type='hybrid' or s.type='online line') + and c.sis_source_id like '202450-%' + and cc.path_type='email' + and not cc.path like '%noemail%' + and not cc.path='sstaff@gavilan.edu'; -- -- @@ -92,4 +153,53 @@ join canvas.enrollments e on u.id=e.user_id join canvas.courses c on e.course_id=c.id join canvas.schedule s on c.id=s.canvascourse where s.sem='202430' and e.workflow_state='active' and e.type='StudentEnrollment' -order by u.sortable_name, c.course_code; \ No newline at end of file +order by u.sortable_name, c.course_code; + + + +-- users in orientation shell + +select e.user_id from canvas.enrollments e +where e.course_id = 19094 and e.type='StudentEnrollment' and e.workflow_state='active'; + + +-- new users this (single) semester +SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e +JOIN canvas.users AS u ON e.user_id=u.id +JOIN canvas.courses AS c ON e.course_id=c.id +WHERE c.sis_source_id LIKE '202450-%%' +AND e.workflow_state='active' +AND e.type='StudentEnrollment' +AND u.id NOT IN ( + SELECT u.id FROM canvas.enrollments AS e + JOIN canvas.users AS u ON e.user_id=u.id + JOIN canvas.courses AS c ON e.course_id=c.id + WHERE c.sis_source_id NOT LIKE '202450-%%' + AND e.workflow_state='active' + AND e.type='StudentEnrollment' + GROUP BY u.id +) +GROUP BY u.id +ORDER BY num DESC, u.sortable_name + + + +-- new users this (summer+fall) semester +SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e +JOIN canvas.users AS u ON e.user_id=u.id +JOIN canvas.courses AS c ON e.course_id=c.id +WHERE (c.sis_source_id LIKE '202450-%%' or c.sis_source_id LIKE '202470-%%') +AND e.workflow_state='active' +AND e.type='StudentEnrollment' +AND u.id NOT IN ( + SELECT u.id FROM canvas.enrollments AS e + JOIN canvas.users AS u ON e.user_id=u.id + JOIN canvas.courses AS c ON e.course_id=c.id + WHERE (c.sis_source_id NOT LIKE '202450-%%' and c.sis_source_id NOT LIKE '202470-%%') + AND e.workflow_state='active' + AND e.type='StudentEnrollment' + GROUP BY u.id +) +GROUP BY u.id +ORDER BY num DESC, u.sortable_name + diff --git a/users.py b/users.py index 018c84a..c076c4b 100644 --- a/users.py +++ b/users.py @@ -11,7 +11,10 @@ from collections import defaultdict from pipelines import fetch, fetch_stream, getSemesterSchedule, header, url, FetchError, put_file from courses import course_enrollment, users_in_semester from localcache import users_this_semester_db, unwanted_req_paths, timeblock_24hr_from_dt, dt_from_24hr_timeblock -from localcache import teachers_courses_semester +from localcache import teachers_courses_semester, course_mode, sem_schedule +from localcache2 import all_2x_sem_courses_teachers, all_sem_courses_teachers +from pipelines import dean, dean_names + from util import dept_from_name, most_common_item from os.path import exists, getmtime @@ -2234,10 +2237,6 @@ def compare_db_tables(): def training_find_goos(): - from localcache2 import all_sem_courses_teachers - from localcache import course_mode - from localcache import sem_schedule - from pipelines import dean from openpyxl import Workbook, load_workbook from openpyxl.chart import BarChart, Series, Reference from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill @@ -2268,19 +2267,21 @@ def training_find_goos(): print() def cross_ref_training(): - from localcache2 import all_sem_courses_teachers - from localcache import course_mode - from localcache import sem_schedule - from pipelines import dean, dean_names from openpyxl import Workbook, load_workbook from openpyxl.chart import BarChart, Series, Reference from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill - wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx") + wb = load_workbook("C:/Users/phowell/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx") print(wb.sheetnames) # report for email report = codecs.open('cache/gott_report.txt','w','utf-8') + # update local list of teachers from ilearn? + RELOAD_TEACHERS = 0 + if RELOAD_TEACHERS: + teacherRolesUpdateCache() + + # TODO inefficient but just read it again all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) records = {} @@ -2303,18 +2304,18 @@ def cross_ref_training(): alldepts = set() - courses = all_sem_courses_teachers() + courses = all_2x_sem_courses_teachers('202450', '202470') # all_sem_courses_teachers() for c in courses: print(c) - goo = c[6] - crn = c[2].split(' ')[-1].split('/')[0] + goo = c[8] + crn = c[4] name = c[2] - teacher = c[4] - ctype = course_mode(crn,'sp24') + teacher = c[5] + ctype = c[3] dept1 = re.search(r'([A-Z]+)(\d+)',c[2].split(' ')[0]).group(1) alldepts.add(dept1) d = list(c) - d.append(ctype) + #d.append(ctype) if not ctype: print(f"not finding mode for {name}") continue @@ -2325,7 +2326,7 @@ def cross_ref_training(): alldepts = list(alldepts) alldepts.sort() - sheet = wb.create_sheet("Spring 2024 Summary") + sheet = wb.create_sheet("New Summary") r = 1 deptfont = Font(bold=True) @@ -2354,7 +2355,7 @@ def cross_ref_training(): waived = 0 sects = teachers[t] print(f"Sections for {t}: {sects}") - goo = sects[0][6] + goo = sects[0][8] print(t) sheet.cell(row=r, column=1).value = f"{t}" sheet.cell(row=r, column=2).value = f"{goo}" @@ -2425,12 +2426,12 @@ def cross_ref_training(): print(f" GOTT abc self paced") r += 1 for s in sects: - sheet.cell(row=r, column=2).value = f"{s[7]}" + sheet.cell(row=r, column=2).value = f"{s[3]}" sheet.cell(row=r, column=3).value = f"{s[1]}" r += 1 if (not completed) and (not waived): - report.write(f"\t\t{s[7]}\t{s[1]}\n") + report.write(f"\t\t{s[3]}\t{s[1]}\n") if (not completed) and (not waived): report.write(f"\n") @@ -2441,7 +2442,7 @@ def cross_ref_training(): sheet.column_dimensions['B'].width = 30 sheet.column_dimensions['C'].width = 75 formatted_date = datetime.datetime.now().strftime('%Y%m%d') - wb.save(f"C:/Users/peter/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx") + wb.save(f"C:/Users/phowell/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx") def cross_ref_training_withcsv(): from localcache2 import all_sem_courses_teachers diff --git a/util.py b/util.py index 674cef4..69367ee 100644 --- a/util.py +++ b/util.py @@ -105,6 +105,13 @@ def dept_from_name(n): print(("Couldn't find dept from: " + n)) return '' +# ENGL250 returns 250 +def num_from_name(n): + m = re.search('^([a-zA-Z]+)\s?([\d\/]+[A-Z]?)',n) + if m: return m.group(2) + print(("Couldn't find num from: " + n)) + return '' + def most_common_item(li): d = defaultdict(int) for x in li: