From 69b4c2d86d258f47f2b769b5d62b86088f16fb8f Mon Sep 17 00:00:00 2001 From: Coding with Peter Date: Sat, 17 Feb 2024 14:56:15 -0800 Subject: [PATCH] convert to canvas data 2. gott certs --- courses.py | 44 +++++++++++++-------- depricated.py | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ localcache2.py | 41 ++++++++++++++++++- pipelines.py | 10 +++-- tasks.py | 104 ++++++------------------------------------------- 5 files changed, 187 insertions(+), 114 deletions(-) diff --git a/courses.py b/courses.py index fb9999d..f5fc03b 100644 --- a/courses.py +++ b/courses.py @@ -7,7 +7,8 @@ from datetime import datetime from util import print_table, int_or_zero, float_or_zero from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem from pipelines import sems -from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload +from localcache import db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload +from localcache2 import users_new_this_semester from collections import defaultdict @@ -228,7 +229,7 @@ def users_in_semester(): # # All students in STEM (or any list of depts.. match the course_code). Return SET of canvas ids. -def users_in_by_depts_live(depts=[], termid='171'): +def users_in_by_depts_live(depts=[], termid='181'): courses_by_by_dept = {} students_by_by_dept = {} @@ -912,7 +913,8 @@ def enroll_id_list_to_shell(id_list, shell_id, v=0): def enroll_stem_students_live(): - the_term = '180' # su23 fa23 = 180 + import localcache2 + the_term = '181' # su23 fa23 = 180 do_removes = 0 depts = "MATH BIO CHEM CSIS PHYS PSCI GEOG ASTR ECOL ENVS ENGR".split(" ") users_to_enroll = users_in_by_depts_live(depts, the_term) # term id @@ -928,7 +930,7 @@ def enroll_stem_students_live(): remove_us = users_in_stem_shell.difference(users_to_enroll) print("\n\nTO ENROLL %s" % str(enroll_us)) - (connection,cursor) = db() + (connection,cursor) = localcache2.db() #xyz = input('enter to continue') @@ -941,7 +943,7 @@ def enroll_stem_students_live(): print("\n\nTO REMOVE %s" % str(remove_us)) for j in remove_us: try: - q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j + q = "SELECT name,id FROM canvas.users WHERE id=%s" % j cursor.execute(q) s = cursor.fetchall() if s: @@ -956,7 +958,7 @@ def enroll_stem_students_live(): for j in enroll_us: try: - q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j + q = "SELECT name,id FROM canvas.users WHERE id=%s" % j cursor.execute(q) s = cursor.fetchall() if s: @@ -1126,32 +1128,33 @@ def enroll_art_students_live(): print("done.") def enroll_orientation_students(): + import localcache2 ori_shell_id = "19094" # 2024 # "" # 2023 orientation shell 15924 # 2022: "9768" + print("Getting users in orientation shell") users_in_ori_shell = set( \ [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ]) for the_semester in ["202430"]: - #the_semester = "202350" # "202350" - users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll)) print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell)) enroll_us = users_to_enroll.difference(users_in_ori_shell) - print("\n\nTO ENROLL %s" % str(enroll_us)) - print("%i new users to enroll." % len(enroll_us)) + print("\n\nTO ENROLL %s\n" % str(enroll_us)) + print("%i new users to enroll.\n" % len(enroll_us)) eee = 0 uuu = 0 - (connection,cursor) = db() + (connection,cursor) = localcache2.db() for j in enroll_us: s = "" try: - q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j + q = "SELECT name,id FROM canvas.users WHERE id=%s" % j + #print(q) cursor.execute(q) s = cursor.fetchall() if s: @@ -1160,11 +1163,13 @@ def enroll_orientation_students(): t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment', 'enrollment[enrollment_state]': 'active' } + #print(t) #print(data) - r3 = requests.post(t, headers=header, params=data) - eee += 1 - #print(r3.text) - time.sleep(0.400) + if 1: + r3 = requests.post(t, headers=header, params=data) + eee += 1 + #print(r3.text) + time.sleep(0.250) except Exception as e: print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e))) # return (eee,uuu) @@ -2006,6 +2011,13 @@ def fetch_announcements(): print("Announcements saved to ", filename) +def change_link_in_all_terms_pages(): + old_link = "https://www.gavilan.edu/ezproxy" + new_link = "https://www.gavilan.edu/ezproxy_new" + + term = 181 + + courses = getCoursesInTerm(term,get_fresh=1,show=0,active=1) if __name__ == "__main__": diff --git a/depricated.py b/depricated.py index d35f11c..69013e7 100644 --- a/depricated.py +++ b/depricated.py @@ -189,6 +189,108 @@ def stats(): pass +######### from tasks.py + + +def certificates_gott_build_2020(): + #send_email("Peter Howell", "Peter", "phowell@gavilan.edu", "test", "this is a test") + + #g2e = user_goo_to_email() + g2e = {} # missing function? + g2name = {} + ix = {} # everyone + ix1 = {} # only gott 1 + ix2 = {} # only gott 2 + + cc = csv.reader( open('cache/completers_gott1_su20.csv','r'), delimiter=',') + + '''for row in cc: + # name, goo, section, x, count + doc = DocxTemplate("cache/certificates/gott 1 template.docx") + doc.render({ 'name' : row[0] }) + fn = "cache/certificates/gott_1_%s." % re.sub('\s', '_', row[0].lower()) + print(fn+'docx') + try: + goo = row[1] + email = g2e[ goo ] + print(email) + g2name[goo] = row[0] + ix1[ goo ] = fn+"pdf" + ix[ goo ] = email + except: + print("can't find email") + doc.save(fn+'docx') + #convert_to_pdf(fn+'docx', fn+'pdf') + ''' + + for row in csv.reader( open('cache/completers_gott2_wi24.csv','r'), delimiter=','): + # name, goo, section, x, count + doc = DocxTemplate("cache/certificates/gott 2 template.docx") + doc.render({ 'name' : row[0] }) + fn = "cache/certificates/gott_2_wi24_%s." % re.sub('\s', '_', row[0].lower()) + print(fn+'docx') + try: + goo = row[1] + email = g2e[ goo ] + print( email ) + g2name[goo] = row[0] + ix2[ goo ] = fn+"pdf" + ix[ goo ] = email + except: + print("can't find email") + doc.save(fn+'docx') + #convert_to_pdf(fn+'docx', fn+'pdf') + + # + ''' + g1f = open('cache/gott_emails_1.csv', 'w') + g2f = open('cache/gott_emails_2.csv', 'w') + g12f = open('cache/gott_emails_12.csv', 'w') + for k in ix.keys(): + if k in ix1 and not k in ix2: + print(k + " only gott 1") + file1 = ix1[k] + email = ix[k] + file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1] + fname = g2name[k] + g1f.write("%s, %s, %s\n" % (fname, email, file1)) + elif k in ix2 and not k in ix1: + print(k + " only in gott 2") + file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1] + email = ix[k] + fname = g2name[k] + g2f.write("%s, %s, %s\n" % (fname, email, file2)) + elif k in ix1 and k in ix2: + print(k + " in both") + file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1] + file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1] + email = ix[k] + fname = g2name[k] + g12f.write("%s, %s, %s, %s\n" % (fname, email, file1, file2)) + ''' + +# Email experiment +def mail_test(): + outlook = win32com.client.Dispatch('outlook.application') #get a reference to Outlook + mail = outlook.CreateItem(0) #create a new mail item + mail.To = 'executives@bigcompany.com' + mail.Subject = 'Finance Status Report '+datetime.today().strftime('%m/%d') + + mail.HTMLBody = ''' +

Hi Team,

+ +

This email is to provide a status of the our current sales numbers

+ + + + + + + + +

Thanks and have a great day!

+ ''' + mail.Display() diff --git a/localcache2.py b/localcache2.py index a1908b1..12bb6ba 100644 --- a/localcache2.py +++ b/localcache2.py @@ -171,7 +171,45 @@ ORDER BY c.sis_source_id, wp.title;""" +def users_new_this_semester(sem=''): + if not len(sem): + sem = input("which semester? (ex: 202150) ") + users_to_enroll = set() + where1 = "c.sis_source_id LIKE '%s-%%'" % sem + where2 = "c.sis_source_id NOT LIKE '%s-%%'" % sem + + + + q = """SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e +JOIN canvas.users AS u ON e.user_id=u.id +JOIN canvas.courses AS c ON e.course_id=c.id +WHERE %s +AND e.workflow_state='active' +AND e.type='StudentEnrollment' +AND u.id NOT IN ( + SELECT u.id FROM canvas.enrollments AS e + JOIN canvas.users AS u ON e.user_id=u.id + JOIN canvas.courses AS c ON e.course_id=c.id + WHERE %s + AND e.workflow_state='active' + AND e.type='StudentEnrollment' + GROUP BY u.id +) +GROUP BY u.id +ORDER BY num DESC, u.sortable_name""" % (where1,where2) + + + (connection,cursor) = db() + cursor.execute(q) + #s = cursor.fetchall() + #if s: + for u in cursor: + users_to_enroll.add(str(u[0])) + print(u) + print("%i new users this semester." % len(users_to_enroll)) + print(users_to_enroll) + return users_to_enroll @@ -182,7 +220,8 @@ if __name__ == "__main__": options = { 1: ['all teachers', teachers_by_term], 2: ['courses in term', courses_in_term], - 3: ['pages in term', pages_in_term] + 3: ['pages in term', pages_in_term], + 4: ['new students this semester', users_new_this_semester], } diff --git a/pipelines.py b/pipelines.py index b56bd82..fcda51b 100644 --- a/pipelines.py +++ b/pipelines.py @@ -1248,7 +1248,7 @@ def fetch_current_rosters(): print(' Saved %i data files in rosters folder.' % i) ff.write( f" Saved {i} data files: {seen_files}\n") - if i: + if i>2: if 'courses.csv' in seen_files: courses = open('cache/rosters/courses-%s.csv' % dt_label,'r') courses.readline() @@ -1276,10 +1276,12 @@ def fetch_current_rosters(): else: print("--> Don't see files.") sftp.close() + time.sleep(59) def fetch_current_rosters_auto(): - fetch_minute = "58" - schedule.every().hour.at(f":{fetch_minute}").do(fetch_current_rosters) + fetch_minute = "56,57,58,59,00,01".split(",") + for m in fetch_minute: + schedule.every().hour.at(f":{m}").do(fetch_current_rosters) #schedule.every().day.at("12:35").do(sync_non_interactive) #schedule.every().day.at("21:00").do(sync_non_interactive) @@ -2266,7 +2268,7 @@ if __name__ == "__main__": print ('') options = { 1: ['Re-create schedule csv and json files from raw html',recent_schedules] , 2: ['Fetch rosters',fetch_current_rosters] , - 3: ['Fetch rosters AND canvas data automatically',fetch_current_rosters_auto] , + 3: ['Fetch rosters on schedule',fetch_current_rosters_auto] , 4: ['Compute how registration is filling up classes', schedule_filling] , 5: ['Manually convert 3 csv files to joined json enrollment file.', convert_roster_files] , 6: ['Canvas data: interactive sync', interactive ], diff --git a/tasks.py b/tasks.py index 286df0f..ed52f46 100644 --- a/tasks.py +++ b/tasks.py @@ -165,6 +165,7 @@ def send_email(fullname, firstname, addr, subj, content): def convert_to_pdf(name1, name2): wd = 'C:\\Users\\peter\\Documents\\gavilan\\canvasapp\\' + wd = 'I:/canvasapp/' print( wd + name1 ) try: word = win32.DispatchEx("Word.Application") @@ -180,103 +181,19 @@ def convert_to_pdf(name1, name2): # Build (docx/pdf) certificates for gott graduates def certificates_gott_build(): - #send_email("Peter Howell", "Peter", "phowell@gavilan.edu", "test", "this is a test") - - #g2e = user_goo_to_email() - g2e = {} # missing function? - g2name = {} - ix = {} # everyone - ix1 = {} # only gott 1 - ix2 = {} # only gott 2 - - cc = csv.reader( open('cache/completers_gott1_su20.csv','r'), delimiter=',') - cc2 = csv.reader( open('cache/completers_gott2_su20.csv','r'), delimiter=',') - - for row in cc: - # name, goo, section, x, count - doc = DocxTemplate("cache/certificates/gott 1 template.docx") - doc.render({ 'name' : row[0] }) - fn = "cache/certificates/gott_1_%s." % re.sub('\s', '_', row[0].lower()) - print(fn+'docx') - try: - goo = row[1] - email = g2e[ goo ] - print(email) - g2name[goo] = row[0] - ix1[ goo ] = fn+"pdf" - ix[ goo ] = email - except: - print("can't find email") - doc.save(fn+'docx') - #convert_to_pdf(fn+'docx', fn+'pdf') - - for row in cc2: - # name, goo, section, x, count + i = 0 + for row in csv.reader( open('cache/completers_gott2_wi24.csv','r'), delimiter=','): + i += 1 + if i < 3: continue + print(row[0]) + lname,fname = row[0].split(",") + name = fname.strip() + " " + lname.strip() doc = DocxTemplate("cache/certificates/gott 2 template.docx") doc.render({ 'name' : row[0] }) - fn = "cache/certificates/gott_2_%s." % re.sub('\s', '_', row[0].lower()) + fn = "cache/certificates/gott_2_wi24%s." % re.sub('\s', '_', name.lower()) print(fn+'docx') - try: - goo = row[1] - email = g2e[ goo ] - print( email ) - g2name[goo] = row[0] - ix2[ goo ] = fn+"pdf" - ix[ goo ] = email - except: - print("can't find email") doc.save(fn+'docx') - #convert_to_pdf(fn+'docx', fn+'pdf') - - # - g1f = open('cache/gott_emails_1.csv', 'w') - g2f = open('cache/gott_emails_2.csv', 'w') - g12f = open('cache/gott_emails_12.csv', 'w') - for k in ix.keys(): - if k in ix1 and not k in ix2: - print(k + " only gott 1") - file1 = ix1[k] - email = ix[k] - file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1] - fname = g2name[k] - g1f.write("%s, %s, %s\n" % (fname, email, file1)) - elif k in ix2 and not k in ix1: - print(k + " only in gott 2") - file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1] - email = ix[k] - fname = g2name[k] - g2f.write("%s, %s, %s\n" % (fname, email, file2)) - elif k in ix1 and k in ix2: - print(k + " in both") - file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1] - file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1] - email = ix[k] - fname = g2name[k] - g12f.write("%s, %s, %s, %s\n" % (fname, email, file1, file2)) - - -# Email experiment -def mail_test(): - outlook = win32com.client.Dispatch('outlook.application') #get a reference to Outlook - mail = outlook.CreateItem(0) #create a new mail item - mail.To = 'executives@bigcompany.com' - mail.Subject = 'Finance Status Report '+datetime.today().strftime('%m/%d') - - mail.HTMLBody = ''' -

Hi Team,

- -

This email is to provide a status of the our current sales numbers

- - - - - - - - -

Thanks and have a great day!

- ''' - mail.Display() + convert_to_pdf(fn+'docx', fn+'pdf') # Change LTI Settings. Experimental @@ -1461,6 +1378,7 @@ if __name__ == "__main__": 12: ['update auth', update_auth], 13: ['print a calendar', print_a_calendar], 14: ['create a week calendar in word', word_calendar], + 15: ['create GOTT certificates', certificates_gott_build], } if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):