convert to canvas data 2. gott certs

courses.py

@@ -7,7 +7,8 @@ from datetime import datetime
 from util import print_table, int_or_zero, float_or_zero
 from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem
 from pipelines import sems
-from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
+from localcache import db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
+from localcache2 import users_new_this_semester
 from collections import defaultdict
 
 
@@ -228,7 +229,7 @@ def users_in_semester():
 
 #
 # All students in STEM (or any list of depts.. match the course_code). Return SET of canvas ids.
-def users_in_by_depts_live(depts=[], termid='171'):
+def users_in_by_depts_live(depts=[], termid='181'):
     courses_by_by_dept = {}
     students_by_by_dept = {}
 
@@ -912,7 +913,8 @@ def enroll_id_list_to_shell(id_list, shell_id, v=0):
 
 
 def enroll_stem_students_live():
-    the_term = '180'  # su23   fa23 = 180
+    import localcache2
+    the_term = '181'  # su23   fa23 = 180
     do_removes = 0
     depts = "MATH BIO CHEM CSIS PHYS PSCI GEOG ASTR ECOL ENVS ENGR".split(" ")
     users_to_enroll = users_in_by_depts_live(depts, the_term)      # term id
@@ -928,7 +930,7 @@ def enroll_stem_students_live():
     remove_us = users_in_stem_shell.difference(users_to_enroll)
 
     print("\n\nTO ENROLL %s" % str(enroll_us))
-    (connection,cursor) = db()
+    (connection,cursor) = localcache2.db()
                                     
     #xyz = input('enter to continue')
 
@@ -941,7 +943,7 @@ def enroll_stem_students_live():
         print("\n\nTO REMOVE %s" % str(remove_us))
         for j in remove_us:
             try:
-                q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+                q = "SELECT name,id FROM canvas.users WHERE id=%s" % j
                 cursor.execute(q)
                 s = cursor.fetchall()
                 if s:
@@ -956,7 +958,7 @@ def enroll_stem_students_live():
 
     for j in enroll_us:
         try:
-            q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+            q = "SELECT name,id FROM canvas.users WHERE id=%s" % j
             cursor.execute(q)
             s = cursor.fetchall()
             if s:
@@ -1126,32 +1128,33 @@ def enroll_art_students_live():
     print("done.")
 
 def enroll_orientation_students():
+    import localcache2
     ori_shell_id = "19094"    #  2024    #  ""    #   2023 orientation shell  15924   #      2022: "9768"
 
+    print("Getting users in orientation shell")
     users_in_ori_shell = set( \
             [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
 
     for the_semester in ["202430"]:
-        #the_semester = "202350"   # "202350"
-
         users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
         print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
         print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
 
         enroll_us = users_to_enroll.difference(users_in_ori_shell)
 
-        print("\n\nTO ENROLL %s" % str(enroll_us))
+        print("\n\nTO ENROLL %s\n" % str(enroll_us))
-        print("%i new users to enroll." % len(enroll_us))
+        print("%i new users to enroll.\n" % len(enroll_us))
 
         eee = 0
         uuu = 0
         
-        (connection,cursor) = db()
+        (connection,cursor) = localcache2.db()
 
         for j in enroll_us:
             s = ""
             try:
-                q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+                q = "SELECT name,id FROM canvas.users WHERE id=%s" % j
+                #print(q)
                 cursor.execute(q)
                 s = cursor.fetchall()
                 if s:
@@ -1160,11 +1163,13 @@ def enroll_orientation_students():
                 t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
                 data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
                         'enrollment[enrollment_state]': 'active' }
+                #print(t)
                 #print(data)
+                if 1:
                     r3 = requests.post(t, headers=header, params=data)
                     eee += 1
                     #print(r3.text)
-                time.sleep(0.400)
+                    time.sleep(0.250)
             except Exception as e:
                 print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
     # return (eee,uuu)
@@ -2006,6 +2011,13 @@ def fetch_announcements():
     print("Announcements saved to ", filename)
 
     
+def change_link_in_all_terms_pages():
+    old_link = "https://www.gavilan.edu/ezproxy"
+    new_link = "https://www.gavilan.edu/ezproxy_new"
+
+    term = 181
+
+    courses = getCoursesInTerm(term,get_fresh=1,show=0,active=1)
     
 
 if __name__ == "__main__":

depricated.py

@@ -189,6 +189,108 @@ def stats():
             pass
 
 
+#########  from tasks.py
+        
+
+def certificates_gott_build_2020():
+    #send_email("Peter Howell", "Peter", "phowell@gavilan.edu", "test", "this is a test")
+    
+    #g2e = user_goo_to_email()
+    g2e = {}  # missing function?
+    g2name = {}
+    ix = {}     # everyone 
+    ix1 = {}    # only gott 1
+    ix2 = {}    # only gott 2
+
+    cc = csv.reader( open('cache/completers_gott1_su20.csv','r'), delimiter=',')
+
+    '''for row in cc:
+        # name, goo, section, x, count
+        doc = DocxTemplate("cache/certificates/gott 1 template.docx")
+        doc.render({ 'name' : row[0] })
+        fn = "cache/certificates/gott_1_%s." % re.sub('\s', '_', row[0].lower())
+        print(fn+'docx')
+        try:
+            goo = row[1]
+            email = g2e[ goo ]
+            print(email)
+            g2name[goo] = row[0]
+            ix1[ goo ] = fn+"pdf"
+            ix[ goo ] = email
+        except:
+            print("can't find email")
+        doc.save(fn+'docx')
+        #convert_to_pdf(fn+'docx', fn+'pdf')
+    '''
+
+    for row in csv.reader( open('cache/completers_gott2_wi24.csv','r'), delimiter=','):
+        # name, goo, section, x, count
+        doc = DocxTemplate("cache/certificates/gott 2 template.docx")
+        doc.render({ 'name' : row[0] })
+        fn = "cache/certificates/gott_2_wi24_%s." % re.sub('\s', '_', row[0].lower())
+        print(fn+'docx')
+        try:
+            goo = row[1]
+            email = g2e[ goo ]
+            print( email )
+            g2name[goo] = row[0]
+            ix2[ goo ] = fn+"pdf"
+            ix[ goo ] = email
+        except:
+            print("can't find email")
+        doc.save(fn+'docx')
+        #convert_to_pdf(fn+'docx', fn+'pdf')
+    
+    # 
+    '''
+    g1f = open('cache/gott_emails_1.csv', 'w')
+    g2f = open('cache/gott_emails_2.csv', 'w')
+    g12f = open('cache/gott_emails_12.csv', 'w')
+    for k in ix.keys():
+        if k in ix1 and not k in ix2:
+            print(k + " only gott 1")
+            file1 = ix1[k]
+            email = ix[k]
+            file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1]
+            fname = g2name[k]
+            g1f.write("%s, %s, %s\n" % (fname, email, file1))
+        elif k in ix2 and not k in ix1:
+            print(k + " only in gott 2")
+            file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1]
+            email = ix[k]
+            fname = g2name[k]
+            g2f.write("%s, %s, %s\n" % (fname, email, file2))
+        elif k in ix1 and k in ix2:
+            print(k + " in both")
+            file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1]
+            file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1]
+            email = ix[k]
+            fname = g2name[k]
+            g12f.write("%s, %s, %s, %s\n" % (fname, email, file1, file2))
+    '''
+
+# Email experiment
+def mail_test():
+    outlook = win32com.client.Dispatch('outlook.application')   #get a reference to Outlook
+    mail = outlook.CreateItem(0)                      #create a new mail item
+    mail.To = 'executives@bigcompany.com'
+    mail.Subject = 'Finance Status Report '+datetime.today().strftime('%m/%d')  
+
+    mail.HTMLBody = '''
+    <p>Hi Team,</p>
+
+    <p>This email is to provide a status of the our current sales numbers</p>
+
+
+    <img src='C:\\Users\\sam\\Desktop\\EmailAuto\\paste1.png'>
+
+
+    <img src='C:\\Users\\sam\\Desktop\\EmailAuto\\paste2.png'>
+
+
+    <p>Thanks and have a great day!</p>
+    '''
+    mail.Display()
 
 
 

localcache2.py

@@ -171,7 +171,45 @@ ORDER BY c.sis_source_id, wp.title;"""
 
 
 
+def users_new_this_semester(sem=''):
+    if not len(sem):
+        sem = input("which semester? (ex: 202150) ")
+    users_to_enroll = set()
 
+    where1 = "c.sis_source_id LIKE '%s-%%'" % sem
+    where2 = "c.sis_source_id NOT LIKE '%s-%%'" % sem
+
+    
+
+    q = """SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e
+JOIN canvas.users AS u ON e.user_id=u.id
+JOIN canvas.courses AS c ON e.course_id=c.id
+WHERE %s
+AND e.workflow_state='active'
+AND e.type='StudentEnrollment'
+AND u.id NOT IN (
+	SELECT u.id FROM canvas.enrollments AS e
+	JOIN canvas.users AS u ON e.user_id=u.id
+	JOIN canvas.courses AS c ON e.course_id=c.id
+	WHERE %s
+	AND e.workflow_state='active'
+	AND e.type='StudentEnrollment'
+	GROUP BY u.id
+)
+GROUP BY u.id
+ORDER BY num DESC, u.sortable_name""" % (where1,where2)
+
+
+    (connection,cursor) = db()
+    cursor.execute(q)
+    #s = cursor.fetchall()
+    #if s:
+    for u in cursor:
+        users_to_enroll.add(str(u[0]))
+        print(u)
+    print("%i new users this semester." % len(users_to_enroll))
+    print(users_to_enroll)
+    return users_to_enroll
 
 
 
@@ -182,7 +220,8 @@ if __name__ == "__main__":
     options = { 
             1: ['all teachers', teachers_by_term],
             2: ['courses in term', courses_in_term],
-            3: ['pages in term', pages_in_term]
+            3: ['pages in term', pages_in_term],
+            4: ['new students this semester', users_new_this_semester],
     }
 
 

pipelines.py

@@ -1248,7 +1248,7 @@ def fetch_current_rosters():
             print('   Saved %i data files in rosters folder.' % i)
             ff.write( f"   Saved {i} data files: {seen_files}\n")
             
-            if i:
+            if i>2:
                 if 'courses.csv' in seen_files:
                     courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
                     courses.readline()
@@ -1276,10 +1276,12 @@ def fetch_current_rosters():
         else:
             print("--> Don't see files.")
     sftp.close()
+    time.sleep(59)
 
 def fetch_current_rosters_auto():
-    fetch_minute = "58"
+    fetch_minute = "56,57,58,59,00,01".split(",")
-    schedule.every().hour.at(f":{fetch_minute}").do(fetch_current_rosters)
+    for m in fetch_minute:
+        schedule.every().hour.at(f":{m}").do(fetch_current_rosters)
     
     #schedule.every().day.at("12:35").do(sync_non_interactive)
     #schedule.every().day.at("21:00").do(sync_non_interactive)
@@ -2266,7 +2268,7 @@ if __name__ == "__main__":
     print ('')
     options = { 1: ['Re-create schedule csv and json files from raw html',recent_schedules] ,  
                 2: ['Fetch rosters',fetch_current_rosters] , 
-                3: ['Fetch rosters AND canvas data automatically',fetch_current_rosters_auto] , 
+                3: ['Fetch rosters on schedule',fetch_current_rosters_auto] , 
                 4: ['Compute how registration is filling up classes', schedule_filling] , 
                 5: ['Manually convert 3 csv files to joined json enrollment file.', convert_roster_files] , 
                 6: ['Canvas data: interactive sync', interactive ], 

tasks.py

@@ -165,6 +165,7 @@ def send_email(fullname, firstname, addr, subj, content):
 
 def convert_to_pdf(name1, name2):
     wd = 'C:\\Users\\peter\\Documents\\gavilan\\canvasapp\\'
+    wd = 'I:/canvasapp/'
     print( wd + name1 )
     try:
         word = win32.DispatchEx("Word.Application")
@@ -180,103 +181,19 @@ def convert_to_pdf(name1, name2):
 
 # Build (docx/pdf) certificates for gott graduates
 def certificates_gott_build():
-    #send_email("Peter Howell", "Peter", "phowell@gavilan.edu", "test", "this is a test")
+    i = 0
-    
+    for row in csv.reader( open('cache/completers_gott2_wi24.csv','r'), delimiter=','):
-    #g2e = user_goo_to_email()
+        i += 1
-    g2e = {}  # missing function?
+        if i < 3: continue
-    g2name = {}
+        print(row[0])
-    ix = {}     # everyone 
+        lname,fname = row[0].split(",")
-    ix1 = {}    # only gott 1
+        name = fname.strip() + " " + lname.strip()
-    ix2 = {}    # only gott 2
-
-    cc = csv.reader( open('cache/completers_gott1_su20.csv','r'), delimiter=',')
-    cc2 = csv.reader( open('cache/completers_gott2_su20.csv','r'), delimiter=',')
-
-    for row in cc:
-        # name, goo, section, x, count
-        doc = DocxTemplate("cache/certificates/gott 1 template.docx")
-        doc.render({ 'name' : row[0] })
-        fn = "cache/certificates/gott_1_%s." % re.sub('\s', '_', row[0].lower())
-        print(fn+'docx')
-        try:
-            goo = row[1]
-            email = g2e[ goo ]
-            print(email)
-            g2name[goo] = row[0]
-            ix1[ goo ] = fn+"pdf"
-            ix[ goo ] = email
-        except:
-            print("can't find email")
-        doc.save(fn+'docx')
-        #convert_to_pdf(fn+'docx', fn+'pdf')
-
-    for row in cc2:
-        # name, goo, section, x, count
         doc = DocxTemplate("cache/certificates/gott 2 template.docx")
         doc.render({ 'name' : row[0] })
-        fn = "cache/certificates/gott_2_%s." % re.sub('\s', '_', row[0].lower())
+        fn = "cache/certificates/gott_2_wi24%s." % re.sub('\s', '_', name.lower())
         print(fn+'docx')
-        try:
-            goo = row[1]
-            email = g2e[ goo ]
-            print( email )
-            g2name[goo] = row[0]
-            ix2[ goo ] = fn+"pdf"
-            ix[ goo ] = email
-        except:
-            print("can't find email")
         doc.save(fn+'docx')
-        #convert_to_pdf(fn+'docx', fn+'pdf')
+        convert_to_pdf(fn+'docx', fn+'pdf')
-    
-    # 
-    g1f = open('cache/gott_emails_1.csv', 'w')
-    g2f = open('cache/gott_emails_2.csv', 'w')
-    g12f = open('cache/gott_emails_12.csv', 'w')
-    for k in ix.keys():
-        if k in ix1 and not k in ix2:
-            print(k + " only gott 1")
-            file1 = ix1[k]
-            email = ix[k]
-            file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1]
-            fname = g2name[k]
-            g1f.write("%s, %s, %s\n" % (fname, email, file1))
-        elif k in ix2 and not k in ix1:
-            print(k + " only in gott 2")
-            file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1]
-            email = ix[k]
-            fname = g2name[k]
-            g2f.write("%s, %s, %s\n" % (fname, email, file2))
-        elif k in ix1 and k in ix2:
-            print(k + " in both")
-            file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1]
-            file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1]
-            email = ix[k]
-            fname = g2name[k]
-            g12f.write("%s, %s, %s, %s\n" % (fname, email, file1, file2))
-
-
-# Email experiment
-def mail_test():
-    outlook = win32com.client.Dispatch('outlook.application')   #get a reference to Outlook
-    mail = outlook.CreateItem(0)                      #create a new mail item
-    mail.To = 'executives@bigcompany.com'
-    mail.Subject = 'Finance Status Report '+datetime.today().strftime('%m/%d')  
-
-    mail.HTMLBody = '''
-    <p>Hi Team,</p>
-
-    <p>This email is to provide a status of the our current sales numbers</p>
-
-
-    <img src='C:\\Users\\sam\\Desktop\\EmailAuto\\paste1.png'>
-
-
-    <img src='C:\\Users\\sam\\Desktop\\EmailAuto\\paste2.png'>
-
-
-    <p>Thanks and have a great day!</p>
-    '''
-    mail.Display()
 
 
 # Change LTI Settings. Experimental
@@ -1461,6 +1378,7 @@ if __name__ == "__main__":
                 12: ['update auth', update_auth],
                 13: ['print a calendar', print_a_calendar],
                 14: ['create a week calendar in word', word_calendar],
+                15: ['create GOTT certificates', certificates_gott_build],
               }
     
     if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):