updates to web mir srvr, outcomes, oritn shell fil

courses.py

@@ -1,9 +1,10 @@
 import json, re, requests, codecs, sys, time, funcy, os
 import pandas as pd
+from datetime import datetime   
+import pytz
 from dateutil import parser
 from datetime import datetime
 from util import print_table, int_or_zero, float_or_zero
-
 from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem
 from pipelines import sems
 from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
@@ -968,12 +969,14 @@ def enroll_art_students_live():
 
 def enroll_orientation_students():
     ori_shell_id = "15924"    #   2023 orientation shell   #      2022: "9768"
-    the_semester = "202330"
 
-    users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
     users_in_ori_shell = set( \
             [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
 
+    for the_semester in ["202350", "202370"]:
+        #the_semester = "202350"   # "202350"
+
+        users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
         print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
         print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
 
@@ -1006,7 +1009,7 @@ def enroll_orientation_students():
                 time.sleep(0.400)
             except Exception as e:
                 print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
-    return (eee,uuu)
+    # return (eee,uuu)
 
 def enroll_o_s_students():
     #full_reload()
@@ -1475,9 +1478,6 @@ def quick_sem_course_list(term=180):
         print(C['name'])
     
 
-from datetime import datetime   
-import pytz
-
 
 def create_calendar_event():
     events = codecs.open('cache/events.csv','r','utf-8').readlines()

localcache.py

@@ -1281,28 +1281,33 @@ GROUP BY u.canvasid"""  % sem             ## AND e."type"="StudentEnrollment"
     
 
 # Everyone whose first semester is .....
-def users_new_this_semester(sem=''):
+def users_new_this_semester(sem=""):
-    if not sem:
+    if not len(sem):
         sem = input("which semester? (ex: 202150) ")
     users_to_enroll = set()
 
+    where1 = "c.sis LIKE '%s-%%'" % sem
+    where2 = "c.sis NOT LIKE '%s-%%'" % sem
+
+    
+
     q = """SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e
 JOIN users AS u ON e.user_id=u.id
 JOIN courses AS c ON e.course_id=c.id
-WHERE c.sis LIKE "%s-%%" 
+WHERE %s
 AND e.workflow="active"
 AND e."type"="StudentEnrollment"
 AND u.canvasid NOT IN (
 	SELECT u.canvasid FROM enrollment AS e
 	JOIN users AS u ON e.user_id=u.id
 	JOIN courses AS c ON e.course_id=c.id
-	WHERE c.sis NOT LIKE "%s-%%" 
+	WHERE %s
 	AND e.workflow="active"
 	AND e."type"="StudentEnrollment"
 	GROUP BY u.canvasid
 )
 GROUP BY u.canvasid
-ORDER BY num DESC, u.sortablename""" % (sem,sem)
+ORDER BY num DESC, u.sortablename""" % (where1,where2)
 
 
     (connection,cursor) = db()

outcomes2022.py

@@ -30,21 +30,35 @@ NUM_THREADS = 20
 get_fresh = 0
 sem_courses = getCoursesInTerm(178,get_fresh)
 
+def escape_commas(s):
+    if ',' in s:
+        return '"' + s.replace('"', '""') + '"'
+    else:
+        return s
+
 # shorter list for test?
 #sem_courses = sem_courses[:50]
 
 print("Got %i courses in current semester." % len(sem_courses))
 
+outputfile = codecs.open('cache/slo/outcomes2022.output.txt','w','utf-8')
+outputfile.write( "coursename,assessed,courseid,outcome_id,points,title,displayname,description,guid\n")
 
 def course_slo_getter(q):
     (name,id) = q
     info = {'ilearnname':name,'ilearnid':id}
     print("  + Thread getting %s %s" % (str(name),str(id)))
+
+    # Get GROUPS for a course
     u1 = url + "/api/v1/courses/%s/outcome_groups" % str(id)
     og_for_course = fetch(u1)
     if len(og_for_course):
+
+        # There is a GROUP...
         for og in og_for_course:
             if "outcomes_url" in og:
+
+                # There are OUTCOMES...
                 outcomes = fetch(url + og["outcomes_url"])
                 og['outcomes'] = outcomes 
                 og['full_outcomes'] = {}
@@ -52,6 +66,13 @@ def course_slo_getter(q):
                     print(" -> " + url + oo['outcome']['url'])
                     this_outcome = fetch( url + oo['outcome']['url'] )
                     og['full_outcomes'][this_outcome['id']] = this_outcome
+                    saveme = [name, this_outcome['assessed'], id, this_outcome['id'], this_outcome['points_possible'], this_outcome['title'], this_outcome['display_name'], this_outcome['description'], this_outcome['vendor_guid'] ]
+                    saveme2 = [escape_commas(str(x)) for x in saveme]
+                    outputfile.write(",".join(saveme2) + "\n")
+                    outputfile.flush()
+    if type(og_for_course) == list:
+        og_for_course.insert(0,info)
+    else:
         og_for_course.update(info)
     print("  - Thread %s DONE" % str(id))
     return og_for_course
@@ -127,7 +148,7 @@ def ilearn_shell_slo_to_csv(shell_slos):
         
         
         
-ilearn_shell_slo_to_csv(output)
+#ilearn_shell_slo_to_csv(output)
 
         
         

pipelines.py

@@ -1067,7 +1067,7 @@ def recent_schedules():
 
 
 # Take the generically named rosters uploads files and move them to a semester folder and give them a date.    
-def move_to_folder(sem,year,folder):
+def move_to_folder(sem,year,folder,files):
     semester = year+sem
     semester_path = 'cache/rosters/%s' % semester
     if not os.path.isdir('cache/rosters/'+semester):
@@ -1077,8 +1077,11 @@ def move_to_folder(sem,year,folder):
     if not os.path.isdir(semester_path):
         print("+  Creating folder: %s" % semester_path)
         os.makedirs(semester_path)
+    if 'courses.csv' in files:
         os.rename('cache/rosters/courses-%s.csv' % folder,     'cache/rosters/%s/courses.%s.csv' % (semester,now))
+    if 'enrollments.csv' in files:
         os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now))
+    if 'users.csv' in files:
         os.rename('cache/rosters/users-%s.csv' % folder,       'cache/rosters/%s/users.%s.csv' % (semester,now))
     
 
@@ -1152,30 +1155,33 @@ def fetch_current_rosters():
         print("--> %s I see these files at instructure ftp site: " % dt_label )
         [print("   %s" % f) for f in files]
         i = 0
-        got_courses = 0
+        seen_files = []
         if len(files)>0:  # and 'users.csv' in files:
             try:
                 if 'users.csv' in files:
                     sftp.get('users.csv','cache/rosters/users-'+dt_label+'.csv')
                     i += 1
+                    seen_files.append('users.csv')
             except:
                 print(' * users.csv not present')
             try:
                 if 'courses.csv' in files:
                     sftp.get('courses.csv','cache/rosters/courses-'+dt_label+'.csv')
                     i += 1
-                    got_courses = 1
+                    seen_files.append('courses.csv')
             except:
                 print(' * courses.csv not present')
             try:
                if 'enrollments.csv' in files:
                     sftp.get('enrollments.csv','cache/rosters/enrollments-'+dt_label+'.csv')
                     i += 1
+                    seen_files.append('enrollments.csv')
             except:
                 print(' * enrollments.csv not present')
             print('   Saved %i data files in rosters folder.' % i)
             
-            if got_courses:
+            if i:
+                if 'courses.csv' in seen_files:
                     courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
                     courses.readline()
                     a = courses.readline()
@@ -1188,11 +1194,11 @@ def fetch_current_rosters():
                     sem = {'30':'spring', '50':'summer', '70':'fall' }
                     this_sem = sem[ss]
                     print(" -> This semester is: %s, %s" % (this_sem,year))
-                
+                if len(seen_files)==3:
                     print(' -> %s building data file...' % dt_label)
                     convert_roster_files(this_sem,year,dt_label)
                 print(' -> moving files...')
-                move_to_folder(this_sem,year,dt_label)
+                move_to_folder(this_sem,year,dt_label,seen_files)
             else:
                 print(" * No courses file. Not moving files.")
         else:
@@ -1200,14 +1206,14 @@ def fetch_current_rosters():
     sftp.close()
 
 def fetch_current_rosters_auto():
-    
+    fetch_minute = "58"
-    schedule.every().hour.at(":58").do(fetch_current_rosters)
+    schedule.every().hour.at(f":{fetch_minute}").do(fetch_current_rosters)
     
     schedule.every().day.at("12:35").do(sync_non_interactive)
     schedule.every().day.at("21:00").do(sync_non_interactive)
     
     
-    print("running every hour on the :57\n")
+    print(f"running every hour on the :{fetch_minute}\n")
     while True:
         try:
             schedule.run_pending()
@@ -1219,7 +1225,7 @@ def fetch_current_rosters_auto():
             ff.write(traceback.format_exc()+"\n---------\n\n")
             ff.close()
             #schedule.CancelJob
-        time.sleep(15)
+        time.sleep(1)
 
 
 

server.py

@@ -188,6 +188,7 @@ def homepage():
             a('Pi frame on','/displaypi/on') + br + br + \
             a('Pi frame off','/displaypi/off') + br + br + \
             a('Knowledge Base','/x/writing/index') + br + \
+            a('Gav web mirror','/mirror') + br + \
             a('Graph of users','/x/user/1') + "<br />" + \
             a('Courses in a dept','/x/dept/csis') + "<br />" + \
             a('People in a course','/x/roster/10633') + br + br + \