From 8eb91d162115d83cfd81d942717dadad55080ed5 Mon Sep 17 00:00:00 2001 From: Coding with Peter Date: Tue, 9 May 2023 09:07:36 -0700 Subject: [PATCH] updates to web mir srvr, outcomes, oritn shell fil --- courses.py | 70 ++++++++++++++++++++++++------------------------- localcache.py | 15 +++++++---- outcomes2022.py | 25 ++++++++++++++++-- pipelines.py | 60 +++++++++++++++++++++++------------------- server.py | 1 + 5 files changed, 102 insertions(+), 69 deletions(-) diff --git a/courses.py b/courses.py index 206b37e..1c924a5 100644 --- a/courses.py +++ b/courses.py @@ -1,9 +1,10 @@ import json, re, requests, codecs, sys, time, funcy, os import pandas as pd +from datetime import datetime +import pytz from dateutil import parser from datetime import datetime from util import print_table, int_or_zero, float_or_zero - from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem from pipelines import sems from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload @@ -968,45 +969,47 @@ def enroll_art_students_live(): def enroll_orientation_students(): ori_shell_id = "15924" # 2023 orientation shell # 2022: "9768" - the_semester = "202330" - users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB users_in_ori_shell = set( \ [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ]) - print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll)) - print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell)) + for the_semester in ["202350", "202370"]: + #the_semester = "202350" # "202350" - enroll_us = users_to_enroll.difference(users_in_ori_shell) + users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB + print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll)) + print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell)) - print("\n\nTO ENROLL %s" % str(enroll_us)) - print("%i new users to enroll." % len(enroll_us)) + enroll_us = users_to_enroll.difference(users_in_ori_shell) - eee = 0 - uuu = 0 - - (connection,cursor) = db() + print("\n\nTO ENROLL %s" % str(enroll_us)) + print("%i new users to enroll." % len(enroll_us)) - for j in enroll_us: - s = "" - try: - q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j - cursor.execute(q) - s = cursor.fetchall() - if s: - s = s[0] - print(" + Enrolling: %s" % s[0]) - t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id - data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment', - 'enrollment[enrollment_state]': 'active' } - #print(data) - r3 = requests.post(t, headers=header, params=data) - eee += 1 - #print(r3.text) - time.sleep(0.400) - except Exception as e: - print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e))) - return (eee,uuu) + eee = 0 + uuu = 0 + + (connection,cursor) = db() + + for j in enroll_us: + s = "" + try: + q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j + cursor.execute(q) + s = cursor.fetchall() + if s: + s = s[0] + print(" + Enrolling: %s" % s[0]) + t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id + data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment', + 'enrollment[enrollment_state]': 'active' } + #print(data) + r3 = requests.post(t, headers=header, params=data) + eee += 1 + #print(r3.text) + time.sleep(0.400) + except Exception as e: + print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e))) + # return (eee,uuu) def enroll_o_s_students(): #full_reload() @@ -1475,9 +1478,6 @@ def quick_sem_course_list(term=180): print(C['name']) -from datetime import datetime -import pytz - def create_calendar_event(): events = codecs.open('cache/events.csv','r','utf-8').readlines() diff --git a/localcache.py b/localcache.py index ee843c3..6a93ab0 100644 --- a/localcache.py +++ b/localcache.py @@ -1281,28 +1281,33 @@ GROUP BY u.canvasid""" % sem ## AND e."type"="StudentEnrollment" # Everyone whose first semester is ..... -def users_new_this_semester(sem=''): - if not sem: +def users_new_this_semester(sem=""): + if not len(sem): sem = input("which semester? (ex: 202150) ") users_to_enroll = set() + where1 = "c.sis LIKE '%s-%%'" % sem + where2 = "c.sis NOT LIKE '%s-%%'" % sem + + + q = """SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e JOIN users AS u ON e.user_id=u.id JOIN courses AS c ON e.course_id=c.id -WHERE c.sis LIKE "%s-%%" +WHERE %s AND e.workflow="active" AND e."type"="StudentEnrollment" AND u.canvasid NOT IN ( SELECT u.canvasid FROM enrollment AS e JOIN users AS u ON e.user_id=u.id JOIN courses AS c ON e.course_id=c.id - WHERE c.sis NOT LIKE "%s-%%" + WHERE %s AND e.workflow="active" AND e."type"="StudentEnrollment" GROUP BY u.canvasid ) GROUP BY u.canvasid -ORDER BY num DESC, u.sortablename""" % (sem,sem) +ORDER BY num DESC, u.sortablename""" % (where1,where2) (connection,cursor) = db() diff --git a/outcomes2022.py b/outcomes2022.py index fb679f3..f2ef36e 100644 --- a/outcomes2022.py +++ b/outcomes2022.py @@ -30,21 +30,35 @@ NUM_THREADS = 20 get_fresh = 0 sem_courses = getCoursesInTerm(178,get_fresh) +def escape_commas(s): + if ',' in s: + return '"' + s.replace('"', '""') + '"' + else: + return s + # shorter list for test? #sem_courses = sem_courses[:50] print("Got %i courses in current semester." % len(sem_courses)) +outputfile = codecs.open('cache/slo/outcomes2022.output.txt','w','utf-8') +outputfile.write( "coursename,assessed,courseid,outcome_id,points,title,displayname,description,guid\n") def course_slo_getter(q): (name,id) = q info = {'ilearnname':name,'ilearnid':id} print(" + Thread getting %s %s" % (str(name),str(id))) + + # Get GROUPS for a course u1 = url + "/api/v1/courses/%s/outcome_groups" % str(id) og_for_course = fetch(u1) if len(og_for_course): + + # There is a GROUP... for og in og_for_course: if "outcomes_url" in og: + + # There are OUTCOMES... outcomes = fetch(url + og["outcomes_url"]) og['outcomes'] = outcomes og['full_outcomes'] = {} @@ -52,7 +66,14 @@ def course_slo_getter(q): print(" -> " + url + oo['outcome']['url']) this_outcome = fetch( url + oo['outcome']['url'] ) og['full_outcomes'][this_outcome['id']] = this_outcome - og_for_course.update(info) + saveme = [name, this_outcome['assessed'], id, this_outcome['id'], this_outcome['points_possible'], this_outcome['title'], this_outcome['display_name'], this_outcome['description'], this_outcome['vendor_guid'] ] + saveme2 = [escape_commas(str(x)) for x in saveme] + outputfile.write(",".join(saveme2) + "\n") + outputfile.flush() + if type(og_for_course) == list: + og_for_course.insert(0,info) + else: + og_for_course.update(info) print(" - Thread %s DONE" % str(id)) return og_for_course @@ -127,7 +148,7 @@ def ilearn_shell_slo_to_csv(shell_slos): -ilearn_shell_slo_to_csv(output) +#ilearn_shell_slo_to_csv(output) \ No newline at end of file diff --git a/pipelines.py b/pipelines.py index af436d5..2ddf2e1 100644 --- a/pipelines.py +++ b/pipelines.py @@ -1067,7 +1067,7 @@ def recent_schedules(): # Take the generically named rosters uploads files and move them to a semester folder and give them a date. -def move_to_folder(sem,year,folder): +def move_to_folder(sem,year,folder,files): semester = year+sem semester_path = 'cache/rosters/%s' % semester if not os.path.isdir('cache/rosters/'+semester): @@ -1077,9 +1077,12 @@ def move_to_folder(sem,year,folder): if not os.path.isdir(semester_path): print("+ Creating folder: %s" % semester_path) os.makedirs(semester_path) - os.rename('cache/rosters/courses-%s.csv' % folder, 'cache/rosters/%s/courses.%s.csv' % (semester,now)) - os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now)) - os.rename('cache/rosters/users-%s.csv' % folder, 'cache/rosters/%s/users.%s.csv' % (semester,now)) + if 'courses.csv' in files: + os.rename('cache/rosters/courses-%s.csv' % folder, 'cache/rosters/%s/courses.%s.csv' % (semester,now)) + if 'enrollments.csv' in files: + os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now)) + if 'users.csv' in files: + os.rename('cache/rosters/users-%s.csv' % folder, 'cache/rosters/%s/users.%s.csv' % (semester,now)) @@ -1152,47 +1155,50 @@ def fetch_current_rosters(): print("--> %s I see these files at instructure ftp site: " % dt_label ) [print(" %s" % f) for f in files] i = 0 - got_courses = 0 + seen_files = [] if len(files)>0: # and 'users.csv' in files: try: if 'users.csv' in files: sftp.get('users.csv','cache/rosters/users-'+dt_label+'.csv') i += 1 + seen_files.append('users.csv') except: print(' * users.csv not present') try: if 'courses.csv' in files: sftp.get('courses.csv','cache/rosters/courses-'+dt_label+'.csv') i += 1 - got_courses = 1 + seen_files.append('courses.csv') except: print(' * courses.csv not present') try: if 'enrollments.csv' in files: sftp.get('enrollments.csv','cache/rosters/enrollments-'+dt_label+'.csv') i += 1 + seen_files.append('enrollments.csv') except: print(' * enrollments.csv not present') print(' Saved %i data files in rosters folder.' % i) - if got_courses: - courses = open('cache/rosters/courses-%s.csv' % dt_label,'r') - courses.readline() - a = courses.readline() - print(a) - courses.close() - parts = a.split(',') - year = parts[1][0:4] - ss = parts[1][4:6] - #print parts[1] - sem = {'30':'spring', '50':'summer', '70':'fall' } - this_sem = sem[ss] - print(" -> This semester is: %s, %s" % (this_sem,year)) - - print(' -> %s building data file...' % dt_label) - convert_roster_files(this_sem,year,dt_label) + if i: + if 'courses.csv' in seen_files: + courses = open('cache/rosters/courses-%s.csv' % dt_label,'r') + courses.readline() + a = courses.readline() + print(a) + courses.close() + parts = a.split(',') + year = parts[1][0:4] + ss = parts[1][4:6] + #print parts[1] + sem = {'30':'spring', '50':'summer', '70':'fall' } + this_sem = sem[ss] + print(" -> This semester is: %s, %s" % (this_sem,year)) + if len(seen_files)==3: + print(' -> %s building data file...' % dt_label) + convert_roster_files(this_sem,year,dt_label) print(' -> moving files...') - move_to_folder(this_sem,year,dt_label) + move_to_folder(this_sem,year,dt_label,seen_files) else: print(" * No courses file. Not moving files.") else: @@ -1200,14 +1206,14 @@ def fetch_current_rosters(): sftp.close() def fetch_current_rosters_auto(): - - schedule.every().hour.at(":58").do(fetch_current_rosters) + fetch_minute = "58" + schedule.every().hour.at(f":{fetch_minute}").do(fetch_current_rosters) schedule.every().day.at("12:35").do(sync_non_interactive) schedule.every().day.at("21:00").do(sync_non_interactive) - print("running every hour on the :57\n") + print(f"running every hour on the :{fetch_minute}\n") while True: try: schedule.run_pending() @@ -1219,7 +1225,7 @@ def fetch_current_rosters_auto(): ff.write(traceback.format_exc()+"\n---------\n\n") ff.close() #schedule.CancelJob - time.sleep(15) + time.sleep(1) diff --git a/server.py b/server.py index c65f3d3..5aaaba7 100644 --- a/server.py +++ b/server.py @@ -188,6 +188,7 @@ def homepage(): a('Pi frame on','/displaypi/on') + br + br + \ a('Pi frame off','/displaypi/off') + br + br + \ a('Knowledge Base','/x/writing/index') + br + \ + a('Gav web mirror','/mirror') + br + \ a('Graph of users','/x/user/1') + "
" + \ a('Courses in a dept','/x/dept/csis') + "
" + \ a('People in a course','/x/roster/10633') + br + br + \