updates to web mir srvr, outcomes, oritn shell fil
This commit is contained in:
Coding with Peter
parent
d5ae9960f3
commit
8eb91d1621
68
courses.py
68
courses.py
|
|
@ -1,9 +1,10 @@
|
|||
import json, re, requests, codecs, sys, time, funcy, os
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
from dateutil import parser
|
||||
from datetime import datetime
|
||||
from util import print_table, int_or_zero, float_or_zero
|
||||
|
||||
from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem
|
||||
from pipelines import sems
|
||||
from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
|
||||
|
|
@ -968,45 +969,47 @@ def enroll_art_students_live():
|
|||
|
||||
def enroll_orientation_students():
|
||||
ori_shell_id = "15924" # 2023 orientation shell # 2022: "9768"
|
||||
the_semester = "202330"
|
||||
|
||||
users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB
|
||||
users_in_ori_shell = set( \
|
||||
[ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
|
||||
|
||||
print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
|
||||
print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
|
||||
for the_semester in ["202350", "202370"]:
|
||||
#the_semester = "202350" # "202350"
|
||||
|
||||
enroll_us = users_to_enroll.difference(users_in_ori_shell)
|
||||
users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB
|
||||
print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
|
||||
print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
|
||||
|
||||
print("\n\nTO ENROLL %s" % str(enroll_us))
|
||||
print("%i new users to enroll." % len(enroll_us))
|
||||
enroll_us = users_to_enroll.difference(users_in_ori_shell)
|
||||
|
||||
eee = 0
|
||||
uuu = 0
|
||||
print("\n\nTO ENROLL %s" % str(enroll_us))
|
||||
print("%i new users to enroll." % len(enroll_us))
|
||||
|
||||
(connection,cursor) = db()
|
||||
eee = 0
|
||||
uuu = 0
|
||||
|
||||
for j in enroll_us:
|
||||
s = ""
|
||||
try:
|
||||
q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
|
||||
cursor.execute(q)
|
||||
s = cursor.fetchall()
|
||||
if s:
|
||||
s = s[0]
|
||||
print(" + Enrolling: %s" % s[0])
|
||||
t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
|
||||
data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
|
||||
'enrollment[enrollment_state]': 'active' }
|
||||
#print(data)
|
||||
r3 = requests.post(t, headers=header, params=data)
|
||||
eee += 1
|
||||
#print(r3.text)
|
||||
time.sleep(0.400)
|
||||
except Exception as e:
|
||||
print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
|
||||
return (eee,uuu)
|
||||
(connection,cursor) = db()
|
||||
|
||||
for j in enroll_us:
|
||||
s = ""
|
||||
try:
|
||||
q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
|
||||
cursor.execute(q)
|
||||
s = cursor.fetchall()
|
||||
if s:
|
||||
s = s[0]
|
||||
print(" + Enrolling: %s" % s[0])
|
||||
t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
|
||||
data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
|
||||
'enrollment[enrollment_state]': 'active' }
|
||||
#print(data)
|
||||
r3 = requests.post(t, headers=header, params=data)
|
||||
eee += 1
|
||||
#print(r3.text)
|
||||
time.sleep(0.400)
|
||||
except Exception as e:
|
||||
print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
|
||||
# return (eee,uuu)
|
||||
|
||||
def enroll_o_s_students():
|
||||
#full_reload()
|
||||
|
|
@ -1475,9 +1478,6 @@ def quick_sem_course_list(term=180):
|
|||
print(C['name'])
|
||||
|
||||
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
|
||||
|
||||
def create_calendar_event():
|
||||
events = codecs.open('cache/events.csv','r','utf-8').readlines()
|
||||
|
|
|
|||
|
|
@ -1281,28 +1281,33 @@ GROUP BY u.canvasid""" % sem ## AND e."type"="StudentEnrollment"
|
|||
|
||||
|
||||
# Everyone whose first semester is .....
|
||||
def users_new_this_semester(sem=''):
|
||||
if not sem:
|
||||
def users_new_this_semester(sem=""):
|
||||
if not len(sem):
|
||||
sem = input("which semester? (ex: 202150) ")
|
||||
users_to_enroll = set()
|
||||
|
||||
where1 = "c.sis LIKE '%s-%%'" % sem
|
||||
where2 = "c.sis NOT LIKE '%s-%%'" % sem
|
||||
|
||||
|
||||
|
||||
q = """SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e
|
||||
JOIN users AS u ON e.user_id=u.id
|
||||
JOIN courses AS c ON e.course_id=c.id
|
||||
WHERE c.sis LIKE "%s-%%"
|
||||
WHERE %s
|
||||
AND e.workflow="active"
|
||||
AND e."type"="StudentEnrollment"
|
||||
AND u.canvasid NOT IN (
|
||||
SELECT u.canvasid FROM enrollment AS e
|
||||
JOIN users AS u ON e.user_id=u.id
|
||||
JOIN courses AS c ON e.course_id=c.id
|
||||
WHERE c.sis NOT LIKE "%s-%%"
|
||||
WHERE %s
|
||||
AND e.workflow="active"
|
||||
AND e."type"="StudentEnrollment"
|
||||
GROUP BY u.canvasid
|
||||
)
|
||||
GROUP BY u.canvasid
|
||||
ORDER BY num DESC, u.sortablename""" % (sem,sem)
|
||||
ORDER BY num DESC, u.sortablename""" % (where1,where2)
|
||||
|
||||
|
||||
(connection,cursor) = db()
|
||||
|
|
|
|||
|
|
@ -30,21 +30,35 @@ NUM_THREADS = 20
|
|||
get_fresh = 0
|
||||
sem_courses = getCoursesInTerm(178,get_fresh)
|
||||
|
||||
def escape_commas(s):
|
||||
if ',' in s:
|
||||
return '"' + s.replace('"', '""') + '"'
|
||||
else:
|
||||
return s
|
||||
|
||||
# shorter list for test?
|
||||
#sem_courses = sem_courses[:50]
|
||||
|
||||
print("Got %i courses in current semester." % len(sem_courses))
|
||||
|
||||
outputfile = codecs.open('cache/slo/outcomes2022.output.txt','w','utf-8')
|
||||
outputfile.write( "coursename,assessed,courseid,outcome_id,points,title,displayname,description,guid\n")
|
||||
|
||||
def course_slo_getter(q):
|
||||
(name,id) = q
|
||||
info = {'ilearnname':name,'ilearnid':id}
|
||||
print(" + Thread getting %s %s" % (str(name),str(id)))
|
||||
|
||||
# Get GROUPS for a course
|
||||
u1 = url + "/api/v1/courses/%s/outcome_groups" % str(id)
|
||||
og_for_course = fetch(u1)
|
||||
if len(og_for_course):
|
||||
|
||||
# There is a GROUP...
|
||||
for og in og_for_course:
|
||||
if "outcomes_url" in og:
|
||||
|
||||
# There are OUTCOMES...
|
||||
outcomes = fetch(url + og["outcomes_url"])
|
||||
og['outcomes'] = outcomes
|
||||
og['full_outcomes'] = {}
|
||||
|
|
@ -52,7 +66,14 @@ def course_slo_getter(q):
|
|||
print(" -> " + url + oo['outcome']['url'])
|
||||
this_outcome = fetch( url + oo['outcome']['url'] )
|
||||
og['full_outcomes'][this_outcome['id']] = this_outcome
|
||||
og_for_course.update(info)
|
||||
saveme = [name, this_outcome['assessed'], id, this_outcome['id'], this_outcome['points_possible'], this_outcome['title'], this_outcome['display_name'], this_outcome['description'], this_outcome['vendor_guid'] ]
|
||||
saveme2 = [escape_commas(str(x)) for x in saveme]
|
||||
outputfile.write(",".join(saveme2) + "\n")
|
||||
outputfile.flush()
|
||||
if type(og_for_course) == list:
|
||||
og_for_course.insert(0,info)
|
||||
else:
|
||||
og_for_course.update(info)
|
||||
print(" - Thread %s DONE" % str(id))
|
||||
return og_for_course
|
||||
|
||||
|
|
@ -127,7 +148,7 @@ def ilearn_shell_slo_to_csv(shell_slos):
|
|||
|
||||
|
||||
|
||||
ilearn_shell_slo_to_csv(output)
|
||||
#ilearn_shell_slo_to_csv(output)
|
||||
|
||||
|
||||
|
||||
60
pipelines.py
60
pipelines.py
|
|
@ -1067,7 +1067,7 @@ def recent_schedules():
|
|||
|
||||
|
||||
# Take the generically named rosters uploads files and move them to a semester folder and give them a date.
|
||||
def move_to_folder(sem,year,folder):
|
||||
def move_to_folder(sem,year,folder,files):
|
||||
semester = year+sem
|
||||
semester_path = 'cache/rosters/%s' % semester
|
||||
if not os.path.isdir('cache/rosters/'+semester):
|
||||
|
|
@ -1077,9 +1077,12 @@ def move_to_folder(sem,year,folder):
|
|||
if not os.path.isdir(semester_path):
|
||||
print("+ Creating folder: %s" % semester_path)
|
||||
os.makedirs(semester_path)
|
||||
os.rename('cache/rosters/courses-%s.csv' % folder, 'cache/rosters/%s/courses.%s.csv' % (semester,now))
|
||||
os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now))
|
||||
os.rename('cache/rosters/users-%s.csv' % folder, 'cache/rosters/%s/users.%s.csv' % (semester,now))
|
||||
if 'courses.csv' in files:
|
||||
os.rename('cache/rosters/courses-%s.csv' % folder, 'cache/rosters/%s/courses.%s.csv' % (semester,now))
|
||||
if 'enrollments.csv' in files:
|
||||
os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now))
|
||||
if 'users.csv' in files:
|
||||
os.rename('cache/rosters/users-%s.csv' % folder, 'cache/rosters/%s/users.%s.csv' % (semester,now))
|
||||
|
||||
|
||||
|
||||
|
|
@ -1152,47 +1155,50 @@ def fetch_current_rosters():
|
|||
print("--> %s I see these files at instructure ftp site: " % dt_label )
|
||||
[print(" %s" % f) for f in files]
|
||||
i = 0
|
||||
got_courses = 0
|
||||
seen_files = []
|
||||
if len(files)>0: # and 'users.csv' in files:
|
||||
try:
|
||||
if 'users.csv' in files:
|
||||
sftp.get('users.csv','cache/rosters/users-'+dt_label+'.csv')
|
||||
i += 1
|
||||
seen_files.append('users.csv')
|
||||
except:
|
||||
print(' * users.csv not present')
|
||||
try:
|
||||
if 'courses.csv' in files:
|
||||
sftp.get('courses.csv','cache/rosters/courses-'+dt_label+'.csv')
|
||||
i += 1
|
||||
got_courses = 1
|
||||
seen_files.append('courses.csv')
|
||||
except:
|
||||
print(' * courses.csv not present')
|
||||
try:
|
||||
if 'enrollments.csv' in files:
|
||||
sftp.get('enrollments.csv','cache/rosters/enrollments-'+dt_label+'.csv')
|
||||
i += 1
|
||||
seen_files.append('enrollments.csv')
|
||||
except:
|
||||
print(' * enrollments.csv not present')
|
||||
print(' Saved %i data files in rosters folder.' % i)
|
||||
|
||||
if got_courses:
|
||||
courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
|
||||
courses.readline()
|
||||
a = courses.readline()
|
||||
print(a)
|
||||
courses.close()
|
||||
parts = a.split(',')
|
||||
year = parts[1][0:4]
|
||||
ss = parts[1][4:6]
|
||||
#print parts[1]
|
||||
sem = {'30':'spring', '50':'summer', '70':'fall' }
|
||||
this_sem = sem[ss]
|
||||
print(" -> This semester is: %s, %s" % (this_sem,year))
|
||||
|
||||
print(' -> %s building data file...' % dt_label)
|
||||
convert_roster_files(this_sem,year,dt_label)
|
||||
if i:
|
||||
if 'courses.csv' in seen_files:
|
||||
courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
|
||||
courses.readline()
|
||||
a = courses.readline()
|
||||
print(a)
|
||||
courses.close()
|
||||
parts = a.split(',')
|
||||
year = parts[1][0:4]
|
||||
ss = parts[1][4:6]
|
||||
#print parts[1]
|
||||
sem = {'30':'spring', '50':'summer', '70':'fall' }
|
||||
this_sem = sem[ss]
|
||||
print(" -> This semester is: %s, %s" % (this_sem,year))
|
||||
if len(seen_files)==3:
|
||||
print(' -> %s building data file...' % dt_label)
|
||||
convert_roster_files(this_sem,year,dt_label)
|
||||
print(' -> moving files...')
|
||||
move_to_folder(this_sem,year,dt_label)
|
||||
move_to_folder(this_sem,year,dt_label,seen_files)
|
||||
else:
|
||||
print(" * No courses file. Not moving files.")
|
||||
else:
|
||||
|
|
@ -1200,14 +1206,14 @@ def fetch_current_rosters():
|
|||
sftp.close()
|
||||
|
||||
def fetch_current_rosters_auto():
|
||||
|
||||
schedule.every().hour.at(":58").do(fetch_current_rosters)
|
||||
fetch_minute = "58"
|
||||
schedule.every().hour.at(f":{fetch_minute}").do(fetch_current_rosters)
|
||||
|
||||
schedule.every().day.at("12:35").do(sync_non_interactive)
|
||||
schedule.every().day.at("21:00").do(sync_non_interactive)
|
||||
|
||||
|
||||
print("running every hour on the :57\n")
|
||||
print(f"running every hour on the :{fetch_minute}\n")
|
||||
while True:
|
||||
try:
|
||||
schedule.run_pending()
|
||||
|
|
@ -1219,7 +1225,7 @@ def fetch_current_rosters_auto():
|
|||
ff.write(traceback.format_exc()+"\n---------\n\n")
|
||||
ff.close()
|
||||
#schedule.CancelJob
|
||||
time.sleep(15)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@ def homepage():
|
|||
a('Pi frame on','/displaypi/on') + br + br + \
|
||||
a('Pi frame off','/displaypi/off') + br + br + \
|
||||
a('Knowledge Base','/x/writing/index') + br + \
|
||||
a('Gav web mirror','/mirror') + br + \
|
||||
a('Graph of users','/x/user/1') + "<br />" + \
|
||||
a('Courses in a dept','/x/dept/csis') + "<br />" + \
|
||||
a('People in a course','/x/roster/10633') + br + br + \
|
||||
|
|
|
|||
Loading…
Reference in New Issue