diff --git a/courses.py b/courses.py
index 206b37e..1c924a5 100644
--- a/courses.py
+++ b/courses.py
@@ -1,9 +1,10 @@
import json, re, requests, codecs, sys, time, funcy, os
import pandas as pd
+from datetime import datetime
+import pytz
from dateutil import parser
from datetime import datetime
from util import print_table, int_or_zero, float_or_zero
-
from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem
from pipelines import sems
from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
@@ -968,45 +969,47 @@ def enroll_art_students_live():
def enroll_orientation_students():
ori_shell_id = "15924" # 2023 orientation shell # 2022: "9768"
- the_semester = "202330"
- users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB
users_in_ori_shell = set( \
[ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
- print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
- print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
+ for the_semester in ["202350", "202370"]:
+ #the_semester = "202350" # "202350"
- enroll_us = users_to_enroll.difference(users_in_ori_shell)
+ users_to_enroll = users_new_this_semester(the_semester) ### ##### USES LOCAL DB
+ print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
+ print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
- print("\n\nTO ENROLL %s" % str(enroll_us))
- print("%i new users to enroll." % len(enroll_us))
+ enroll_us = users_to_enroll.difference(users_in_ori_shell)
- eee = 0
- uuu = 0
-
- (connection,cursor) = db()
+ print("\n\nTO ENROLL %s" % str(enroll_us))
+ print("%i new users to enroll." % len(enroll_us))
- for j in enroll_us:
- s = ""
- try:
- q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
- cursor.execute(q)
- s = cursor.fetchall()
- if s:
- s = s[0]
- print(" + Enrolling: %s" % s[0])
- t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
- data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
- 'enrollment[enrollment_state]': 'active' }
- #print(data)
- r3 = requests.post(t, headers=header, params=data)
- eee += 1
- #print(r3.text)
- time.sleep(0.400)
- except Exception as e:
- print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
- return (eee,uuu)
+ eee = 0
+ uuu = 0
+
+ (connection,cursor) = db()
+
+ for j in enroll_us:
+ s = ""
+ try:
+ q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+ cursor.execute(q)
+ s = cursor.fetchall()
+ if s:
+ s = s[0]
+ print(" + Enrolling: %s" % s[0])
+ t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
+ data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
+ 'enrollment[enrollment_state]': 'active' }
+ #print(data)
+ r3 = requests.post(t, headers=header, params=data)
+ eee += 1
+ #print(r3.text)
+ time.sleep(0.400)
+ except Exception as e:
+ print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
+ # return (eee,uuu)
def enroll_o_s_students():
#full_reload()
@@ -1475,9 +1478,6 @@ def quick_sem_course_list(term=180):
print(C['name'])
-from datetime import datetime
-import pytz
-
def create_calendar_event():
events = codecs.open('cache/events.csv','r','utf-8').readlines()
diff --git a/localcache.py b/localcache.py
index ee843c3..6a93ab0 100644
--- a/localcache.py
+++ b/localcache.py
@@ -1281,28 +1281,33 @@ GROUP BY u.canvasid""" % sem ## AND e."type"="StudentEnrollment"
# Everyone whose first semester is .....
-def users_new_this_semester(sem=''):
- if not sem:
+def users_new_this_semester(sem=""):
+ if not len(sem):
sem = input("which semester? (ex: 202150) ")
users_to_enroll = set()
+ where1 = "c.sis LIKE '%s-%%'" % sem
+ where2 = "c.sis NOT LIKE '%s-%%'" % sem
+
+
+
q = """SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e
JOIN users AS u ON e.user_id=u.id
JOIN courses AS c ON e.course_id=c.id
-WHERE c.sis LIKE "%s-%%"
+WHERE %s
AND e.workflow="active"
AND e."type"="StudentEnrollment"
AND u.canvasid NOT IN (
SELECT u.canvasid FROM enrollment AS e
JOIN users AS u ON e.user_id=u.id
JOIN courses AS c ON e.course_id=c.id
- WHERE c.sis NOT LIKE "%s-%%"
+ WHERE %s
AND e.workflow="active"
AND e."type"="StudentEnrollment"
GROUP BY u.canvasid
)
GROUP BY u.canvasid
-ORDER BY num DESC, u.sortablename""" % (sem,sem)
+ORDER BY num DESC, u.sortablename""" % (where1,where2)
(connection,cursor) = db()
diff --git a/outcomes2022.py b/outcomes2022.py
index fb679f3..f2ef36e 100644
--- a/outcomes2022.py
+++ b/outcomes2022.py
@@ -30,21 +30,35 @@ NUM_THREADS = 20
get_fresh = 0
sem_courses = getCoursesInTerm(178,get_fresh)
+def escape_commas(s):
+ if ',' in s:
+ return '"' + s.replace('"', '""') + '"'
+ else:
+ return s
+
# shorter list for test?
#sem_courses = sem_courses[:50]
print("Got %i courses in current semester." % len(sem_courses))
+outputfile = codecs.open('cache/slo/outcomes2022.output.txt','w','utf-8')
+outputfile.write( "coursename,assessed,courseid,outcome_id,points,title,displayname,description,guid\n")
def course_slo_getter(q):
(name,id) = q
info = {'ilearnname':name,'ilearnid':id}
print(" + Thread getting %s %s" % (str(name),str(id)))
+
+ # Get GROUPS for a course
u1 = url + "/api/v1/courses/%s/outcome_groups" % str(id)
og_for_course = fetch(u1)
if len(og_for_course):
+
+ # There is a GROUP...
for og in og_for_course:
if "outcomes_url" in og:
+
+ # There are OUTCOMES...
outcomes = fetch(url + og["outcomes_url"])
og['outcomes'] = outcomes
og['full_outcomes'] = {}
@@ -52,7 +66,14 @@ def course_slo_getter(q):
print(" -> " + url + oo['outcome']['url'])
this_outcome = fetch( url + oo['outcome']['url'] )
og['full_outcomes'][this_outcome['id']] = this_outcome
- og_for_course.update(info)
+ saveme = [name, this_outcome['assessed'], id, this_outcome['id'], this_outcome['points_possible'], this_outcome['title'], this_outcome['display_name'], this_outcome['description'], this_outcome['vendor_guid'] ]
+ saveme2 = [escape_commas(str(x)) for x in saveme]
+ outputfile.write(",".join(saveme2) + "\n")
+ outputfile.flush()
+ if type(og_for_course) == list:
+ og_for_course.insert(0,info)
+ else:
+ og_for_course.update(info)
print(" - Thread %s DONE" % str(id))
return og_for_course
@@ -127,7 +148,7 @@ def ilearn_shell_slo_to_csv(shell_slos):
-ilearn_shell_slo_to_csv(output)
+#ilearn_shell_slo_to_csv(output)
\ No newline at end of file
diff --git a/pipelines.py b/pipelines.py
index af436d5..2ddf2e1 100644
--- a/pipelines.py
+++ b/pipelines.py
@@ -1067,7 +1067,7 @@ def recent_schedules():
# Take the generically named rosters uploads files and move them to a semester folder and give them a date.
-def move_to_folder(sem,year,folder):
+def move_to_folder(sem,year,folder,files):
semester = year+sem
semester_path = 'cache/rosters/%s' % semester
if not os.path.isdir('cache/rosters/'+semester):
@@ -1077,9 +1077,12 @@ def move_to_folder(sem,year,folder):
if not os.path.isdir(semester_path):
print("+ Creating folder: %s" % semester_path)
os.makedirs(semester_path)
- os.rename('cache/rosters/courses-%s.csv' % folder, 'cache/rosters/%s/courses.%s.csv' % (semester,now))
- os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now))
- os.rename('cache/rosters/users-%s.csv' % folder, 'cache/rosters/%s/users.%s.csv' % (semester,now))
+ if 'courses.csv' in files:
+ os.rename('cache/rosters/courses-%s.csv' % folder, 'cache/rosters/%s/courses.%s.csv' % (semester,now))
+ if 'enrollments.csv' in files:
+ os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now))
+ if 'users.csv' in files:
+ os.rename('cache/rosters/users-%s.csv' % folder, 'cache/rosters/%s/users.%s.csv' % (semester,now))
@@ -1152,47 +1155,50 @@ def fetch_current_rosters():
print("--> %s I see these files at instructure ftp site: " % dt_label )
[print(" %s" % f) for f in files]
i = 0
- got_courses = 0
+ seen_files = []
if len(files)>0: # and 'users.csv' in files:
try:
if 'users.csv' in files:
sftp.get('users.csv','cache/rosters/users-'+dt_label+'.csv')
i += 1
+ seen_files.append('users.csv')
except:
print(' * users.csv not present')
try:
if 'courses.csv' in files:
sftp.get('courses.csv','cache/rosters/courses-'+dt_label+'.csv')
i += 1
- got_courses = 1
+ seen_files.append('courses.csv')
except:
print(' * courses.csv not present')
try:
if 'enrollments.csv' in files:
sftp.get('enrollments.csv','cache/rosters/enrollments-'+dt_label+'.csv')
i += 1
+ seen_files.append('enrollments.csv')
except:
print(' * enrollments.csv not present')
print(' Saved %i data files in rosters folder.' % i)
- if got_courses:
- courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
- courses.readline()
- a = courses.readline()
- print(a)
- courses.close()
- parts = a.split(',')
- year = parts[1][0:4]
- ss = parts[1][4:6]
- #print parts[1]
- sem = {'30':'spring', '50':'summer', '70':'fall' }
- this_sem = sem[ss]
- print(" -> This semester is: %s, %s" % (this_sem,year))
-
- print(' -> %s building data file...' % dt_label)
- convert_roster_files(this_sem,year,dt_label)
+ if i:
+ if 'courses.csv' in seen_files:
+ courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
+ courses.readline()
+ a = courses.readline()
+ print(a)
+ courses.close()
+ parts = a.split(',')
+ year = parts[1][0:4]
+ ss = parts[1][4:6]
+ #print parts[1]
+ sem = {'30':'spring', '50':'summer', '70':'fall' }
+ this_sem = sem[ss]
+ print(" -> This semester is: %s, %s" % (this_sem,year))
+ if len(seen_files)==3:
+ print(' -> %s building data file...' % dt_label)
+ convert_roster_files(this_sem,year,dt_label)
print(' -> moving files...')
- move_to_folder(this_sem,year,dt_label)
+ move_to_folder(this_sem,year,dt_label,seen_files)
else:
print(" * No courses file. Not moving files.")
else:
@@ -1200,14 +1206,14 @@ def fetch_current_rosters():
sftp.close()
def fetch_current_rosters_auto():
-
- schedule.every().hour.at(":58").do(fetch_current_rosters)
+ fetch_minute = "58"
+ schedule.every().hour.at(f":{fetch_minute}").do(fetch_current_rosters)
schedule.every().day.at("12:35").do(sync_non_interactive)
schedule.every().day.at("21:00").do(sync_non_interactive)
- print("running every hour on the :57\n")
+ print(f"running every hour on the :{fetch_minute}\n")
while True:
try:
schedule.run_pending()
@@ -1219,7 +1225,7 @@ def fetch_current_rosters_auto():
ff.write(traceback.format_exc()+"\n---------\n\n")
ff.close()
#schedule.CancelJob
- time.sleep(15)
+ time.sleep(1)
diff --git a/server.py b/server.py
index c65f3d3..5aaaba7 100644
--- a/server.py
+++ b/server.py
@@ -188,6 +188,7 @@ def homepage():
a('Pi frame on','/displaypi/on') + br + br + \
a('Pi frame off','/displaypi/off') + br + br + \
a('Knowledge Base','/x/writing/index') + br + \
+ a('Gav web mirror','/mirror') + br + \
a('Graph of users','/x/user/1') + "
" + \
a('Courses in a dept','/x/dept/csis') + "
" + \
a('People in a course','/x/roster/10633') + br + br + \