diff --git a/courses.py b/courses.py index 1c924a5..9675e80 100644 --- a/courses.py +++ b/courses.py @@ -261,24 +261,24 @@ def users_in_depts_live(depts=[], termid='171'): -def course_enrollment(id=''): - print("Getting enrollments for course id %s" % str(id)) +def course_enrollment(id='', verbose=0): + if verbose: print("Getting enrollments for course id %s" % str(id)) if not id: id = input('Course id? ') t = url + '/api/v1/courses/%s/enrollments?role[]=StudentEnrollment' % str(id) - print(t) - emts = fetch(t,0) - print(emts) + if verbose: print(t) + emts = fetch(t,verbose) + if verbose: print(emts) emt_by_id = {} for E in emts: - print(E) + if verbose: print(E) try: emt_by_id[E['user_id']] = E except Exception as exp: - print("Skipped that class with this exception: %s" % str(exp)) + print("Skipped [%s] with this exception: %s" % (str(E), str(exp))) ff = codecs.open('cache/courses/%s.json' % str(id), 'w', 'utf-8') ff.write(json.dumps(emt_by_id, indent=2)) - print( " %i results" % len(emts) ) + if verbose: print( " %i results" % len(emts) ) return emt_by_id diff --git a/localcache.py b/localcache.py index 6a93ab0..d074fd6 100644 --- a/localcache.py +++ b/localcache.py @@ -1221,6 +1221,19 @@ AND e.workflow="active" """ % (canvasid) return [a,b] return [x[0] for x in allrows] +# get enrollments in a course, name and canvasid +def get_course_enrollments(courseid): + q = """SELECT u.name, u.canvasid FROM courses AS c +JOIN enrollment AS e ON e.course_id=c.id +JOIN users AS u ON u.id=e.user_id +WHERE c.canvasid=%s +AND e.type="StudentEnrollment" +AND e.workflow="active";""" % str(courseid) + (connection,cursor) = db() + cursor.execute(q) + allrows = cursor.fetchall() + return allrows + # get teacher name from local db def course_quick_stats(canvasid): diff --git a/stats.py b/stats.py index d79ede0..97e500b 100644 --- a/stats.py +++ b/stats.py @@ -18,13 +18,22 @@ - Check if grades were used and make sense - Compute mean, % > 70, median, etc. +- Anonymization steps + - replace teacher names w/ id number + - replace student names w/ id number + - replace course names w/ course code + - Script 2 - given all semester schedules, generate lists of: - CRNs which are online, online live, hybrid, inperson, excluded - CRNs in which teacher and course have passed pocr (and semester is greater than their pass date) - CRNs in which teacher passed pocr for a different course (and semester is greater than their pass date) - CRNs to exclude, for example SP20, because of covid. Possibly SU20 and FA20 + + - CRNs with are POCR approved - CRNs in which teacher has done more than the minimum training in online teaching + - Student ids which have participated in the online orientation over a certain threshold + - Next steps: generate the x-reference for what categories teachers are in, and integrate into the main data file. @@ -32,6 +41,21 @@ - """ +import codecs, os +import json, csv, requests, sys, re +from multiprocessing import Semaphore +from statistics import mean, median, stdev +from pipelines import fetch, url +from courses import getCoursesInTerm, course_enrollment +from localcache import get_course_enrollments +from collections import defaultdict + +all_grades_file = f"cache/grades_all.csv" +all_courses_file = f"cache/course_grades_all.csv" +all_courses_file2 = f"cache/course_grades_compact.csv" +all_courses_file3 = f"cache/course_grades_full.csv" +student_orientation_participation = f'cache/participation_orientation_courses.json' + def num(s): @@ -41,17 +65,7 @@ def num(s): except ValueError: return float(s) -import json, csv, requests, sys, re -from multiprocessing import Semaphore -from statistics import mean, median, stdev -from pipelines import fetch, url -from courses import getCoursesInTerm -from collections import defaultdict -all_grades_file = f"cache/grades_all.csv" -all_courses_file = f"cache/course_grades_all.csv" -all_courses_file2 = f"cache/course_grades_compact.csv" -all_courses_file3 = f"cache/course_grades_full.csv" def sem_num_to_code(sem_num): p = re.search(r'^(\d\d\d\d)(\d\d)$', sem_num) @@ -116,9 +130,54 @@ def grades(writer, sem, COURSE_ID, course_code): print("Exception:", e) -schedules = {} +def get_student_orientations(): + courses = {'iLearn Student Orientation 2022':'9768', # 8170 students + 'Kickstart Online Orientation - Transfer':'36', # 6149 + 'Kickstart Online Orientation - New to College':'35', # 5392 + 'LIB732 SP18':'3295', # 2193 + 'LIB732 FA17':'2037', # 1868 + 'LIB732 SP17':'69', # 1645 + 'Kickstart Online Orientation - Returning':'37', # 1463 + 'iLearn Student Orientation 2023':'15924', # 1292 + 'LIB732 SU17':'1439' # 1281 + } -import codecs, os + views_bycourse = {} + all_student_ids = set() + + # get pageviews of each orientation course + for c,i in courses.items(): + print(c) + cache_file_name = f'cache/participation_course_{i}.json' + student_ids = [x[1] for x in get_course_enrollments(i)] + all_student_ids.update(student_ids) + if os.path.exists(cache_file_name): + pv = json.loads(codecs.open(cache_file_name,'r','utf-8').read()) + else: + pv = get_student_page_views(i, student_ids) + codecs.open(cache_file_name,'w','utf-8').write(json.dumps(pv,indent=2)) + views_bycourse[i] = pv + + # add up pageviews for each student + views_bystudent = {} + for student_id in all_student_ids: + views_bystudent[student_id] = sum([views_bycourse[i].get(student_id,0) for i in courses.values()]) + codecs.open(student_orientation_participation,'w','utf-8').write(json.dumps(views_bystudent,indent=2)) + +def get_student_page_views(course_id, student_ids): + page_views = {} + verbose = 0 + + for student_id in student_ids: + a = f'/api/v1/courses/{course_id}/analytics/users/{student_id}/activity' + response = fetch(url + a, verbose) + page_views[student_id] = sum(response.get('page_views', {}).values()) + + if verbose: print(page_views) + return page_views + +schedules = {} +orientations = {} def load_schedules(): global schedules @@ -129,6 +188,13 @@ def load_schedules(): sem = m.group(1) schedules[sem] = json.loads( codecs.open('cache/schedule/' + f, 'r', 'utf-8').read() ) +def load_orientations(): + global orientations + if not orientations: + orientations = json.loads( codecs.open(student_orientation_participation,'r','utf-8').read() ) + return orientations + + def to_crn_fallback(name): #print(name) name = name.lower() @@ -295,7 +361,7 @@ def process_one_course_grades_full(block, out_f, teacher_to_code, course_to_code return try: - # "course_code course pocr_status teacher_code mode student_id scaled_score" + # "course_code course pocr_status orientation_status teacher_code mode student_id scaled_score" (final_mean, final_median, final_stdev, final_min, final_max, final_count) = [round(f(final_scores)) for f in fxns] final_pct_passed = above_70(final_scores, final_max) @@ -308,10 +374,13 @@ def process_one_course_grades_full(block, out_f, teacher_to_code, course_to_code good_code = ilearn_name_to_course_code(course_code) pocr = 1 if lookup_pocr(teacher, good_code, sem2) else 0 + o = load_orientations() + for row in block: student_id = row[3] + orientation = o[student_id] if student_id in o else 0 scaled_score = round(num(row[7]) / final_max, 2) - out_f.writerow([crs_code, good_code, pocr, tch_code, mode, student_id, scaled_score]) + out_f.writerow([crs_code, good_code, pocr, orientation, tch_code, mode, student_id, scaled_score]) print(course_code) except Exception as e: print("Exception:", e) @@ -352,7 +421,7 @@ def process_grades(): out_fullrows = codecs.open(all_courses_file3,'w','utf-8') out_f = csv.writer(out_fullrows) - out_f.writerow("course_code course pocr_status teacher_code mode student_id scaled_score".split(" ")) + out_f.writerow("course_code course pocr_status orientation_status teacher_code mode student_id scaled_score".split(" ")) out_compact = codecs.open(all_courses_file2,'w','utf-8') out_c = csv.writer(out_compact) @@ -393,6 +462,7 @@ if __name__ == "__main__": 2: ['process grades csv file',process_grades] , 3: ['test shortname parse',nametest] , 4: ['test sem codes',codetest] , + 5: ['get student data from orientations', get_student_orientations], } print ('')