updates to stats

courses.py

@@ -261,24 +261,24 @@ def users_in_depts_live(depts=[], termid='171'):
 
 
                 
-def course_enrollment(id=''):
+def course_enrollment(id='', verbose=0):
-    print("Getting enrollments for course id %s" % str(id))
+    if verbose: print("Getting enrollments for course id %s" % str(id))
     if not id:
         id = input('Course id? ')
     t = url + '/api/v1/courses/%s/enrollments?role[]=StudentEnrollment' % str(id)
-    print(t)
+    if verbose: print(t)
-    emts = fetch(t,0)
+    emts = fetch(t,verbose)
-    print(emts)
+    if verbose: print(emts)
     emt_by_id = {}
     for E in emts:
-        print(E)
+        if verbose: print(E)
         try:
             emt_by_id[E['user_id']] = E
         except Exception as exp:
-            print("Skipped that class with this exception: %s" % str(exp))
+            print("Skipped [%s] with this exception: %s" % (str(E), str(exp)))
     ff = codecs.open('cache/courses/%s.json' % str(id), 'w', 'utf-8')
     ff.write(json.dumps(emt_by_id, indent=2))
-    print( "  %i results" % len(emts) )
+    if verbose: print( "  %i results" % len(emts) )
     return emt_by_id
 
 

localcache.py

@@ -1221,6 +1221,19 @@ AND e.workflow="active" """ % (canvasid)
     return [a,b]
     return [x[0] for x in allrows]
 
+# get enrollments in a course, name and canvasid
+def get_course_enrollments(courseid):
+    q = """SELECT u.name, u.canvasid FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.canvasid=%s
+AND e.type="StudentEnrollment"
+AND e.workflow="active";""" % str(courseid)
+    (connection,cursor) = db()
+    cursor.execute(q)
+    allrows = cursor.fetchall()
+    return allrows
+
 
 # get teacher name from local db
 def course_quick_stats(canvasid):

stats.py

@@ -18,13 +18,22 @@
     - Check if grades were used and make sense
     - Compute mean, % > 70, median, etc.
 
+- Anonymization steps
+    - replace teacher names w/ id number
+    - replace student names w/ id number
+    - replace course names w/ course code
+
 - Script 2 - given all semester schedules, generate lists of:
     - CRNs which are online, online live, hybrid, inperson, excluded
     - CRNs in which teacher and course have passed pocr (and semester is greater than their pass date)
     - CRNs in which teacher passed pocr for a different course (and semester is greater than their pass date)
     - CRNs to exclude, for example SP20, because of covid. Possibly SU20 and FA20
+
+    - CRNs with are POCR approved
     - CRNs in which teacher has done more than the minimum training in online teaching
 
+    - Student ids which have participated in the online orientation over a certain threshold
+
 - Next steps: generate the x-reference for what categories teachers are in, and
   integrate into the main data file.
 
@@ -32,6 +41,21 @@
 
 - 
 """
+import codecs, os
+import json, csv, requests, sys, re
+from multiprocessing import Semaphore
+from statistics import mean, median, stdev
+from pipelines import fetch, url
+from courses import getCoursesInTerm, course_enrollment
+from localcache import get_course_enrollments
+from collections import defaultdict
+
+all_grades_file = f"cache/grades_all.csv"
+all_courses_file = f"cache/course_grades_all.csv"
+all_courses_file2 = f"cache/course_grades_compact.csv"
+all_courses_file3 = f"cache/course_grades_full.csv"
+student_orientation_participation = f'cache/participation_orientation_courses.json'
+
 
 
 def num(s):
@@ -41,17 +65,7 @@ def num(s):
     except ValueError:
         return float(s)
 
-import json, csv, requests, sys, re
-from multiprocessing import Semaphore
-from statistics import mean, median, stdev
-from pipelines import fetch, url
-from courses import getCoursesInTerm
-from collections import defaultdict
 
-all_grades_file = f"cache/grades_all.csv"
-all_courses_file = f"cache/course_grades_all.csv"
-all_courses_file2 = f"cache/course_grades_compact.csv"
-all_courses_file3 = f"cache/course_grades_full.csv"
 
 def sem_num_to_code(sem_num):
     p = re.search(r'^(\d\d\d\d)(\d\d)$', sem_num)
@@ -116,9 +130,54 @@ def grades(writer, sem, COURSE_ID, course_code):
             print("Exception:", e)
                     
 
-schedules = {}
+def get_student_orientations():
+    courses = {'iLearn Student Orientation 2022':'9768',                # 8170 students
+               'Kickstart Online Orientation - Transfer':'36',          # 6149
+               'Kickstart Online Orientation - New to College':'35',    # 5392
+               'LIB732 SP18':'3295',                                    # 2193
+               'LIB732 FA17':'2037',                                    # 1868
+               'LIB732 SP17':'69',                                      # 1645
+               'Kickstart Online Orientation - Returning':'37',         # 1463
+               'iLearn Student Orientation 2023':'15924',               # 1292
+               'LIB732 SU17':'1439'                                     # 1281
+    }
 
-import codecs, os
+    views_bycourse = {}
+    all_student_ids = set()
+
+    # get pageviews of each orientation course
+    for c,i in courses.items():
+        print(c)
+        cache_file_name = f'cache/participation_course_{i}.json'
+        student_ids = [x[1] for x in get_course_enrollments(i)]
+        all_student_ids.update(student_ids)
+        if os.path.exists(cache_file_name):
+            pv = json.loads(codecs.open(cache_file_name,'r','utf-8').read())
+        else:
+            pv = get_student_page_views(i, student_ids)
+            codecs.open(cache_file_name,'w','utf-8').write(json.dumps(pv,indent=2))
+        views_bycourse[i] = pv
+    
+    # add up pageviews for each student
+    views_bystudent = {}
+    for student_id in all_student_ids:
+        views_bystudent[student_id] = sum([views_bycourse[i].get(student_id,0) for i in courses.values()])
+    codecs.open(student_orientation_participation,'w','utf-8').write(json.dumps(views_bystudent,indent=2))
+    
+def get_student_page_views(course_id, student_ids):
+    page_views = {}
+    verbose = 0
+
+    for student_id in student_ids:
+        a = f'/api/v1/courses/{course_id}/analytics/users/{student_id}/activity'
+        response = fetch(url + a, verbose)
+        page_views[student_id] = sum(response.get('page_views', {}).values())
+
+    if verbose: print(page_views)
+    return page_views
+
+schedules = {}
+orientations = {}
 
 def load_schedules():
     global schedules
@@ -129,6 +188,13 @@ def load_schedules():
                 sem = m.group(1)
                 schedules[sem] = json.loads( codecs.open('cache/schedule/' + f, 'r', 'utf-8').read() )
 
+def load_orientations():
+    global orientations
+    if not orientations:
+        orientations = json.loads( codecs.open(student_orientation_participation,'r','utf-8').read() )
+    return orientations
+
+
 def to_crn_fallback(name):
     #print(name)
     name = name.lower()
@@ -295,7 +361,7 @@ def process_one_course_grades_full(block, out_f, teacher_to_code, course_to_code
         return
     try:
 
-        # "course_code course pocr_status teacher_code mode student_id scaled_score"
+        # "course_code course pocr_status orientation_status teacher_code mode student_id scaled_score"
 
         (final_mean, final_median, final_stdev, final_min, final_max, final_count) = [round(f(final_scores)) for f in fxns]
         final_pct_passed = above_70(final_scores, final_max)
@@ -308,10 +374,13 @@ def process_one_course_grades_full(block, out_f, teacher_to_code, course_to_code
         good_code = ilearn_name_to_course_code(course_code)
         pocr = 1 if lookup_pocr(teacher, good_code, sem2) else 0
 
+        o = load_orientations()
+
         for row in block:
             student_id = row[3]
+            orientation = o[student_id] if student_id in o else 0
             scaled_score = round(num(row[7]) / final_max, 2)
-            out_f.writerow([crs_code, good_code, pocr, tch_code, mode, student_id, scaled_score])
+            out_f.writerow([crs_code, good_code, pocr, orientation, tch_code, mode, student_id, scaled_score])
         print(course_code)
     except Exception as e:
         print("Exception:", e)
@@ -352,7 +421,7 @@ def process_grades():
 
     out_fullrows = codecs.open(all_courses_file3,'w','utf-8')
     out_f = csv.writer(out_fullrows)
-    out_f.writerow("course_code course pocr_status teacher_code mode student_id scaled_score".split(" "))
+    out_f.writerow("course_code course pocr_status orientation_status teacher_code mode student_id scaled_score".split(" "))
 
     out_compact = codecs.open(all_courses_file2,'w','utf-8')
     out_c = csv.writer(out_compact)
@@ -393,6 +462,7 @@ if __name__ == "__main__":
                 2: ['process grades csv file',process_grades] , 
                 3: ['test shortname parse',nametest] , 
                 4: ['test sem codes',codetest] , 
+                5: ['get student data from orientations', get_student_orientations],
               }
     print ('')