464 lines
16 KiB
Python
464 lines
16 KiB
Python
# Local data, saving and manipulating
|
|
|
|
import os, re, gzip, codecs, funcy, pytz, json, random, functools, requests, sys, csv, time, psycopg2
|
|
import pandas as pd
|
|
import numpy as np
|
|
from collections import defaultdict
|
|
from datetime import datetime as dt
|
|
from datetime import timedelta
|
|
from dateutil.parser import parse
|
|
from os.path import exists, getmtime
|
|
from pipelines import sync_non_interactive, url, header, gp, dean
|
|
from tabulate import tabulate
|
|
|
|
#########
|
|
######### LOCAL DB
|
|
#########
|
|
|
|
CON = ''
|
|
CURSOR = ''
|
|
|
|
def db():
|
|
global CON,CURSOR
|
|
CON = psycopg2.connect(database="db",
|
|
host="192.168.1.6",
|
|
user="postgres",
|
|
password="rolley34",
|
|
port="5432")
|
|
|
|
CURSOR = CON.cursor()
|
|
return CON,CURSOR
|
|
|
|
|
|
'''
|
|
# Help the next function to upload new users directly to conf database on gavilan.
|
|
def employees_refresh_flex(data):
|
|
try:
|
|
data['a'] = 'set/newuser'
|
|
data['sis_user_id'] = data['sis_user_id'][3:]
|
|
print("\nUploading this: \n")
|
|
print(json.dumps(data, indent=2))
|
|
print("\n")
|
|
a = input("Continue (y) or skip (n) ? ")
|
|
if a == 'y':
|
|
# This is what I was missing..........
|
|
# req.add_header("Content-type", "application/x-www-form-urlencoded")
|
|
r3 = requests.post('https://www.gavilan.edu/staff/flex/2020/api.php', params=data)
|
|
print(r3.text)
|
|
#print(r3.headers)
|
|
except Exception as ex:
|
|
print("Failed on: %s\nErr: %s" % (str(data),str(ex)))
|
|
|
|
|
|
|
|
# Everyone in iLearn DB with an xyz@gavilan.edu email address.
|
|
def all_gav_employees():
|
|
(connection,cursor) = db()
|
|
connection.row_factory = dict_factory
|
|
q = """SELECT u.canvasid, u.name, u.created, u.sortablename, h.address, h.type, h.workflow_state,
|
|
h.updated_at, p.last_request_at, p.last_login_at, p.current_login_at, p.last_login_ip,
|
|
p.current_login_ip, p.sis_user_id, p.unique_name FROM users AS u
|
|
JOIN comm_channel AS h ON u.id=h.user_id
|
|
JOIN pseudonym AS p ON p.user_id=u.id
|
|
WHERE h.address LIKE "%@gavilan.edu"
|
|
ORDER BY u.sortablename"""
|
|
cursor = connection.cursor()
|
|
cursor.execute(q)
|
|
everyone = cursor.fetchall()
|
|
everyone_set = set()
|
|
for E in everyone:
|
|
try:
|
|
everyone_set.add( E['address'].lower() )
|
|
except Exception as e:
|
|
print("Exception: %s\nwith: %s" % (str(e), str(E)))
|
|
|
|
oo = open('cache/temp1.txt','w')
|
|
oo.write(json.dumps(list(everyone_set), indent=2))
|
|
existing = requests.get('https://gavilan.edu/staff/flex/2020/api.php?a=get/users')
|
|
ex = json.loads( existing.text )
|
|
already_enrolled = set()
|
|
for usr in ex['users']:
|
|
try:
|
|
#already_enrolled.add( (usr['goo'], usr['email'].lower(), usr['name']) )
|
|
already_enrolled.add( usr['email'].lower() )
|
|
except Exception as e:
|
|
print("Exception: %s\nWith: %s" % (str(e),str(usr)))
|
|
|
|
oo.write( "\n"*20 + '------------------------------------------\n'*20 + '------ - - - - - - ' )
|
|
oo.write(json.dumps(list(already_enrolled), indent=2))
|
|
|
|
# conf_users wants: goo, email, name, active
|
|
# and emails have random capitalization
|
|
# name is First Last, and sometimes with Middle in there.
|
|
#
|
|
|
|
# using sets: to_enroll = [ x for x in students if x not in already_enrolled ]
|
|
new_emp = [ x for x in everyone_set if x not in already_enrolled ]
|
|
|
|
# take the all_employee list, filter -> anyone who's in 'existing' is removed
|
|
|
|
# funcy.where( lambda x: x['email'] == ae[4] , existing )
|
|
|
|
#new_emp = list(funcy.filter( lambda ae: funcy.where( existing, email=ae['email'] ), all_emp ))
|
|
#new_emp = list(funcy.where( existing, email=b'phowell@gavilan.edu')) #ae['email'] ))
|
|
print(new_emp)
|
|
oo.write( "\n"*20 + '------------------------------------------\n'*20 + '------ - - - - - - ' )
|
|
oo.write(json.dumps(list(new_emp), indent=2))
|
|
|
|
# Now, iLearn db (everyone)... find the rows that match the email addresses
|
|
# that we've decided we need to add (new_emp)
|
|
|
|
#print(everyone)
|
|
#print( "searching for %s" % j )
|
|
#print( "searched for %s, found: %s" % (j, str(to_add) ))
|
|
#print("\nUploading...\n")
|
|
for j in new_emp:
|
|
#j = new_emp[0]
|
|
print(j)
|
|
to_add = list(funcy.where( everyone, address=j ))
|
|
if to_add:
|
|
employees_refresh_flex(to_add[0])
|
|
else:
|
|
print("Didn't find an entry for that account.")
|
|
print("done uploading")
|
|
|
|
'''
|
|
|
|
def user_from_goo(goo):
|
|
goo = "G00" + goo
|
|
q = f"SELECT * FROM canvas.pseudonyms p JOIN canvas.users u ON p.user_id=u.id WHERE p.sis_user_id='{goo}';"
|
|
(connection,cursor) = db()
|
|
cursor.execute(q, None) # execute query with optional parameters
|
|
row = cursor.fetchone() # fetch a single row
|
|
if row:
|
|
# convert row to dict using column names as keys
|
|
return dict(zip([desc[0] for desc in cursor.description], row))
|
|
|
|
|
|
def course_from_id(id):
|
|
q = f"SELECT * FROM canvas.courses c WHERE c.id={id}"
|
|
(connection,cursor) = db()
|
|
cursor.execute(q, None) # execute query with optional parameters
|
|
row = cursor.fetchone() # fetch a single row
|
|
if row:
|
|
# convert row to dict using column names as keys
|
|
return dict(zip([desc[0] for desc in cursor.description], row))
|
|
|
|
def teachers_by_term(TERM = "202430"):
|
|
q = f"""SELECT c.id, c.name, c.course_code, c.sis_source_id, c.created_at, c.start_at, c.workflow_state, e.last_attended_at,
|
|
u.id, u.sortable_name, u.created_at FROM canvas.courses AS c
|
|
JOIN canvas.enrollments AS e ON e.course_id=c.id
|
|
JOIN canvas.users AS u ON u.id=e.user_id
|
|
WHERE c.sis_source_id LIKE '{TERM}%' AND e.type='TeacherEnrollment' ORDER BY u.sortable_name, c.course_code;"""
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
all_teachers = cursor.fetchall()
|
|
|
|
table = [ [t[9],t[1],t[3],t[6]] for t in all_teachers]
|
|
print(tabulate(table))
|
|
|
|
#for t in all_teachers:
|
|
# print("\t".join( [str(x) for x in [t[9],t[1],t[3],t[6]]]))
|
|
return all_teachers
|
|
|
|
|
|
|
|
def courses_in_term(TERM = "202430"):
|
|
q = f"""SELECT c.id, c.name, c.course_code, c.sis_source_id, c.workflow_state FROM canvas.courses AS c
|
|
WHERE c.sis_source_id LIKE '{TERM}%' ORDER BY c.course_code;"""
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
all = cursor.fetchall()
|
|
|
|
#table = [ [t[9],t[1],t[3],t[6]] for t in all_teachers]
|
|
print(tabulate(all))
|
|
|
|
|
|
|
|
def pages_in_term(TERM="202430"): #
|
|
q = f"""SELECT c.id, c.course_code, c.sis_source_id, wp.id as wp_id, wp.title, wp.url, c.name , wp.body
|
|
FROM canvas.courses c
|
|
JOIN canvas.wiki_pages wp ON wp.context_id=c.id
|
|
WHERE c.sis_source_id LIKE '{TERM}%'
|
|
ORDER BY c.sis_source_id, wp.title;"""
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
all = cursor.fetchall()
|
|
#print(tabulate(all))
|
|
return all
|
|
|
|
|
|
def user_ids_in_shell(shellid):
|
|
q = f"""select e.user_id from canvas.enrollments e
|
|
where e.course_id = {shellid} and e.type='StudentEnrollment' and e.workflow_state='active';"""
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
students = {str(row[0]) for row in cursor.fetchall()}
|
|
print(f"{len(students)} students currently in shell id {shellid}.")
|
|
return students
|
|
|
|
|
|
def users_new_this_semester(sem=''):
|
|
if not len(sem):
|
|
sem = input("which semester? (ex: 202150) ")
|
|
users_to_enroll = set()
|
|
|
|
where1 = "c.sis_source_id LIKE '%s-%%'" % sem
|
|
where2 = "c.sis_source_id NOT LIKE '%s-%%'" % sem
|
|
|
|
|
|
|
|
q = """SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e
|
|
JOIN canvas.users AS u ON e.user_id=u.id
|
|
JOIN canvas.courses AS c ON e.course_id=c.id
|
|
WHERE %s
|
|
AND e.workflow_state='active'
|
|
AND e.type='StudentEnrollment'
|
|
AND u.id NOT IN (
|
|
SELECT u.id FROM canvas.enrollments AS e
|
|
JOIN canvas.users AS u ON e.user_id=u.id
|
|
JOIN canvas.courses AS c ON e.course_id=c.id
|
|
WHERE %s
|
|
AND e.workflow_state='active'
|
|
AND e.type='StudentEnrollment'
|
|
GROUP BY u.id
|
|
)
|
|
GROUP BY u.id
|
|
ORDER BY num DESC, u.sortable_name""" % (where1,where2)
|
|
|
|
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
#s = cursor.fetchall() # TODO see below
|
|
#if s:
|
|
for u in cursor:
|
|
users_to_enroll.add(str(u[0]))
|
|
print(u)
|
|
print("%i new users this semester." % len(users_to_enroll))
|
|
print(users_to_enroll)
|
|
return users_to_enroll
|
|
|
|
|
|
# when registrations opens for SUMMER+FALL, get new students from both semesters combined.
|
|
def users_new_this_2x_semester(sem1='', sem2=''):
|
|
if not len(sem1) or (not len(sem2)):
|
|
print("Need 2 semesters")
|
|
return 0
|
|
|
|
where1 = f"(c.sis_source_id LIKE '{sem1}-%%' or c.sis_source_id LIKE '{sem2}-%%')"
|
|
where2 = f"(c.sis_source_id NOT LIKE '{sem1}-%%' and c.sis_source_id NOT LIKE '{sem2}-%%')"
|
|
|
|
q = """SELECT u.id, u.name, u.sortable_name, string_agg(c.course_code, ','), COUNT(e.id) AS num FROM canvas.enrollments AS e
|
|
JOIN canvas.users AS u ON e.user_id=u.id
|
|
JOIN canvas.courses AS c ON e.course_id=c.id
|
|
WHERE %s
|
|
AND e.workflow_state='active'
|
|
AND e.type='StudentEnrollment'
|
|
AND u.id NOT IN (
|
|
SELECT u.id FROM canvas.enrollments AS e
|
|
JOIN canvas.users AS u ON e.user_id=u.id
|
|
JOIN canvas.courses AS c ON e.course_id=c.id
|
|
WHERE %s
|
|
AND e.workflow_state='active'
|
|
AND e.type='StudentEnrollment'
|
|
GROUP BY u.id
|
|
)
|
|
GROUP BY u.id
|
|
ORDER BY num DESC, u.sortable_name""" % (where1,where2)
|
|
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
users_to_enroll = {str(row[0]) for row in cursor.fetchall()}
|
|
print("%i new users this semester." % len(users_to_enroll))
|
|
return users_to_enroll
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def all_sem_courses_teachers(SEM="202450"):
|
|
q = f"""SELECT c.id, c.name, c.course_code, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id, s.type, s.crn FROM canvas.courses AS c
|
|
JOIN canvas.enrollments AS e ON e.course_id=c.id
|
|
JOIN canvas.users AS u ON u.id=e.user_id
|
|
JOIN canvas.pseudonyms AS p ON p.user_id=u.id
|
|
JOIN canvas.schedule as s on c.id=s.canvascourse
|
|
WHERE c.sis_source_id LIKE '{SEM}-%'
|
|
AND NOT c.workflow_state='deleted'
|
|
AND e.type='TeacherEnrollment'
|
|
ORDER BY u.sortable_name;"""
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
courses = cursor.fetchall()
|
|
print(courses)
|
|
return courses
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def all_2x_sem_courses_teachers(sem1, sem2):
|
|
q = f"""SELECT c.id, c.name, c.course_code, s.type, s.crn, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id FROM canvas.courses AS c
|
|
JOIN canvas.enrollments AS e ON e.course_id=c.id
|
|
JOIN canvas.users AS u ON u.id=e.user_id
|
|
JOIN canvas.pseudonyms AS p ON p.user_id=u.id
|
|
JOIN canvas.schedule as s on c.id=s.canvascourse
|
|
WHERE (c.sis_source_id LIKE '{sem1}-%' or c.sis_source_id LIKE '{sem2}-%')
|
|
AND NOT c.workflow_state='deleted'
|
|
AND e.type='TeacherEnrollment'
|
|
ORDER BY u.sortable_name;"""
|
|
(connection,cursor) = db()
|
|
cursor.execute(q)
|
|
courses = cursor.fetchall()
|
|
print(courses)
|
|
return courses
|
|
|
|
|
|
|
|
|
|
|
|
def create_schedule_table_if_not_exists():
|
|
conn,cur = db()
|
|
|
|
# Check if table exists
|
|
cur.execute("SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'schedule' and table_schema = 'canvas')")
|
|
table_exists = cur.fetchone()[0]
|
|
|
|
if not table_exists:
|
|
# Create table if it doesn't exist
|
|
cur.execute("""CREATE TABLE canvas.schedule (
|
|
id SERIAL PRIMARY KEY,
|
|
canvascourse INTEGER REFERENCES canvas.courses,
|
|
crn VARCHAR(20),
|
|
code VARCHAR(30),
|
|
units VARCHAR(20),
|
|
teacher TEXT,
|
|
start VARCHAR(30),
|
|
"end" VARCHAR(30),
|
|
type VARCHAR(20),
|
|
loc VARCHAR(80),
|
|
site VARCHAR(50),
|
|
partofday VARCHAR(40),
|
|
cap INTEGER,
|
|
act INTEGER,
|
|
sem VARCHAR(10)
|
|
)
|
|
""")
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
|
|
# Populate schedule table and correlate to courses table
|
|
def courses_to_sched():
|
|
|
|
# TODO: fix units when they are variable... change to float in between range. round to 0.5 unit.
|
|
|
|
EXECUTE = 1
|
|
|
|
seasons = {'10':'wi','30':'sp','50':'su','70':'fa'}
|
|
seasons2 = {'wi':'10', 'sp':'30', 'su':'50', 'fa':'70'}
|
|
|
|
create_schedule_table_if_not_exists()
|
|
conn,cur = db()
|
|
|
|
# get all ilearn courses
|
|
query = """SELECT c.id, c.workflow_state, c.sis_source_id, c.course_code, c.enrollment_term_id, t.name
|
|
FROM canvas.courses c
|
|
JOIN canvas.enrollment_terms t ON c.enrollment_term_id = t.id
|
|
ORDER BY c.sis_source_id, c.course_code;"""
|
|
|
|
cur.execute(query)
|
|
|
|
sis_to_sched = {}
|
|
for row in cur.fetchall():
|
|
sis_source_id = row[2] # c.sis_source_id
|
|
sis_to_sched.setdefault(sis_source_id, []).append(row)
|
|
|
|
|
|
vals_cache = []
|
|
i = 0
|
|
for year in ['16','17','18','19','20','21','22','23','24']:
|
|
for sem in ['sp','su','fa']:
|
|
term = f"{sem}{year}"
|
|
sis_code = f"20{year}{seasons2[sem]}"
|
|
print(term)
|
|
sched = 0
|
|
try:
|
|
sched = requests.get(f"http://gavilan.cc/schedule/{term}_sched_expanded.json").json()
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
query = "INSERT INTO canvas.schedule (canvascourse, crn, code, units, teacher, start,\"end\", type, loc, site, partofday, cap, act, sem) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
|
|
|
|
if sched:
|
|
for c in sched:
|
|
try:
|
|
pod = ''
|
|
if 'partofday' in c: pod = c['partofday']
|
|
|
|
full_sis_code = sis_code+'-'+c['crn']
|
|
|
|
if full_sis_code in sis_to_sched:
|
|
q = [sis_to_sched[full_sis_code][0][0], c['crn'], c['code'], c['cred'], c['teacher'], c['start'], c['end'], c['type'], c['loc'], c['site'], pod, int(c['cap']), int(c['act']), sis_code]
|
|
vals_cache.append( q ) # [ str(x) for x in q ] )
|
|
print(f"{i}: {q}")
|
|
i += 1
|
|
if i % 100 == 0:
|
|
if EXECUTE:
|
|
cur.executemany(query, vals_cache)
|
|
conn.commit()
|
|
vals_cache = []
|
|
t = time.process_time()
|
|
delta = t - last_time
|
|
last_time = t
|
|
print(f"Loop {i} - committed to db in %0.3fs. " % delta, flush=True)
|
|
else:
|
|
print(f"{full_sis_code} not in canvas courses.")
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
if EXECUTE:
|
|
cur.executemany(query, vals_cache)
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
print ('')
|
|
options = {
|
|
1: ['all teachers', teachers_by_term],
|
|
2: ['courses in term', courses_in_term],
|
|
3: ['pages in term', pages_in_term],
|
|
4: ['new students this semester', users_new_this_semester],
|
|
5: ['all semester courses + teachers', all_sem_courses_teachers],
|
|
6: ['Populate schedule table and correlate to courses table', courses_to_sched],
|
|
}
|
|
|
|
|
|
if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
|
|
resp = int(sys.argv[1])
|
|
print("\n\nPerforming: %s\n\n" % options[resp][0])
|
|
|
|
else:
|
|
print ('')
|
|
for key in options:
|
|
print(str(key) + '.\t' + options[key][0])
|
|
|
|
print('')
|
|
resp = input('Choose: ')
|
|
|
|
# Call the function in the options dict
|
|
options[ int(resp)][1]()
|
|
|
|
|