#get_schedule('201770')
# from pipelines - canvas data
# todo: where does the most recent schedule come from?
# Input: xxxx_sched.json. Output: xxxx_latestarts.txt
def list_latestarts():
#term = input("Name of current semester file? (ex: sp18) ")
term = "sp23" # sems[0]
term_in = "cache/" + term + "_sched.json"
term_out = "cache/" + term + "_latestarts.txt"
print("Writing output to " + term_out)
infile = open(term_in, "r")
outfile = open(term_out, "w")
sched = json.loads(infile.read())
#print sched
by_date = {}
for C in sched:
parts = C['date'].split("-")
start = parts[0]
codes = C['code'].split(' ')
dept = codes[0]
if dept in ['JLE','JFT','CWE']:
continue
if re.search('TBA',start): continue
try:
startd = parser.parse(start)
except Exception as e:
print(e, "\nproblem parsing ", start)
#print startd
if not startd in by_date:
by_date[startd] = []
by_date[startd].append(C)
for X in sorted(by_date.keys()):
#print "Start: " + str(X)
if len(by_date[X]) < 200:
prettydate = X.strftime("%A, %B %d")
print(prettydate + ": " + str(len(by_date[X])) + " courses")
outfile.write(prettydate + ": " + str(len(by_date[X])) + " courses" + "\n")
for Y in by_date[X]:
#print "\t" + Y['code'] + " " + Y['crn'] + "\t" + Y['teacher']
print(Y)
#outfile.write("\t" + Y['code'] + " " + Y['crn'] + "\t" + Y['teacher'] + "\t" + Y['type'] +"\n")
outfile.write("\t" + Y['code'] + " " + Y['crn'] + "\t" + Y['teacher'] + "\t" + Y['type'] + "\t" + "\n")
online_courses = {}
def prep_online_courses_df():
global online_courses
schedule = current_schedule() # from banner
online_courses = schedule[lambda x: x.type=='online']
def course_is_online(crn):
global online_courses
#print "looking up: " + str(crn)
#print online_courses
course = online_courses[lambda x: x.crn==int(crn)]
return len(course)
def get_crn_from_name(name):
#print "name is: "
#print(name)
m = re.search( r'(\d\d\d\d\d)', name)
if m: return int(m.groups(1)[0])
else: return 0
def get_enrlmts_for_user(user,enrollments):
#active enrollments
u_en = enrollments[ lambda x: (x['user_id'] == user) & (x['workflow']=='active') ]
return u_en[['type','course_id']]
"""
timestamp = nowAsStr()
requestParts = [ method,
host,
'', #content Type Header
'', #content MD5 Header
path,
'', #alpha-sorted Query Params
timestamp,
apiSecret ]
#Build the request
requestMessage = '\n'.join( requestParts )
requestMessage = requestMessage.encode('ASCII')
print((requestMessage.__repr__()))
hmacObject = hmac.new(bytearray(apiSecret,'ASCII'), bytearray('','ASCII'), hashlib.sha256) #
hmacObject.update(requestMessage)
hmac_digest = hmacObject.digest()
sig = base64.b64encode(hmac_digest)
headerDict = {
'Authorization' : 'HMACAuth ' + apiKey + ':' + str(sig),
'Date' : timestamp
}
"""
# Don't know
def demo():
resp = do_request('/api/account/self/file/sync')
mylog.write(json.dumps(resp, indent=4))
sample_table = resp['files'][10]
filename = sample_table['filename']
print(sample_table['table'])
response = requests.request(method='GET', url=sample_table['url'], stream=True)
if(response.status_code != 200):
print(('Request response went bad. Got back a ', response.status_code, ' code, meaning the request was ', response.reason))
else:
#Use the downloaded data
with open(local_data_folder + filename, 'wb') as fd:
for chunk in response.iter_content(chunk_size=128):
fd.write(chunk)
print("Success")
if filename.split('.')[-1] == 'gz':
plain_filename = 'canvas_data/' + ".".join(filename.split('.')[:-1])
pf = open(plain_filename,'w')
with gzip.open('canvas_data/' + filename , 'rb') as f:
pf.write(f.read())
# How to drop columns
#columns = ['Col1', 'Col2', ...]
#df.drop(columns, inplace=True, axis=1)
# left join, one on column, one on index
#merged = pd.merge(result,users,left_index=True,right_on='id', how='left')
"""
You can call set_index on the result of the dataframe:
In [2]:
data=[['Australia',100],['France',200],['Germany',300],['America',400]]
pd.DataFrame(data,columns=['Country','Volume']).set_index('Country')
Out[2]:
Volume
Country
Australia 100
France 200
Germany 300
America 400
"""
def stats():
# nothing seems to happen here?
#input = csv.DictReader(codecs.open(schedfile,'r','utf-8'))
input = csv.DictReader(open(schedfile,'r'))
out2 = open('temp2.csv','w')
clean = {}
for r in input:
if r['crn']: clean[ r['crn'] ] = r
for c,r in list(clean.items()):
try:
if int(r['cap'])==0: continue
else: prct = (1.0 * int( r['act'] )) / int(r['cap'])
if prct < 0.01: continue
o_str = ''
if r['location'].strip()=='ONLINE': o_str = 'online'
#print r['location']
date_parts = r['date'].split('-')
start = strptime(date_parts[0], '%m/%d')
if start > semester_begin: o_str += "\tlatestart " + date_parts[0]
out2.write( "".join([c, "\t", r['sub'], "\t", r['crs'], "\t", str(round(prct,2)), "% full\t", o_str, "\n"]) )
except:
pass
######### from curriculum. py
# open('cache/programs/programs_1.txt','r').read()
"""
SEE serve.py .... i mean ... interactive.py
def dict_generator(indict, pre=None):
pre = pre[:] if pre else []
if isinstance(indict, dict):
for key, value in indict.items():
if isinstance(value, dict):
for d in dict_generator(value, pre + [key]):
yield d
elif isinstance(value, list) or isinstance(value, tuple):
for v in value:
for d in dict_generator(v, pre + [key]):
yield d
else:
yield str(pre) + " " + str([key, value]) + "\n"
else:
yield pre + [indict]
yield str(pre) + " " + str([indict]) + "\n"
def print_dict(v, prefix='',indent=''):
if isinstance(v, dict):
return [ print_dict(v2, "{}['{}']".format(prefix, k) + "
", indent+" " ) for k, v2 in v.items() ]
elif isinstance(v, list):
return [ print_dict( v2, "{}[{}]".format(prefix , i) + "
", indent+" ") for i, v2 in enumerate(v) ]
else:
return '{} = {}'.format(prefix, repr(v)) + "\n"
def walk_file():
j = json.loads(open('cache/programs/programs_2.txt','r').read())
return print_dict(j)
from flask import Flask
from flask import request
def tag(x,y): return "<%s>%s" % (x,y,x)
def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
def a(t,h): return '%s' % (h,t)
def server_save(key,value):
codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
def flask_thread(q):
app = Flask(__name__)
@app.route("/")
def home():
return tag('h1','This is my server.') + "
" + a('want to shut down?','/sd')
@app.route("/save//")
def s(key,val):
server_save(key,val)
return tag('h1','Saved.') + "
" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
@app.route("/crazy")
def hello():
r = ''
r += tag('style', 'textarea { white-space:nowrap; }')
r += tag('body', \
tagc('div','container-fluid', \
tagc('div','row', \
tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
return r
@app.route("/sd")
def sd():
print('SIGINT or CTRL-C detected. Exiting gracefully')
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
return "Server has shut down."
app.run()
from queue import Queue
q = Queue()
def serve():
import webbrowser
import threading
x = threading.Thread(target=flask_thread, args=(q,))
x.start()
webbrowser.open_new_tab("http://localhost:5000")
#s = open('cache/programs/index.json','w')
#s.write( json.dumps({'departments':sorted(list(dept_index)), 'programs':prog_index}, indent=2) )
#s.close()
"""
### courses.py
##########
########## CALCULATING SEMESTER STUFF
##########
def summarize_proportion_online_classes(u):
# u is a "group" from the groupby fxn
#print u
if NUM_ONLY:
if ((1.0 * u.sum()) / u.size) > 0.85: return '2'
if ((1.0 * u.sum()) / u.size) < 0.15: return '0'
return '1'
else:
if ((1.0 * u.sum()) / u.size) > 0.85: return 'online-only'
if ((1.0 * u.sum()) / u.size) < 0.15: return 'f2f-only'
return 'mixed'
def summarize_num_term_classes(u):
# u is a "group" from the groupby fxn
# term is sp18 now
#print u
return u.size
# Prompt for course id, return list of user dicts. TODO this duplicates courses.py ??
def getUsersInCourse(id=0): # returns list
if not id:
id = str(input("The Course ID? "))
id = str(id)
return fetch('/api/v1/courses/%s/users' % id, 0)
#### curriculum.py
def recur_look_for_leafs(item,indent=0,show=1):
global leafcount, displaynames
ii = indent * " "
is_leaf = am_i_a_leaf(item)
if type(item) == type({}):
status = ""
if show:
status = "Dict"
if is_leaf:
leafcount += 1
status = "Leaf Dict"
if status:
print("\n%s%s" % (ii,status))
indent += 1
ii = indent * " "
for K,V in list(item.items()):
if show or is_leaf:
print("%s%s:" % (ii, K), end="")
if K =='displayName': displaynames.append(V)
recur_look_for_leafs(V,indent+1,show or is_leaf)
elif type(item) == type([]):
status = ""
if show: status = "List (" + str( len(item) ) + ")"
if is_leaf: status = "Leaf List (" + str( len(item) ) + ")"
if status:
print("\n%s%s" % (ii,status))
indent += 1
ii = indent * " "
for V in item:
recur_look_for_leafs(V,indent+1, show or is_leaf)
elif type(item) == type("abc"):
if show: print("%s%s" % (' ', item))
elif type(item) == type(55):
if show: print("%s%i" % (' ', item))
elif type(item) == type(5.5):
if show: print("%s%f" % (' ', item))
elif type(item) == type(False):
if show: print("%s%s" % (' ', str(item)))
def am_i_a_leaf(item):
if type(item) == type({}):
for K,V in list(item.items()):
if type(V) == type({}) or type(V) == type([]):
return False
elif type(item) == type([]):
for V in item:
if type(V) == type({}) or type(V) == type([]):
return False
elif type(item) == type("abc"): return True
elif type(item) == type(55): return True
elif type(item) == type(5.5): return True
elif type(item) == type(False):
if item == False: return True
elif item == True: return True
return True
def sampleclass():
theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
#print(json.dumps(theclass,indent=2))
recur_look_for_leafs(theclass)
print(leafcount)
print(sorted(displaynames))
def matchstyle():
theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
print("\n".join(recur_matcher(theclass)))
# 7: ['pattern matcher style', matchstyle],
# 8: ['pattern matcher - test on all classes', match_style_test],
##### from localcache
def user_role_and_online():
# cross list users, classes enrolled, and their roles
global role_table, term_courses
role_table = enrollment_file()
user_table = users_file()
user_table = user_table[ user_table['name']!="Test Student" ]
term_table = term_file()
current = term_table[lambda d: d.course_section=='2020 Spring'] # current semester from canvas
term_id = current['id'].values[0]
course_table = courses_file() # from canvas
schedule = current_schedule() # from banner...
term_courses = course_table[lambda d: d.termid==term_id] # courses this semester ... now add a crn column
term_courses['crn'] = term_courses['code'].map( lambda x: get_crn_from_name(x) )
# add is_online flag (for courses listed in schedule as online-only)
term_courses['is_online'] = term_courses['crn'].map( lambda x: course_is_online( x ) ) # kinda redundant
ban_can = term_courses.merge(schedule,on='crn',how='left') #join the schedule from banner to the courses from canvas
role_table = role_table.where(lambda x: x.workflow=='active')
# this join limits to current semester if 'inner', or all semesters if 'left'
courses_and_enrol = role_table.merge(ban_can,left_on='course_id',right_on='id', how='left')
user_table = user_table.drop(columns="rootactid tz created vis school position gender locale public bd cc state".split(" "))
c_e_user = courses_and_enrol.merge(user_table,left_on='user_id',right_on='id',how='left')
prop_online = pd.DataFrame(c_e_user.groupby(['user_id'])['is_online'].aggregate(summarize_proportion_online_classes).rename('proportion_online'))
num_trm_crs = pd.DataFrame(c_e_user.groupby(['user_id'])['is_online'].aggregate(summarize_num_term_classes).rename('num_term_crs'))
stu_tch_rol = pd.DataFrame(c_e_user.groupby(['user_id'])['type'].aggregate(summarize_student_teacher_role).rename('main_role'))
user_table = user_table.merge(prop_online,left_on='id',right_index=True)
user_table = user_table.merge(num_trm_crs,left_on='id',right_index=True)
user_table = user_table.merge(stu_tch_rol,left_on='id',right_index=True)
# remove name-less entries
user_table = user_table.where(lambda x: (x.canvasid!='') ) # math.isnan(x.canvasid))
return user_table
#print user_table.query('proportion_online=="online-only"')
#print user_table.query('main_role=="teacher"')
#user_table.to_csv('canvas_data/users_online.csv')
"""e_qry = "CREATE TABLE IF NOT EXISTS enrollments (
id integer PRIMARY KEY,
name text NOT NULL,
begin_date text,
end_date text
);"""
"""
['CREATE INDEX "idx_req_userid" ON "requests" ("id","courseid","userid" );',
'CREATE INDEX "idx_users_id" ON "users" ("id","canvasid", );',
'CREATE INDEX "idx_term_id" ON "terms" ("id","canvasid" );',
'CREATE INDEX "idx_enrollment" ON "enrollment" ("cid","course_id","user_id" );',
'CREATE INDEX "idx_courses" ON "courses" ("id","canvasid","termid","code","name" );' ]
took 6 seconds
select * from users where name = "Peter Howell"
select * from users join requests on users.id = requests.userid where name = "Peter Howell"
20k rows in 1.014 seconds!! with index above
without: killed it after 120 seconds
select timestamp, url, useragent, httpmethod, remoteip, controller from users join requests on users.id = requests.userid where name = "Peter Howell" order by requests.timestamp
select courses.name, courses.code, terms.name, requests.url from courses
join terms on courses.termid = terms.id
join requests on courses.id = requests.courseid
where terms.name='2020 Spring ' and courses.code='ACCT20 SP20 40039'
order by courses.code
"""
def more_unused_xreferencing():
"""continue
for line in lines:
r = requests_line(line.decode('utf-8'),filei)
if filei < 5:
print(r)
else:
break
filei += 1
by_date_course = defaultdict( lambda: defaultdict(int) )
by_date_user = defaultdict( lambda: defaultdict(int) )
df_list = []
df_list_crs = []
users = defaultdict( lambda: defaultdict(int) )
#by_user = {}
#by_course = {}
i = 0
limit = 300
#print(r)
date = dt.strptime( r['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
if r['userid'] in users:
users[r['userid']]['freq'] += 1
if users[r['userid']]['lastseen'] < date:
users[r['userid']]['lastseen'] = date
else:
users[r['userid']] = {"id":r['userid'], "lastseen":date, "freq":1}
by_date_course[ r['day'] ][ r['courseid'] ] += 1
by_date_user[ r['day'] ][ r['userid'] ] += 1
#if r['userid'] in by_user: by_user[r['userid']] += 1
#else: by_user[r['userid']] = 1
#if r['courseid'] in by_course: by_course[r['courseid']] += 1
#else: by_course[r['courseid']] = 1
#mylog.write("by_user = " + str(by_user))
df_list.append(pd.DataFrame(data=by_date_user))
df_list_crs.append(pd.DataFrame(data=by_date_course))
i += 1
if i > limit: break
#mylog.write("by_date_course = ")
result = pd.concat(df_list, axis=1,join='outer')
result_crs = pd.concat(df_list_crs, axis=1,join='outer')
#print result_crs
mylog.write(result.to_csv())
# get users
usersf = user_role_and_online()
merged = pd.merge(result,usersf,left_index=True,right_on='id', how='left')
#dropkeys = "rootactid tz created vis school position gender locale public bd cc state".split(" ")
#merged.drop(dropkeys, inplace=True, axis=1)
mglog = open(local_data_folder+'userlogs.csv','w')
mglog.write(merged.to_csv())
# get courses
courses = courses_file()
merged2 = pd.merge(result_crs,courses,left_index=True,right_on='id', how='left')
dropkeys = "rootactid wikiid".split(" ")
merged2.drop(dropkeys, inplace=True, axis=1)
mglogc = open(local_data_folder + 'courselogs.csv','w')
mglogc.write(merged2.to_csv())
# a users / freq / lastseen file
ufl = open(local_data_folder + "user_freq.json","w")
today = datetime.datetime.today()
for U in list(users.keys()):
date = users[U]['lastseen']
users[U]['lastseen'] = date.strftime("%Y-%m-%d")
diff = today - date
users[U]['daysago'] = str(diff.days)
users[U]['hoursago'] = str(int(diff.total_seconds()/3600))
us_frame = pd.DataFrame.from_dict(users,orient='index')
us_with_names = pd.merge(us_frame,usersf,left_index=True,right_on='id', how='left')
#dropkeys = "id id_x id_y globalid rootactid tz created vis school position gender locale public bd cc state".split(" ")
#us_with_names.drop(dropkeys, inplace=True, axis=1)
print(us_with_names)
ufl.write( json.dumps(users, indent=4) )
ufl.close()
mglogd = open('canvas_data/user_freq.csv','w')
mglogd.write(us_with_names.to_csv())
"""
""" -- projects table
CREATE TABLE IF NOT EXISTS projects (
id integer PRIMARY KEY,
name text NOT NULL,
begin_date text,
end_date text
);
"""
pass
def users_p_file():
uf = users_file()
pf = pseudonym_file()
#print pf
upf = uf.merge(pf,left_on='id',right_on='user_id',how='left')
return upf
"""
def com_channel_dim():
all = os.listdir(local_data_folder)
all.sort(key=lambda x: os.stat(os.path.join(local_data_folder,x)).st_mtime)
all.reverse()
#print "sorted file list:"
#print all
for F in all:
if re.search('communication_channel_dim',F):
cc_file = F
break
print("most recent communication channel file is " + cc_file)
cc_users = []
for line in gzip.open(local_data_folder + cc_file,'r'):
line_dict = dict(list(zip(cc_format, line.split("\t"))))
#line_dict['globalid'] = line_dict['globalid'].rstrip()
cc_users.append(line_dict)
df = pd.DataFrame(cc_users)
return df
"""
"""grp_sum_qry = ""SELECT u.sortablename, r.timeblock, SUM(r.viewcount), u.canvasid AS user, c.canvasid AS course
FROM requests_sum1 AS r
JOIN courses AS c ON e.course_id=c.id
JOIN enrollment as e ON r.courseid=c.id
JOIN users AS u ON u.id=e.user_id
WHERE c.canvasid=%s AND e."type"="StudentEnrollment"
GROUP BY u.id,c.id,r.timeblock
ORDER BY u.sortablename DESC, r.timeblock"" % course_id
q = ""SELECT u.sortablename, r.timeblock, r.viewcount, u.canvasid AS user, c.canvasid AS course
FROM requests_sum1 AS r
JOIN courses AS c ON e.course_id=c.id
JOIN enrollment as e ON r.courseid=c.id
JOIN users AS u ON u.id=e.user_id
WHERE c.canvasid=%s AND e."type"="StudentEnrollment" AND u.canvasid=810
ORDER BY u.sortablename DESC, r.timeblock"" % course_id
q = ""SELECT u.sortablename, r.timeblock, r.viewcount, u.canvasid AS user, c.canvasid AS course FROM enrollment as e JOIN courses AS c ON e.course_id=c.id
JOIN requests_sum1 AS r ON r.courseid=c.id
JOIN users AS u ON u.id=e.user_id
WHERE c.canvasid=%s AND e."type"="StudentEnrollment"
ORDER BY u.sortablename, r.timeblock"" % course_id"""
stem_course_id = '11015' # TODO
# NO LONGER USED - SEE COURSES
def enroll_stem_students():
depts = "MATH BIO CHEM PHYS ASTR GEOG".split(" ")
students = set()
for d in depts:
students.update(dept_classes(d))
print(students)
to_enroll = [ x for x in students if x not in already_enrolled ]
print(to_enroll)
print("prev line is people to enroll\nnext line is students already enrolled in stem")
print(already_enrolled)
for s in to_enroll:
t = url + '/api/v1/courses/%s/enrollments' % stem_course_id
data = { 'enrollment[user_id]': s[1], 'enrollment[type]':'StudentEnrollment',
'enrollment[enrollment_state]': 'active' }
print(data)
print(t)
if input('enter to enroll %s or q to quit: ' % s[0]) == 'q':
break
r3 = requests.post(t, headers=header, params=data)
print(r3.text)
#####
##### from users.py pretty much just use sql now
# unused?
def getAllTeachersInTerm(): # a list
# classes taught in last 3 semesters
# How many of them were published and used
# hits in last week/month/year
# most common department
# email addr
all_courses = {}
teachers = {} # keyed by goo
# { 'name':'', 'id':'', 'email':'', 'goo':'', 'classes':[ (#name,#id,#pubd,#hitsbyteacher) ... ] }
# This is a bit different from the 1 year schedule above, because it looks at
# people who were active in their shells in iLearn.
outfile = codecs.open('teacherdata/historical_shells_used.json','w', encoding='utf-8')
for term in last_4_semesters_ids: # [60,]:
print(("Fetching term: " + str(term)))
all_courses[term] = \
fetch('/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100')
i = 0
j = 0
for k,v in list(all_courses.items()): ##### term k, list v
for a_class in v:
print((a_class['name']))
published = 0
if a_class['workflow_state'] in ['available','completed']:
j += 1
published = 1
i += 1
#if i > 20: break
tch = fetch('/api/v1/courses/' + str(a_class['id']) + '/search_users?enrollment_type=teacher')
for r in tch: ##### TEACHER r of COURSE a_class
name = str(r['sortable_name'])
if not 'sis_import_id' in r:
print("This user wasn't available: " + name)
continue
goo = str(r['sis_import_id'])
print((r['sortable_name']))
if not name in teachers:
email = getEmail(r['id'])
teachers[name] = { 'name':r['sortable_name'], 'id':r['id'], 'email':email, 'goo':goo, 'classes':[] }
info = (a_class['name'],a_class['id'],published)
teachers[name]['classes'].append( info )
## TODO: hits in courses by teachers https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
for t,v in list(teachers.items()):
teachers[t]['num_courses'] = len(v['classes'])
teachers[t]['num_active_courses'] = sum( [x[2] for x in v['classes']] )
depts = [ dept_from_name(x[0]) for x in v['classes'] ]
teachers[t]['dept'] = most_common_item(depts)
#print(str(j), "/", str(i), " sections are published")
outfile.write(json.dumps(teachers))
"""
def teacherActivityLog(uid=1): ### Next: save results in a hash and return that....
global results, users, users_by_id
#get_users() # do this if you think 'teachers/users.json' is outdated.
load_users()
#for x in users_by_id.keys():
# if x < 20:
# print x
# print users_by_id[x]
teachers = csv.reader(open('teachers/current_semester.txt','r'), delimiter="\t")
for row in teachers:
print(row[0] + " is id: " + row[1])
uid = row[1]
print("Comes up as: " + str(users_by_id[int(uid)]))
info = users_by_id[int(uid)]
goo = info['login_id']
output_file = open('logs/users/byweek/'+ goo.lower() + '.csv', 'w')
# okay, actually, the first week here is the week before school IRL
start = isoweek.Week.withdate( datetime.date(2017,8,21))
end = isoweek.Week.thisweek()
byweek = []
i = 0
while(1):
results = []
start = start + 1
if start > end: break
myStart = start.day(0).isoformat() + 'T00:00-0700'
myEnd = start.day(6).isoformat() + 'T11:59:59-0700'
t = url + "/api/v1/users/" + str(uid) + "/page_views?start_time=" + myStart + '&end_time=' + myEnd + "&perpage=500"
print(t)
while(t):
print(".", end=' ')
t = fetch(t)
print("")
thisWeek = len(results)
print("Week # " + str(i) + "\t" + str(thisWeek))
byweek.append( "Week # " + str(i) + "\t" + str(thisWeek) )
output_file.write( start.isoformat() + "," + str(thisWeek) + "\n")
i += 1
for j in byweek:
print(j)
"""
"""
def summarize_student_teacher_role(u):
# u is a "group" from the groupby fxn
# term is sp18 now
t = 0
s = 0
for a in u:
if a=='TeacherEnrollment': t += 1
else: s += 1
if NUM_ONLY:
if t > s: return 'teacher'
return 'student'
else:
if t > s: return '1'
return '0'
"""
"""
def user_roles2():
# cross list users, classes enrolled, and their roles
global role_table, term_courses
role_table = enrollment_file()
user_table = users_file()
course_table = courses_file() # from canvas
term_table = term_file()
schedule = current_schedule() # from banner
# current semester
current = term_table[lambda d: d.course_section=='2018 Spring']
term_id = current['id'].values[0]
term_courses = course_table[lambda d: d.termid==term_id] # courses this semester
# add is_online flag (for courses listed in schedule as online-only)
term_courses['is_online'] = term_courses['code'].map( lambda x: course_is_online( get_crn_from_name(x) ) )
new_df = pd.DataFrame(columns=['type','oo','num'])
m = 0
data = []
for u in user_table.iterrows():
if m % 1000 == 0: print("on row " + str(m))
m += 1
data.append(categorize_user(u))
#if m > 1500: break
new_df = pd.DataFrame(data,columns=['i','type','onlineonly','numcls']).set_index('i')
print(new_df)
user_table = user_table.merge(new_df,left_index=True,right_index=True)
user_table.to_csv('canvas_data/users_online.csv')
"""
### IS THIS IN CANVAS_DATA.py?
""" Collate the raw logs into something more compact and useful. Version 1:
- # of accesses, user/day
- # of participations, user/day
-
- where day is the number of days into the semester. Classes shorter than 16 weeks should get a multiplier
-
- 2 initial goals:
a. data for statistics / clustering / regression / learning
b. data for visualization
"""
def req_to_db(fname_list):
fields = ','.join("id timestamp timestamp_year timestamp_month timestamp_day user_id course_id root_account_id course_account_id quiz_id discussion_id conversation_id assignment_id url user_agent http_method remote_ip interaction_micros web_application_controller web_applicaiton_action web_application_context_type web_application_context_id real_user_id session_id user_agent_id http_status http_version".split(" "))
sqlite_file = 'canvas_data/data.db'
conn = sqlite3.connect(sqlite_file)
c = conn.cursor()
# merge all requests into db
by_date_course = defaultdict( lambda: defaultdict(int) )
by_date_user = defaultdict( lambda: defaultdict(int) )
df_list = []
df_list_crs = []
users = defaultdict( lambda: defaultdict(int) )
i = 0
limit = 300
for fname in fname_list:
print((fname+"\n"))
for line in gzip.open('canvas_data/'+fname,'r'):
r = line.split('\t')
#tot = len(fields.split(','))
#i = 0
#for x in fields.split(','):
# print x + "\t" + r[i]
# i+= 1
qry = "insert into requests("+fields+") values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
conn.execute(qry, r)
# New method for below:
# read collated data from sqlite
# collate from more logs
# write back....?
"""
date = datetime.datetime.strptime( r['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
if r['userid'] in users:
users[r['userid']]['freq'] += 1
if users[r['userid']]['lastseen'] < date:
users[r['userid']]['lastseen'] = date
else:
users[r['userid']] = {"id":r['userid'], "lastseen":date, "freq":1}
by_date_course[ r['day'] ][ r['courseid'] ] += 1
by_date_user[ r['day'] ][ r['userid'] ] += 1
#if r['userid'] in by_user: by_user[r['userid']] += 1
#else: by_user[r['userid']] = 1
#if r['courseid'] in by_course: by_course[r['courseid']] += 1
#else: by_course[r['courseid']] = 1
#mylog.write("by_user = " + str(by_user))
df_list.append(pd.DataFrame(data=by_date_user))
df_list_crs.append(pd.DataFrame(data=by_date_course))
"""
i += 1
if i > limit: break
conn.commit()
conn.close()
"""
Making columns:
table_data = [['a', 'b', 'c'], ['aaaaaaaaaa', 'b', 'c'], ['a', 'bbbbbbbbbb', 'c']]
for row in table_data:
print("{: >20} {: >20} {: >20}".format(*row))
Transpose a matrix:
rez = [[m[j][i] for j in range(len(m))] for i in range(len(m[0]))]
"""
"""
ilearn_by_id = {}
ilearn_by_name = {}
for x in ilearn_list:
ilearn_by_id[x[3]] = x
ilearn_by_name[x[0]] = x
for ml in open('cache/teacher_manual_name_lookup.csv','r').readlines():
parts = ml.strip().split(',')
try:
manual_list[parts[0]] = ilearn_by_id[parts[1]]
except Exception as e:
print "Teacher missing: " + parts[0]
il_names = [ x[0] for x in ilearn_list ]
il_byname = {}
for x in ilearn_list: il_byname[x[0]] = x
sched_list_missed = [x for x in sched_list]
#
# key is long name (with middle name) from schedule, value is tuple with everything
name_lookup = manual_list
matches = []
#print ilearn_list
num_in_sched = len(sched_list)
num_in_ilearn = len(ilearn_list)
#for i in range(min(num_in_sched,num_in_ilearn)):
# print "|"+sched_list[i] + "|\t\t|" + ilearn_list[i][0] + "|"
print("Sched names: %i, iLearn names: %i" % (num_in_sched,num_in_ilearn))
for s in sched_list:
for t in il_names:
if first_last(s) == t:
#print ' MATCHED ' + s + ' to ' + t
sched_list_missed.remove(s)
try:
name_lookup[s] = ilearn_by_name[ first_last(s) ]
except Exception as e:
print "Teacher missing (2): " + s
il_names.remove(first_last(s))
matches.append(s)
print "Matched: " + str(matches)
print "\nDidn't match: " + str(len(sched_list_missed)) + " schedule names."
print "\nFinal results: "
print name_lookup
nlf = codecs.open('cache/sched_to_ilearn_names.json','w','utf-8')
nlf.write(json.dumps(name_lookup,indent=2))
# STRING DISTANCE
#sim = find_most_similar(s,i_names)
#print ' CLOSEST MATCHES to ' + s + ' are: ' + str(sim)
#mm.write(s+',\n')
"""
#ilearn_list = sorted(list(set(map(
# lambda x: #(tfi[x]['name'],tfi[x]['email'],tfi[x]['dept'],str(tfi[x]['id']),tfi[x]['goo']),
# tfi.keys()))))
#i_names = [ x[0] for x in ilearn_list ]
#print json.dumps(i_names,indent=2)
#return
# how to filter a dict based on values
# filtered = {k: v for k, v in course_combos.items() if v['dept'] == 'LIB' or v['dept'] == 'CSIS' }
# more pandas
# gapminder['continent'].unique()
#for name,group in bycode:
# #print name
# print name, " ", group['type']
#onl = gg.agg( lambda x: has_online(x) )
#ttl = gg.agg( lambda x: len(x) )
#ttl = ttl.rename(columns={'type':'total_sections'})
#onl.join(gg.agg( lambda x: has_hybrid(x) ),how='outer')
#onl.join(gg.agg( lambda x: has_lecture(x) ), how='outer')
#onl['num_sections'] = 0
#onl['num_lec'] = 0
#onl['num_online'] = 0
#all = pd.merge([onl,hyb,lec])
#print onl
#total=len, f2f=lambda x: ) set(x)
#{ 'num_sections': "count",
# 'num_lec': lambda x: 5,
# 'num_online': lambda x: 5 } )
#print gg
"""
def has_online(series):
# if any items of the series have the string 'online', return 1
for i in series:
if i == 'online': return 1
return 0
def has_lecture(series):
# if any items of the series have the string 'online', return 1
for i in series:
if i == 'online': return 1
return 0
def has_hybrid(series):
# if any items of the series have the string 'online', return 1
for i in series:
if i == 'hybrid': return 1
return 0
"""
#### RIGHT HERE IS WHERE I THINK... MAYBE THIS ISN'T THE RIGHT APPROACH. I DON'T SEEM
#### TO BE ABLE TO QUERY THE FACT BASE. IS THAT TRUE? SHOULD I JUST BE USING TABLES?
#### CHANGING COURSE... USE THE RULES TO UPDATE A DATABASE/TABLE/DATAFRAME
#### OR SET OF DICTS.
# ultimately i want this to be more flexible, so i can categorize degrees as 'available evening' etc
#
# Simple data structure. In this function, a degree is
""" degree = { 'name': 'History AA',
'blocks': [ { 'original_title':'xxx', 'rulecode':'u3',
'courses': [ {'code':'math1a', 'units': '3.0', 'wasonline':False },
{'code':'math2a', 'units': '3.0', 'wasonline':False },
{'code':'math3a', 'units': '3.0', 'wasonline':False } ] },
{ 'original_title':'xyz', 'rulecode':'a',
'courses': [ {'code':'math5a', 'units': '3.0', 'wasonline':False },
{'code':'math6a', 'units': '3.0', 'wasonline':False },
{'code':'math7a', 'units': '3.0', 'wasonline':False } ] } ] }
"""
# Wrapper to get 2 schedules at once
def dl_sched():
global SEMESTER, semester_begin, filename, short_sem
SEMESTER = 'Fall 2019'
short_sem = 'fa19'
semester_begin = strptime('08/26', '%m/%d')
filename = 'fa19_sched.json'
txt = login()
codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
#stats()
#reg_nums()
#todo: these semesters
SEMESTER = 'Summer 2019'
short_sem = 'su19'
semester_begin = strptime('06/17', '%m/%d')
filename = 'su19_sched.json'
txt = login()
codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
#stats()
#reg_nums()
# Send a personalized email regarding ZTC
def send_z_email(fullname, firstname, addr, courses_list):
FULLNAME = fullname #"Sabrina Lawrence"
FNAME = firstname # "Sabrina"
to_email = addr # "slawrence@gavilan.edu"
courses = courses_list # ["CSIS45", "CSIS85"]
course_template = "%s "
url_template = "https://docs.google.com/forms/d/e/1FAIpQLSfZLQp6wHFEdqsmpZ7jz2Y8HtKLo8XTAhrE2fyvTDOEgquBDQ/viewform?usp=pp_url&entry.783353363=%s&entry.1130271051=%s" # % (FULLNAME, COURSE1)
bare_link = "https://forms.gle/pwZJHdWSkyvmH4L19"
COURSELINKS = ''
PLAINCOURSES = ''
for C in courses:
ut = url_template % (FULLNAME, C)
COURSELINKS += course_template % (ut, C)
PLAINCOURSES += C + " "
text_version = open('cache/ztc_mail1.txt','r').read()
html_version = open('cache/ztc_mail1_h.txt','r').read()
# replace these: $FNAME $COURSELINKS $LINK
email = re.sub( r'\$FNAME', FNAME, text_version )
email = re.sub( r'\$COURSELINKS', PLAINCOURSES, email )
email = re.sub( r'\$LINK', bare_link, email )
email_h = re.sub( r'\$FNAME', FNAME, html_version )
email_h = re.sub( r'\$COURSELINKS', COURSELINKS, email_h )
print(email_h+"\n\n"+email)
from O365 import Account
credentials = ('phowell@gavilan.edu', 'xxx')
client_secret = 'xxx' # expires 10/28/2020
tenant_id = "4ad609c3-9156-4b89-9496-0c0600aeb0bb"
# application client id: 29859402-fa55-4646-b717-752d90c61cde
account = Account(credentials, auth_flow_type='credentials', tenant_id=tenant_id)
if account.authenticate():
print('Authenticated!')
#account = Account(credentials)
#if account.authenticate(scopes=['message_all']):
# print('Authenticated!')
m = account.new_message()
m.to.add(addr)
m.subject = 'Quick question about your course textbook'
m.body = "email_h"
m.send()
"""
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
msg = MIMEMultipart('alternative')
msg['Subject'] = "Quick question about your course textbook"
msg['From'] = "gavdisted@gmail.com"
msg['To'] = to_email
msg.attach(MIMEText(email, 'plain'))
msg.attach(MIMEText(email_h, 'html'))
#s = smtplib.SMTP('smtp.gmail.com', 587)
#s.starttls()
#s.login("gavdisted", "xxx")
s = smtplib.SMTP_SSL('smtp.office365.com',587)
s.ehlo()
s.starttls()
s.login('phowell@gavilan.edu', 'xxx')
#s.sendmail(msg['From'], msg['To'], msg.as_string())
s.sendmail(msg['From'], msg['To'], "Testing")
s.quit()"""
def getInactiveTeachersInTerm(t=23): # a list
global results
teachers = {}
emails = {}
outfile = codecs.open('canvas/inactive_teachers.txt','w', encoding='utf-8')
efile = codecs.open('canvas/inactive_teachers_emails.txt','w', encoding='utf-8')
#yn = raw_input('All courses? y=all n=only active ')
#all = 0
#if yn=='y': all = 1
if not t:
t = askForTerms()
else: t = [ t, ]
for term in t:
r = url + '/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100'
while(r): r = fetch(r)
all_courses = results #json.loads(results)
#print "All unpublished courses: "
i = 0
j = 0
for k in all_courses:
j += 1
if k['workflow_state'] != 'available':
i += 1
print(str(i), "\t", k['name'], "\t", k['workflow_state'])
results = []
t2 = url + '/api/v1/courses/' + str(k['id']) + '/search_users?enrollment_type=teacher'
while(t2): t2 = fetch(t2)
#print results
for r in results:
key = r['sortable_name'] + "\t" + str(r['id'])
#if not 'email' in r: pdb.set_trace()
emails[key] = str(r['sis_user_id'])
#print r
if key in teachers:
teachers[key].append(k['name'])
else:
teachers[key] = [ k['name'], ]
#print json.dumps(results, indent=4, sort_keys=True)
#a = raw_input()
print(str(i), "/", str(j), " sections are unpublished")
for t in list(emails.keys()):
efile.write(emails[t] + ", ")
for t in list(teachers.keys()):
outfile.write(t + "\t")
for c in teachers[t]:
outfile.write(c + ",")
outfile.write("\n")
#f.write(json.dumps(teachers, indent=4, sort_keys=True))
print("Output file is in ./teachers/current_semester.txt")
#print json.dumps(all_courses, indent=4, sort_keys=True)
"""for x in all_courses:
qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
t = url + qry
while(t): t = fetch(t)
"""
#for t,v in teachers.items():
# outfile.write( "|".join( [ v['goo'], v['name'], v['email'], v['dept'], str(v['num_courses']), str(v['num_active_courses']) ] ) + "\n" )
#{"goo": "G00275722", "name": "Agaliotis, Paul", "num_courses": 1, "num_active_courses": 1, "id": 5092, "dept": "AMT", "classes": [["AMT120 POWERPLANT TECH FA18 10958", 5322, 1]], "email": "PAgaliotis@gavilan.edu"},
#for t in teachers.keys():
# outfile.write(t + "\t")
# for c in teachers[t]:
# outfile.write(c + ",")
# outfile.write("\n")
#f.write(json.dumps(teachers, indent=4, sort_keys=True))
#print "Output file is in ./teachers/current_semester.txt"
#print json.dumps(all_courses, indent=4, sort_keys=True)
"""for x in all_courses:
qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
t = url + qry
while(t): t = fetch(t)
"""
def course_location(course):
if len(course[0]) > 13:
period = Set( [course_location_raw(course[0][13])], )
else:
period = Set()
if len(course) > 1:
period.add(course_location_raw(course[1][13]))
if len(course) > 2:
period.add(course_location_raw(course[2][13]))
if len(course) > 3:
period.add(course_location_raw(course[3][13]))
if len(course) > 4:
period.add(course_location_raw(course[4][13]))
if len(course) > 5:
period.add(course_location_raw(course[5][13]))
if 'TBA' in period:
period.remove('TBA')
period = list(period)
if len(period)==0:
return ''
if len(period)==1:
return period[0]
if len(period)==2 and 'Online' in period:
period.remove('Online')
return 'Hybrid at ' + period[0]
return '/'.join(period)
def course_time(course):
# is it morning, mid, or evening?
period = Set( [raw_course_time(course[0][7])], )
if len(course) > 1:
#time += ", " + course[1][7]
period.add(raw_course_time(course[1][7]))
if len(course) > 2:
#time += ", " + course[2][7]
period.add(raw_course_time(course[2][7]))
if len(course) > 3:
#time += ", " + course[3][7]
period.add(raw_course_time(course[3][7]))
if len(course) > 4:
#time += ", " + course[4][7]
period.add(raw_course_time(course[4][7]))
if len(course) > 5:
#time += ", " + course[5][7]
period.add(raw_course_time(course[5][7]))
#print raw_course_time(course[0][7]),
if 'TBA' in period:
period.remove('TBA')
period = list(period)
if len(period)==0:
return ''
if len(period)==1:
return period[0]
return '/'.join(period)
def course_teacher(course):
t = Set()
for c in course:
t.add(c[11])
return " / ".join(list(t))
def reg_nums():
courses = []
dates = []
sections = categorize()
today = todays_date_filename()
out = open(today+'.csv','w')
dates = {'loc':{}, 'time':{}, 'start':{}, 'teacher':{}}
i = 1
for f in os.listdir('.'):
m = re.search('reg_'+short_sem+'_(\d+)\.csv',f)
if m:
filein = open(f,'r').readlines()[1:]
d = m.group(1)
dates[d] = {}
for L in filein:
parts = L.split(',') # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
if not re.search('(\d+)',parts[0]): continue
if len(parts)<8: continue
if not parts[8]: continue
if float(parts[8])==0: continue
dates[d][parts[0] + " " + parts[1]] = (1.0* float(parts[9])) / float(parts[8])
if i == 1 and parts[0] in sections:
dates['loc'][parts[0] + " " + parts[1]] = course_location( sections[parts[0]] )
dates['time'][parts[0] + " " + parts[1]] = course_time(sections[parts[0]] )
dates['start'][parts[0] + " " + parts[1]] = course_start( sections[parts[0]] )
dates['teacher'][parts[0] + " " + parts[1]] = course_teacher( sections[parts[0]] )
#dates[d]['act'] = parts[9]
#dates[d]['nam'] = parts[5]
#dates[d]['onl'] = ''
#print parts
#if len(parts)>13 and parts[13]=='ONLINE': dates[d]['onl'] = 'online'
i += 1
"""for d in sorted(dates.keys()):
for c in d:
print d
print dates[d]['crs']"""
df = pd.DataFrame(dates)
df.to_csv(out)
# In the schedule, is this a class or a continuation of the class above?
def categorize():
# todo: must we open all these files?
dates = {}
files = sorted(os.listdir('.'))
files = list( filter( lambda x: re.search('reg(\d+)\.csv',x), files) )
files.reverse()
f = files[0]
filein = codecs.open(f,'r','utf-8').readlines()[1:]
sections = {}
this_section = []
for L in filein:
parts = L.strip().split(',') # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
parts = list( map( lambda x: clean_funny3(x), parts ) )
if not re.search('(\d+)',parts[0]): # This is a continuation
this_section.append(parts)
else: # this is a new section or the first line
if this_section:
sections[ this_section[0][0] ] = this_section
#print "Section: " + this_section[0][0] + " is: " + str(this_section) + "\n"
#print this_section[0][0] + "\t", course_start(this_section)
#print this_section[0][0] + "\t", course_time(this_section)
#print this_section[0][0] + "\t", course_location(this_section)
this_section = [ parts, ]
return sections
# Deprecated. call perl.
def constructSchedule():
term = raw_input("Name of html file? (ex: sp18.html) ")
os.chdir('make-web-sched')
cmd = 'perl make.pl ' + term
print "command: " + cmd
os.system(cmd)
"""
def fetch_dict(target,params={}):
# if there are more results, return the url for more fetching.
# else return false
#print target
global results_dict
r2 = requests.get(target, headers = header, params=params)
output = r2.text
if output.startswith('while('):
output = output[9:]
#print output
mycopy = results_dict.copy()
results_dict = {}
results_dict.update(json.loads(output))
results_dict.update(mycopy)
f.write(json.dumps(results_dict, indent=2))
#print "\n"
if ('link' in r2.headers):
links = r2.headers['link'].split(',')
for L in links:
ll = L.split(';')
link = ll[0].replace("<","")
link = link.replace(">","")
if re.search(r'next', ll[1]):
#print ll[1] + ":\t" + link
return link
return ""
"""
def get_schedule(term='201870', sem='fall'):
"""
sched_data = { 'term_in':term, 'sel_subj':'dummy', 'sel_day':'dummy',
'sel_schd':'dummy', 'sel_insm':'dummy', 'sel_camp':'dummy', 'sel_levl':'dummy', 'sel_sess':'dummy',
'sel_instr':'dummy', 'sel_ptrm':'dummy', 'sel_attr':'dummy', 'sel_subj':'%', 'sel_crse':'', 'sel_title':'',
'sel_schd':'%', 'sel_from_cred':'', 'sel_to_cred':'', 'sel_camp':'%', 'sel_ptrm':'%', 'sel_sess':'%',
'sel_attr':'%', 'begin_hh':'0', 'begin_mi':'0', 'begin_ap':'a', 'end_hh':'0', 'end_mi':'0', 'end_ap':'a' }
initial_headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, sdch, br',
'Accept-Language':'en-US,en;q=0.8',
'Connection':'keep-alive',
'Host':'ssb.gavilan.edu',
'Upgrade-Insecure-Requests':'1',
} #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
headers = { 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.8',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Content-Type':'application/x-www-form-urlencoded',
'Host':'ssb.gavilan.edu',
'Origin':'https://ssb.gavilan.edu',
'Referer':'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term='+term,
'Upgrade-Insecure-Requests':'1',
} #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
initial_url = 'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=' + term
sesh = requests.Session()
#r1 = sesh.get(initial_url,headers=initial_headers)
#sesh.headers.update(headers)
url = 'https://ssb.gavilan.edu/prod/bwckschd.p_get_crse_unsec'
r1 = sesh.get(initial_url)
r = sesh.post(url, data=sched_data)
print r.headers
data = r.text
out = open('data/temp/'+term+'.html','w')
out.write(data)
out.close()"""
os.system('perl parse_schedule.pl data/temp/' + term + '.html' + ' ' + sem)
#####
#####
##### conf.py ?
str="""355 985 1296
354 730 1295
353 319 1290
352 985 1289
351 813 1285
350 281 1285
349 267 1279
348 981 1252
347 994 1252
346 26 1250
345 757 1288
344 368 1288
343 1 1286
259 703 1295
256 693 1293
255 660 1292
254 1 1291
250 482 1287
246 2 1284
245 333 1283
244 27 1282
243 703 1281
242 730 1281
241 482 1280
239 211 1278
238 794 1278
237 2 1277
236 297 1276
235 831 1276
233 482 1251"""
for L in str.split("\n"):
(id,host,session) = L.split("\t")
qry = "INSERT INTO conf_signups (user,session,timestamp) VALUES (%s,%s,'2022-08-08 17:20:00');" % (host,session)
print(qry)
## sched.py
import requests, re, csv, json, funcy, sys
def dates(s):
#print(s)
m = re.match(r'(\d\d\d\d)\-(\d\d)\-(\d\d)',s)
if m:
s = m.group(2) + "/" + m.group(3)
#print(s)
return s
# "Course Code","Start Date","End Date",Term,Delivery,CRN,Status,"Course Name","Course Description","Units/Credit hours","Instructor Last Name","Instructor First Name",Campus/College,"Meeting Days and Times","Pass/No Pass available?","Class Capacity","Available Seats","Waitlist Capacity","Current Waitlist Length","Meeting Locations","Course Notes",ZTC
# ACCT103,2021-06-14,2021-07-23,"Summer 2021",Online,80386,Active,"General Office Accounting","This course is designed to prepare students for entry-level office accounting positions. Emphasis is on practical accounting applications. This course has the option of a letter grade or pass/no pass. ADVISORY: Eligible for Mathematics 430."," 3.00","Valenzuela Roque",Karla,"Gavilan College"," ",T," 30"," 18"," 20"," 0",,,
def parse_www_csv_sched():
old_keys = [ "CRN","Course Code","Units/Credit hours","Course Name","Meeting Days and Times","Class Capacity","Available Seats","Waitlist Capacity","Current Waitlist Length","Instructor Last Name","Start Date","Meeting Locations","ZTC","Delivery","Campus/College","Status","Course Description","Pass/No Pass available?","Course Notes" ]
# "Instructor First Name","End Date","Term",
new_keys = [ "crn", "code","cred", "name", "days", "cap", "rem", "wl_cap", "wl_act", "teacher", "date", "loc", "ztc", "type", "site","status","desc","pnp","note" ]
# "time","act","wl_rem", "partofday",
url = "https://gavilan.edu/_files/php/current_schedule.csv"
sched_txt = requests.get(url).text.splitlines()
sched = {"Fall 2021":[], "Spring 2022":[], "Winter 2022":[], "Summer 2021":[]}
shortsems = {"Fall 2021":"fa21", "Spring 2022":"sp22", "Winter 2022":"wi22", "Summer 2021":"su21","Summer 2022":"su22","Fall 2022":"fa22"}
for row in csv.DictReader(sched_txt):
d = dict(row)
for (old_key,new_key) in zip(old_keys,new_keys):
d[new_key] = d.pop(old_key).strip()
d['teacher'] = d.pop('Instructor First Name').strip() + " " + d['teacher']
d['date'] = dates(d['date']) + '-' + dates(d.pop('End Date').strip())
d['term'] = shortsems[d.pop('Term')]
if d['cred'] == ".00":
d['cred'] = "0"
if d['type'] == "Online":
d["loc"] = "ONLINE"
d["site"] = "Online"
d["type"] = "online"
#d.pop('Instructor First Name').strip() + " " + d['teacher']
#d["code"] = d.pop("Course Code")
#d["crn"] = d.pop("CRN")
sched[row['Term']].append(d) #print(row)
print( json.dumps(sched,indent=2))
for k,v in sched.items():
print("%s: %i" % (k,len(v)))
for v in sched["Fall 2021"]:
print("%s\t %s\t %s\t %s" % ( v['code'], v['days'], v['type'], v['loc'] ))
#print("%s\t %s\t %s\t %s" % ( v['Course Code'], v['Meeting Days and Times'], v['Delivery'], v['Meeting Locations'] ))
def parse_json_test_sched():
j2 = open('cache/classes_json.json','r').readlines()
for L in j2:
o3 = json.loads(L)
print(json.dumps(o3,indent=2))
if __name__ == "__main__":
print ('')
options = {
1: ['fetch and parse the csv on www.', parse_www_csv_sched],
2: ['parse the test json file.', parse_json_test_sched ],
}
if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
resp = int(sys.argv[1])
print("\n\nPerforming: %s\n\n" % options[resp][0])
else:
print ('')
for key in options:
print(str(key) + '.\t' + options[key][0])
print('')
resp = input('Choose: ')
# Call the function in the options dict
options[ int(resp)][1]()