#get_schedule('201770')
# from pipelines - canvas data
"""
timestamp = nowAsStr()
requestParts = [ method,
host,
'', #content Type Header
'', #content MD5 Header
path,
'', #alpha-sorted Query Params
timestamp,
apiSecret ]
#Build the request
requestMessage = '\n'.join( requestParts )
requestMessage = requestMessage.encode('ASCII')
print((requestMessage.__repr__()))
hmacObject = hmac.new(bytearray(apiSecret,'ASCII'), bytearray('','ASCII'), hashlib.sha256) #
hmacObject.update(requestMessage)
hmac_digest = hmacObject.digest()
sig = base64.b64encode(hmac_digest)
headerDict = {
'Authorization' : 'HMACAuth ' + apiKey + ':' + str(sig),
'Date' : timestamp
}
"""
# Don't know
def demo():
resp = do_request('/api/account/self/file/sync')
mylog.write(json.dumps(resp, indent=4))
sample_table = resp['files'][10]
filename = sample_table['filename']
print(sample_table['table'])
response = requests.request(method='GET', url=sample_table['url'], stream=True)
if(response.status_code != 200):
print(('Request response went bad. Got back a ', response.status_code, ' code, meaning the request was ', response.reason))
else:
#Use the downloaded data
with open(local_data_folder + filename, 'wb') as fd:
for chunk in response.iter_content(chunk_size=128):
fd.write(chunk)
print("Success")
if filename.split('.')[-1] == 'gz':
plain_filename = 'canvas_data/' + ".".join(filename.split('.')[:-1])
pf = open(plain_filename,'w')
with gzip.open('canvas_data/' + filename , 'rb') as f:
pf.write(f.read())
# How to drop columns
#columns = ['Col1', 'Col2', ...]
#df.drop(columns, inplace=True, axis=1)
# left join, one on column, one on index
#merged = pd.merge(result,users,left_index=True,right_on='id', how='left')
"""
You can call set_index on the result of the dataframe:
In [2]:
data=[['Australia',100],['France',200],['Germany',300],['America',400]]
pd.DataFrame(data,columns=['Country','Volume']).set_index('Country')
Out[2]:
Volume
Country
Australia 100
France 200
Germany 300
America 400
"""
def stats():
# nothing seems to happen here?
#input = csv.DictReader(codecs.open(schedfile,'r','utf-8'))
input = csv.DictReader(open(schedfile,'r'))
out2 = open('temp2.csv','w')
clean = {}
for r in input:
if r['crn']: clean[ r['crn'] ] = r
for c,r in list(clean.items()):
try:
if int(r['cap'])==0: continue
else: prct = (1.0 * int( r['act'] )) / int(r['cap'])
if prct < 0.01: continue
o_str = ''
if r['location'].strip()=='ONLINE': o_str = 'online'
#print r['location']
date_parts = r['date'].split('-')
start = strptime(date_parts[0], '%m/%d')
if start > semester_begin: o_str += "\tlatestart " + date_parts[0]
out2.write( "".join([c, "\t", r['sub'], "\t", r['crs'], "\t", str(round(prct,2)), "% full\t", o_str, "\n"]) )
except:
pass
######### from curriculum. py
# open('cache/programs/programs_1.txt','r').read()
"""
SEE serve.py .... i mean ... interactive.py
def dict_generator(indict, pre=None):
pre = pre[:] if pre else []
if isinstance(indict, dict):
for key, value in indict.items():
if isinstance(value, dict):
for d in dict_generator(value, pre + [key]):
yield d
elif isinstance(value, list) or isinstance(value, tuple):
for v in value:
for d in dict_generator(v, pre + [key]):
yield d
else:
yield str(pre) + " " + str([key, value]) + "\n"
else:
yield pre + [indict]
yield str(pre) + " " + str([indict]) + "\n"
def print_dict(v, prefix='',indent=''):
if isinstance(v, dict):
return [ print_dict(v2, "{}['{}']".format(prefix, k) + "
", indent+" " ) for k, v2 in v.items() ]
elif isinstance(v, list):
return [ print_dict( v2, "{}[{}]".format(prefix , i) + "
", indent+" ") for i, v2 in enumerate(v) ]
else:
return '{} = {}'.format(prefix, repr(v)) + "\n"
def walk_file():
j = json.loads(open('cache/programs/programs_2.txt','r').read())
return print_dict(j)
from flask import Flask
from flask import request
def tag(x,y): return "<%s>%s" % (x,y,x)
def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
def a(t,h): return '%s' % (h,t)
def server_save(key,value):
codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
def flask_thread(q):
app = Flask(__name__)
@app.route("/")
def home():
return tag('h1','This is my server.') + "
" + a('want to shut down?','/sd')
@app.route("/save//")
def s(key,val):
server_save(key,val)
return tag('h1','Saved.') + "
" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
@app.route("/crazy")
def hello():
r = ''
r += tag('style', 'textarea { white-space:nowrap; }')
r += tag('body', \
tagc('div','container-fluid', \
tagc('div','row', \
tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
return r
@app.route("/sd")
def sd():
print('SIGINT or CTRL-C detected. Exiting gracefully')
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
return "Server has shut down."
app.run()
from queue import Queue
q = Queue()
def serve():
import webbrowser
import threading
x = threading.Thread(target=flask_thread, args=(q,))
x.start()
webbrowser.open_new_tab("http://localhost:5000")
#s = open('cache/programs/index.json','w')
#s.write( json.dumps({'departments':sorted(list(dept_index)), 'programs':prog_index}, indent=2) )
#s.close()
"""
# Prompt for course id, return list of user dicts. TODO this duplicates courses.py ??
def getUsersInCourse(id=0): # returns list
if not id:
id = str(input("The Course ID? "))
id = str(id)
return fetch('/api/v1/courses/%s/users' % id, 0)
#### curriculum.py
def recur_look_for_leafs(item,indent=0,show=1):
global leafcount, displaynames
ii = indent * " "
is_leaf = am_i_a_leaf(item)
if type(item) == type({}):
status = ""
if show:
status = "Dict"
if is_leaf:
leafcount += 1
status = "Leaf Dict"
if status:
print("\n%s%s" % (ii,status))
indent += 1
ii = indent * " "
for K,V in list(item.items()):
if show or is_leaf:
print("%s%s:" % (ii, K), end="")
if K =='displayName': displaynames.append(V)
recur_look_for_leafs(V,indent+1,show or is_leaf)
elif type(item) == type([]):
status = ""
if show: status = "List (" + str( len(item) ) + ")"
if is_leaf: status = "Leaf List (" + str( len(item) ) + ")"
if status:
print("\n%s%s" % (ii,status))
indent += 1
ii = indent * " "
for V in item:
recur_look_for_leafs(V,indent+1, show or is_leaf)
elif type(item) == type("abc"):
if show: print("%s%s" % (' ', item))
elif type(item) == type(55):
if show: print("%s%i" % (' ', item))
elif type(item) == type(5.5):
if show: print("%s%f" % (' ', item))
elif type(item) == type(False):
if show: print("%s%s" % (' ', str(item)))
def am_i_a_leaf(item):
if type(item) == type({}):
for K,V in list(item.items()):
if type(V) == type({}) or type(V) == type([]):
return False
elif type(item) == type([]):
for V in item:
if type(V) == type({}) or type(V) == type([]):
return False
elif type(item) == type("abc"): return True
elif type(item) == type(55): return True
elif type(item) == type(5.5): return True
elif type(item) == type(False):
if item == False: return True
elif item == True: return True
return True
def sampleclass():
theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
#print(json.dumps(theclass,indent=2))
recur_look_for_leafs(theclass)
print(leafcount)
print(sorted(displaynames))
def recur_matcher(item, depth=0):
indent = depth * " "
my_result_lines = []
if type(item) == type({}):
if not match( item,
{'entityMetadata': {'entityTitle': _,'status': _, 'entityType':_, 'entityId':_ }},
lambda title,status,typ,id:
my_result_lines.append("%s%s: %s (id %s) status: %s" % (indent, str(typ), str(title), str(id), str(status))) ,
{'attributes': {'displayName': _}, 'lookUpDisplay': _, },
lambda x,y: my_result_lines.append("%s%s: %s" % (indent, clean(str(x)), clean(str(y)))) ,
{'attributes': {'displayName': _}, 'fieldValue': _, },
lambda x,y: my_result_lines.append("%s%s: %s" % (indent, clean(str(x)), clean(str(y)))) ,
{'sectionName': _},
lambda x: my_result_lines.append("%sSection: %s" % (indent, str(x))) ,
_, nothing
):
for K,V in list(item.items()):
my_result_lines.extend(recur_matcher(V,depth+1))
elif type(item) == type([]):
for V in item:
my_result_lines.extend(recur_matcher(V,depth+1))
return my_result_lines
def matchstyle():
theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
print("\n".join(recur_matcher(theclass)))
# 7: ['pattern matcher style', matchstyle],
# 8: ['pattern matcher - test on all classes', match_style_test],
##### from localcache
stem_course_id = '11015' # TODO
# NO LONGER USED - SEE COURSES
def enroll_stem_students():
depts = "MATH BIO CHEM PHYS ASTR GEOG".split(" ")
students = set()
for d in depts:
students.update(dept_classes(d))
print(students)
to_enroll = [ x for x in students if x not in already_enrolled ]
print(to_enroll)
print("prev line is people to enroll\nnext line is students already enrolled in stem")
print(already_enrolled)
for s in to_enroll:
t = url + '/api/v1/courses/%s/enrollments' % stem_course_id
data = { 'enrollment[user_id]': s[1], 'enrollment[type]':'StudentEnrollment',
'enrollment[enrollment_state]': 'active' }
print(data)
print(t)
if input('enter to enroll %s or q to quit: ' % s[0]) == 'q':
break
r3 = requests.post(t, headers=header, params=data)
print(r3.text)
#####
##### from users.py pretty much just use sql now
# unused?
def getAllTeachersInTerm(): # a list
# classes taught in last 3 semesters
# How many of them were published and used
# hits in last week/month/year
# most common department
# email addr
all_courses = {}
teachers = {} # keyed by goo
# { 'name':'', 'id':'', 'email':'', 'goo':'', 'classes':[ (#name,#id,#pubd,#hitsbyteacher) ... ] }
# This is a bit different from the 1 year schedule above, because it looks at
# people who were active in their shells in iLearn.
outfile = codecs.open('teacherdata/historical_shells_used.json','w', encoding='utf-8')
for term in last_4_semesters_ids: # [60,]:
print(("Fetching term: " + str(term)))
all_courses[term] = \
fetch('/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100')
i = 0
j = 0
for k,v in list(all_courses.items()): ##### term k, list v
for a_class in v:
print((a_class['name']))
published = 0
if a_class['workflow_state'] in ['available','completed']:
j += 1
published = 1
i += 1
#if i > 20: break
tch = fetch('/api/v1/courses/' + str(a_class['id']) + '/search_users?enrollment_type=teacher')
for r in tch: ##### TEACHER r of COURSE a_class
name = str(r['sortable_name'])
if not 'sis_import_id' in r:
print("This user wasn't available: " + name)
continue
goo = str(r['sis_import_id'])
print((r['sortable_name']))
if not name in teachers:
email = getEmail(r['id'])
teachers[name] = { 'name':r['sortable_name'], 'id':r['id'], 'email':email, 'goo':goo, 'classes':[] }
info = (a_class['name'],a_class['id'],published)
teachers[name]['classes'].append( info )
## TODO: hits in courses by teachers https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
for t,v in list(teachers.items()):
teachers[t]['num_courses'] = len(v['classes'])
teachers[t]['num_active_courses'] = sum( [x[2] for x in v['classes']] )
depts = [ dept_from_name(x[0]) for x in v['classes'] ]
teachers[t]['dept'] = most_common_item(depts)
#print(str(j), "/", str(i), " sections are published")
outfile.write(json.dumps(teachers))
"""
def teacherActivityLog(uid=1): ### Next: save results in a hash and return that....
global results, users, users_by_id
#get_users() # do this if you think 'teachers/users.json' is outdated.
load_users()
#for x in users_by_id.keys():
# if x < 20:
# print x
# print users_by_id[x]
teachers = csv.reader(open('teachers/current_semester.txt','r'), delimiter="\t")
for row in teachers:
print(row[0] + " is id: " + row[1])
uid = row[1]
print("Comes up as: " + str(users_by_id[int(uid)]))
info = users_by_id[int(uid)]
goo = info['login_id']
output_file = open('logs/users/byweek/'+ goo.lower() + '.csv', 'w')
# okay, actually, the first week here is the week before school IRL
start = isoweek.Week.withdate( datetime.date(2017,8,21))
end = isoweek.Week.thisweek()
byweek = []
i = 0
while(1):
results = []
start = start + 1
if start > end: break
myStart = start.day(0).isoformat() + 'T00:00-0700'
myEnd = start.day(6).isoformat() + 'T11:59:59-0700'
t = url + "/api/v1/users/" + str(uid) + "/page_views?start_time=" + myStart + '&end_time=' + myEnd + "&perpage=500"
print(t)
while(t):
print(".", end=' ')
t = fetch(t)
print("")
thisWeek = len(results)
print("Week # " + str(i) + "\t" + str(thisWeek))
byweek.append( "Week # " + str(i) + "\t" + str(thisWeek) )
output_file.write( start.isoformat() + "," + str(thisWeek) + "\n")
i += 1
for j in byweek:
print(j)
"""
"""
def summarize_student_teacher_role(u):
# u is a "group" from the groupby fxn
# term is sp18 now
t = 0
s = 0
for a in u:
if a=='TeacherEnrollment': t += 1
else: s += 1
if NUM_ONLY:
if t > s: return 'teacher'
return 'student'
else:
if t > s: return '1'
return '0'
"""
"""
def user_roles2():
# cross list users, classes enrolled, and their roles
global role_table, term_courses
role_table = enrollment_file()
user_table = users_file()
course_table = courses_file() # from canvas
term_table = term_file()
schedule = current_schedule() # from banner
# current semester
current = term_table[lambda d: d.course_section=='2018 Spring']
term_id = current['id'].values[0]
term_courses = course_table[lambda d: d.termid==term_id] # courses this semester
# add is_online flag (for courses listed in schedule as online-only)
term_courses['is_online'] = term_courses['code'].map( lambda x: course_is_online( get_crn_from_name(x) ) )
new_df = pd.DataFrame(columns=['type','oo','num'])
m = 0
data = []
for u in user_table.iterrows():
if m % 1000 == 0: print("on row " + str(m))
m += 1
data.append(categorize_user(u))
#if m > 1500: break
new_df = pd.DataFrame(data,columns=['i','type','onlineonly','numcls']).set_index('i')
print(new_df)
user_table = user_table.merge(new_df,left_index=True,right_index=True)
user_table.to_csv('canvas_data/users_online.csv')
"""
### IS THIS IN CANVAS_DATA.py?
""" Collate the raw logs into something more compact and useful. Version 1:
- # of accesses, user/day
- # of participations, user/day
-
- where day is the number of days into the semester. Classes shorter than 16 weeks should get a multiplier
-
- 2 initial goals:
a. data for statistics / clustering / regression / learning
b. data for visualization
"""
def req_to_db(fname_list):
fields = ','.join("id timestamp timestamp_year timestamp_month timestamp_day user_id course_id root_account_id course_account_id quiz_id discussion_id conversation_id assignment_id url user_agent http_method remote_ip interaction_micros web_application_controller web_applicaiton_action web_application_context_type web_application_context_id real_user_id session_id user_agent_id http_status http_version".split(" "))
sqlite_file = 'canvas_data/data.db'
conn = sqlite3.connect(sqlite_file)
c = conn.cursor()
# merge all requests into db
by_date_course = defaultdict( lambda: defaultdict(int) )
by_date_user = defaultdict( lambda: defaultdict(int) )
df_list = []
df_list_crs = []
users = defaultdict( lambda: defaultdict(int) )
i = 0
limit = 300
for fname in fname_list:
print((fname+"\n"))
for line in gzip.open('canvas_data/'+fname,'r'):
r = line.split('\t')
#tot = len(fields.split(','))
#i = 0
#for x in fields.split(','):
# print x + "\t" + r[i]
# i+= 1
qry = "insert into requests("+fields+") values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
conn.execute(qry, r)
# New method for below:
# read collated data from sqlite
# collate from more logs
# write back....?
"""
date = datetime.datetime.strptime( r['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
if r['userid'] in users:
users[r['userid']]['freq'] += 1
if users[r['userid']]['lastseen'] < date:
users[r['userid']]['lastseen'] = date
else:
users[r['userid']] = {"id":r['userid'], "lastseen":date, "freq":1}
by_date_course[ r['day'] ][ r['courseid'] ] += 1
by_date_user[ r['day'] ][ r['userid'] ] += 1
#if r['userid'] in by_user: by_user[r['userid']] += 1
#else: by_user[r['userid']] = 1
#if r['courseid'] in by_course: by_course[r['courseid']] += 1
#else: by_course[r['courseid']] = 1
#mylog.write("by_user = " + str(by_user))
df_list.append(pd.DataFrame(data=by_date_user))
df_list_crs.append(pd.DataFrame(data=by_date_course))
"""
i += 1
if i > limit: break
conn.commit()
conn.close()
"""
Making columns:
table_data = [['a', 'b', 'c'], ['aaaaaaaaaa', 'b', 'c'], ['a', 'bbbbbbbbbb', 'c']]
for row in table_data:
print("{: >20} {: >20} {: >20}".format(*row))
Transpose a matrix:
rez = [[m[j][i] for j in range(len(m))] for i in range(len(m[0]))]
"""
"""
ilearn_by_id = {}
ilearn_by_name = {}
for x in ilearn_list:
ilearn_by_id[x[3]] = x
ilearn_by_name[x[0]] = x
for ml in open('cache/teacher_manual_name_lookup.csv','r').readlines():
parts = ml.strip().split(',')
try:
manual_list[parts[0]] = ilearn_by_id[parts[1]]
except Exception as e:
print "Teacher missing: " + parts[0]
il_names = [ x[0] for x in ilearn_list ]
il_byname = {}
for x in ilearn_list: il_byname[x[0]] = x
sched_list_missed = [x for x in sched_list]
#
# key is long name (with middle name) from schedule, value is tuple with everything
name_lookup = manual_list
matches = []
#print ilearn_list
num_in_sched = len(sched_list)
num_in_ilearn = len(ilearn_list)
#for i in range(min(num_in_sched,num_in_ilearn)):
# print "|"+sched_list[i] + "|\t\t|" + ilearn_list[i][0] + "|"
print("Sched names: %i, iLearn names: %i" % (num_in_sched,num_in_ilearn))
for s in sched_list:
for t in il_names:
if first_last(s) == t:
#print ' MATCHED ' + s + ' to ' + t
sched_list_missed.remove(s)
try:
name_lookup[s] = ilearn_by_name[ first_last(s) ]
except Exception as e:
print "Teacher missing (2): " + s
il_names.remove(first_last(s))
matches.append(s)
print "Matched: " + str(matches)
print "\nDidn't match: " + str(len(sched_list_missed)) + " schedule names."
print "\nFinal results: "
print name_lookup
nlf = codecs.open('cache/sched_to_ilearn_names.json','w','utf-8')
nlf.write(json.dumps(name_lookup,indent=2))
# STRING DISTANCE
#sim = find_most_similar(s,i_names)
#print ' CLOSEST MATCHES to ' + s + ' are: ' + str(sim)
#mm.write(s+',\n')
"""
#ilearn_list = sorted(list(set(map(
# lambda x: #(tfi[x]['name'],tfi[x]['email'],tfi[x]['dept'],str(tfi[x]['id']),tfi[x]['goo']),
# tfi.keys()))))
#i_names = [ x[0] for x in ilearn_list ]
#print json.dumps(i_names,indent=2)
#return
# how to filter a dict based on values
# filtered = {k: v for k, v in course_combos.items() if v['dept'] == 'LIB' or v['dept'] == 'CSIS' }
# more pandas
# gapminder['continent'].unique()
#for name,group in bycode:
# #print name
# print name, " ", group['type']
#onl = gg.agg( lambda x: has_online(x) )
#ttl = gg.agg( lambda x: len(x) )
#ttl = ttl.rename(columns={'type':'total_sections'})
#onl.join(gg.agg( lambda x: has_hybrid(x) ),how='outer')
#onl.join(gg.agg( lambda x: has_lecture(x) ), how='outer')
#onl['num_sections'] = 0
#onl['num_lec'] = 0
#onl['num_online'] = 0
#all = pd.merge([onl,hyb,lec])
#print onl
#total=len, f2f=lambda x: ) set(x)
#{ 'num_sections': "count",
# 'num_lec': lambda x: 5,
# 'num_online': lambda x: 5 } )
#print gg
"""
def has_online(series):
# if any items of the series have the string 'online', return 1
for i in series:
if i == 'online': return 1
return 0
def has_lecture(series):
# if any items of the series have the string 'online', return 1
for i in series:
if i == 'online': return 1
return 0
def has_hybrid(series):
# if any items of the series have the string 'online', return 1
for i in series:
if i == 'hybrid': return 1
return 0
"""
#### RIGHT HERE IS WHERE I THINK... MAYBE THIS ISN'T THE RIGHT APPROACH. I DON'T SEEM
#### TO BE ABLE TO QUERY THE FACT BASE. IS THAT TRUE? SHOULD I JUST BE USING TABLES?
#### CHANGING COURSE... USE THE RULES TO UPDATE A DATABASE/TABLE/DATAFRAME
#### OR SET OF DICTS.
# ultimately i want this to be more flexible, so i can categorize degrees as 'available evening' etc
#
# Simple data structure. In this function, a degree is
""" degree = { 'name': 'History AA',
'blocks': [ { 'original_title':'xxx', 'rulecode':'u3',
'courses': [ {'code':'math1a', 'units': '3.0', 'wasonline':False },
{'code':'math2a', 'units': '3.0', 'wasonline':False },
{'code':'math3a', 'units': '3.0', 'wasonline':False } ] },
{ 'original_title':'xyz', 'rulecode':'a',
'courses': [ {'code':'math5a', 'units': '3.0', 'wasonline':False },
{'code':'math6a', 'units': '3.0', 'wasonline':False },
{'code':'math7a', 'units': '3.0', 'wasonline':False } ] } ] }
"""
# Wrapper to get 2 schedules at once
def dl_sched():
global SEMESTER, semester_begin, filename, short_sem
SEMESTER = 'Fall 2019'
short_sem = 'fa19'
semester_begin = strptime('08/26', '%m/%d')
filename = 'fa19_sched.json'
txt = login()
codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
#stats()
#reg_nums()
#todo: these semesters
SEMESTER = 'Summer 2019'
short_sem = 'su19'
semester_begin = strptime('06/17', '%m/%d')
filename = 'su19_sched.json'
txt = login()
codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
#stats()
#reg_nums()
# Send a personalized email regarding ZTC
def send_z_email(fullname, firstname, addr, courses_list):
FULLNAME = fullname #"Sabrina Lawrence"
FNAME = firstname # "Sabrina"
to_email = addr # "slawrence@gavilan.edu"
courses = courses_list # ["CSIS45", "CSIS85"]
course_template = "%s "
url_template = "https://docs.google.com/forms/d/e/1FAIpQLSfZLQp6wHFEdqsmpZ7jz2Y8HtKLo8XTAhrE2fyvTDOEgquBDQ/viewform?usp=pp_url&entry.783353363=%s&entry.1130271051=%s" # % (FULLNAME, COURSE1)
bare_link = "https://forms.gle/pwZJHdWSkyvmH4L19"
COURSELINKS = ''
PLAINCOURSES = ''
for C in courses:
ut = url_template % (FULLNAME, C)
COURSELINKS += course_template % (ut, C)
PLAINCOURSES += C + " "
text_version = open('cache/ztc_mail1.txt','r').read()
html_version = open('cache/ztc_mail1_h.txt','r').read()
# replace these: $FNAME $COURSELINKS $LINK
email = re.sub( r'\$FNAME', FNAME, text_version )
email = re.sub( r'\$COURSELINKS', PLAINCOURSES, email )
email = re.sub( r'\$LINK', bare_link, email )
email_h = re.sub( r'\$FNAME', FNAME, html_version )
email_h = re.sub( r'\$COURSELINKS', COURSELINKS, email_h )
print(email_h+"\n\n"+email)
from O365 import Account
credentials = ('phowell@gavilan.edu', 'xxx')
client_secret = 'xxx' # expires 10/28/2020
tenant_id = "4ad609c3-9156-4b89-9496-0c0600aeb0bb"
# application client id: 29859402-fa55-4646-b717-752d90c61cde
account = Account(credentials, auth_flow_type='credentials', tenant_id=tenant_id)
if account.authenticate():
print('Authenticated!')
#account = Account(credentials)
#if account.authenticate(scopes=['message_all']):
# print('Authenticated!')
m = account.new_message()
m.to.add(addr)
m.subject = 'Quick question about your course textbook'
m.body = "email_h"
m.send()
"""
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
msg = MIMEMultipart('alternative')
msg['Subject'] = "Quick question about your course textbook"
msg['From'] = "gavdisted@gmail.com"
msg['To'] = to_email
msg.attach(MIMEText(email, 'plain'))
msg.attach(MIMEText(email_h, 'html'))
#s = smtplib.SMTP('smtp.gmail.com', 587)
#s.starttls()
#s.login("gavdisted", "xxx")
s = smtplib.SMTP_SSL('smtp.office365.com',587)
s.ehlo()
s.starttls()
s.login('phowell@gavilan.edu', 'xxx')
#s.sendmail(msg['From'], msg['To'], msg.as_string())
s.sendmail(msg['From'], msg['To'], "Testing")
s.quit()"""
def getInactiveTeachersInTerm(t=23): # a list
global results
teachers = {}
emails = {}
outfile = codecs.open('canvas/inactive_teachers.txt','w', encoding='utf-8')
efile = codecs.open('canvas/inactive_teachers_emails.txt','w', encoding='utf-8')
#yn = raw_input('All courses? y=all n=only active ')
#all = 0
#if yn=='y': all = 1
if not t:
t = askForTerms()
else: t = [ t, ]
for term in t:
r = url + '/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100'
while(r): r = fetch(r)
all_courses = results #json.loads(results)
#print "All unpublished courses: "
i = 0
j = 0
for k in all_courses:
j += 1
if k['workflow_state'] != 'available':
i += 1
print(str(i), "\t", k['name'], "\t", k['workflow_state'])
results = []
t2 = url + '/api/v1/courses/' + str(k['id']) + '/search_users?enrollment_type=teacher'
while(t2): t2 = fetch(t2)
#print results
for r in results:
key = r['sortable_name'] + "\t" + str(r['id'])
#if not 'email' in r: pdb.set_trace()
emails[key] = str(r['sis_user_id'])
#print r
if key in teachers:
teachers[key].append(k['name'])
else:
teachers[key] = [ k['name'], ]
#print json.dumps(results, indent=4, sort_keys=True)
#a = raw_input()
print(str(i), "/", str(j), " sections are unpublished")
for t in list(emails.keys()):
efile.write(emails[t] + ", ")
for t in list(teachers.keys()):
outfile.write(t + "\t")
for c in teachers[t]:
outfile.write(c + ",")
outfile.write("\n")
#f.write(json.dumps(teachers, indent=4, sort_keys=True))
print("Output file is in ./teachers/current_semester.txt")
#print json.dumps(all_courses, indent=4, sort_keys=True)
"""for x in all_courses:
qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
t = url + qry
while(t): t = fetch(t)
"""
#for t,v in teachers.items():
# outfile.write( "|".join( [ v['goo'], v['name'], v['email'], v['dept'], str(v['num_courses']), str(v['num_active_courses']) ] ) + "\n" )
#{"goo": "G00275722", "name": "Agaliotis, Paul", "num_courses": 1, "num_active_courses": 1, "id": 5092, "dept": "AMT", "classes": [["AMT120 POWERPLANT TECH FA18 10958", 5322, 1]], "email": "PAgaliotis@gavilan.edu"},
#for t in teachers.keys():
# outfile.write(t + "\t")
# for c in teachers[t]:
# outfile.write(c + ",")
# outfile.write("\n")
#f.write(json.dumps(teachers, indent=4, sort_keys=True))
#print "Output file is in ./teachers/current_semester.txt"
#print json.dumps(all_courses, indent=4, sort_keys=True)
"""for x in all_courses:
qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
t = url + qry
while(t): t = fetch(t)
"""
def course_location(course):
if len(course[0]) > 13:
period = Set( [course_location_raw(course[0][13])], )
else:
period = Set()
if len(course) > 1:
period.add(course_location_raw(course[1][13]))
if len(course) > 2:
period.add(course_location_raw(course[2][13]))
if len(course) > 3:
period.add(course_location_raw(course[3][13]))
if len(course) > 4:
period.add(course_location_raw(course[4][13]))
if len(course) > 5:
period.add(course_location_raw(course[5][13]))
if 'TBA' in period:
period.remove('TBA')
period = list(period)
if len(period)==0:
return ''
if len(period)==1:
return period[0]
if len(period)==2 and 'Online' in period:
period.remove('Online')
return 'Hybrid at ' + period[0]
return '/'.join(period)
def course_time(course):
# is it morning, mid, or evening?
period = Set( [raw_course_time(course[0][7])], )
if len(course) > 1:
#time += ", " + course[1][7]
period.add(raw_course_time(course[1][7]))
if len(course) > 2:
#time += ", " + course[2][7]
period.add(raw_course_time(course[2][7]))
if len(course) > 3:
#time += ", " + course[3][7]
period.add(raw_course_time(course[3][7]))
if len(course) > 4:
#time += ", " + course[4][7]
period.add(raw_course_time(course[4][7]))
if len(course) > 5:
#time += ", " + course[5][7]
period.add(raw_course_time(course[5][7]))
#print raw_course_time(course[0][7]),
if 'TBA' in period:
period.remove('TBA')
period = list(period)
if len(period)==0:
return ''
if len(period)==1:
return period[0]
return '/'.join(period)
def course_teacher(course):
t = Set()
for c in course:
t.add(c[11])
return " / ".join(list(t))
def reg_nums():
courses = []
dates = []
sections = categorize()
today = todays_date_filename()
out = open(today+'.csv','w')
dates = {'loc':{}, 'time':{}, 'start':{}, 'teacher':{}}
i = 1
for f in os.listdir('.'):
m = re.search('reg_'+short_sem+'_(\d+)\.csv',f)
if m:
filein = open(f,'r').readlines()[1:]
d = m.group(1)
dates[d] = {}
for L in filein:
parts = L.split(',') # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
if not re.search('(\d+)',parts[0]): continue
if len(parts)<8: continue
if not parts[8]: continue
if float(parts[8])==0: continue
dates[d][parts[0] + " " + parts[1]] = (1.0* float(parts[9])) / float(parts[8])
if i == 1 and parts[0] in sections:
dates['loc'][parts[0] + " " + parts[1]] = course_location( sections[parts[0]] )
dates['time'][parts[0] + " " + parts[1]] = course_time(sections[parts[0]] )
dates['start'][parts[0] + " " + parts[1]] = course_start( sections[parts[0]] )
dates['teacher'][parts[0] + " " + parts[1]] = course_teacher( sections[parts[0]] )
#dates[d]['act'] = parts[9]
#dates[d]['nam'] = parts[5]
#dates[d]['onl'] = ''
#print parts
#if len(parts)>13 and parts[13]=='ONLINE': dates[d]['onl'] = 'online'
i += 1
"""for d in sorted(dates.keys()):
for c in d:
print d
print dates[d]['crs']"""
df = pd.DataFrame(dates)
df.to_csv(out)
# In the schedule, is this a class or a continuation of the class above?
def categorize():
# todo: must we open all these files?
dates = {}
files = sorted(os.listdir('.'))
files = list( filter( lambda x: re.search('reg(\d+)\.csv',x), files) )
files.reverse()
f = files[0]
filein = codecs.open(f,'r','utf-8').readlines()[1:]
sections = {}
this_section = []
for L in filein:
parts = L.strip().split(',') # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
parts = list( map( lambda x: clean_funny3(x), parts ) )
if not re.search('(\d+)',parts[0]): # This is a continuation
this_section.append(parts)
else: # this is a new section or the first line
if this_section:
sections[ this_section[0][0] ] = this_section
#print "Section: " + this_section[0][0] + " is: " + str(this_section) + "\n"
#print this_section[0][0] + "\t", course_start(this_section)
#print this_section[0][0] + "\t", course_time(this_section)
#print this_section[0][0] + "\t", course_location(this_section)
this_section = [ parts, ]
return sections
# Deprecated. call perl.
def constructSchedule():
term = raw_input("Name of html file? (ex: sp18.html) ")
os.chdir('make-web-sched')
cmd = 'perl make.pl ' + term
print "command: " + cmd
os.system(cmd)
"""
def fetch_dict(target,params={}):
# if there are more results, return the url for more fetching.
# else return false
#print target
global results_dict
r2 = requests.get(target, headers = header, params=params)
output = r2.text
if output.startswith('while('):
output = output[9:]
#print output
mycopy = results_dict.copy()
results_dict = {}
results_dict.update(json.loads(output))
results_dict.update(mycopy)
f.write(json.dumps(results_dict, indent=2))
#print "\n"
if ('link' in r2.headers):
links = r2.headers['link'].split(',')
for L in links:
ll = L.split(';')
link = ll[0].replace("<","")
link = link.replace(">","")
if re.search(r'next', ll[1]):
#print ll[1] + ":\t" + link
return link
return ""
"""
def get_schedule(term='201870', sem='fall'):
"""
sched_data = { 'term_in':term, 'sel_subj':'dummy', 'sel_day':'dummy',
'sel_schd':'dummy', 'sel_insm':'dummy', 'sel_camp':'dummy', 'sel_levl':'dummy', 'sel_sess':'dummy',
'sel_instr':'dummy', 'sel_ptrm':'dummy', 'sel_attr':'dummy', 'sel_subj':'%', 'sel_crse':'', 'sel_title':'',
'sel_schd':'%', 'sel_from_cred':'', 'sel_to_cred':'', 'sel_camp':'%', 'sel_ptrm':'%', 'sel_sess':'%',
'sel_attr':'%', 'begin_hh':'0', 'begin_mi':'0', 'begin_ap':'a', 'end_hh':'0', 'end_mi':'0', 'end_ap':'a' }
initial_headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, sdch, br',
'Accept-Language':'en-US,en;q=0.8',
'Connection':'keep-alive',
'Host':'ssb.gavilan.edu',
'Upgrade-Insecure-Requests':'1',
} #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
headers = { 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.8',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Content-Type':'application/x-www-form-urlencoded',
'Host':'ssb.gavilan.edu',
'Origin':'https://ssb.gavilan.edu',
'Referer':'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term='+term,
'Upgrade-Insecure-Requests':'1',
} #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
initial_url = 'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=' + term
sesh = requests.Session()
#r1 = sesh.get(initial_url,headers=initial_headers)
#sesh.headers.update(headers)
url = 'https://ssb.gavilan.edu/prod/bwckschd.p_get_crse_unsec'
r1 = sesh.get(initial_url)
r = sesh.post(url, data=sched_data)
print r.headers
data = r.text
out = open('data/temp/'+term+'.html','w')
out.write(data)
out.close()"""
os.system('perl parse_schedule.pl data/temp/' + term + '.html' + ' ' + sem)
#####
#####
##### conf.py ?
str="""355 985 1296
354 730 1295
353 319 1290
352 985 1289
351 813 1285
350 281 1285
349 267 1279
348 981 1252
347 994 1252
346 26 1250
345 757 1288
344 368 1288
343 1 1286
259 703 1295
256 693 1293
255 660 1292
254 1 1291
250 482 1287
246 2 1284
245 333 1283
244 27 1282
243 703 1281
242 730 1281
241 482 1280
239 211 1278
238 794 1278
237 2 1277
236 297 1276
235 831 1276
233 482 1251"""
for L in str.split("\n"):
(id,host,session) = L.split("\t")
qry = "INSERT INTO conf_signups (user,session,timestamp) VALUES (%s,%s,'2022-08-08 17:20:00');" % (host,session)
print(qry)