1289 lines
42 KiB
Python
1289 lines
42 KiB
Python
|
|
#get_schedule('201770')
|
|
|
|
|
|
# from pipelines - canvas data
|
|
|
|
"""
|
|
timestamp = nowAsStr()
|
|
|
|
requestParts = [ method,
|
|
host,
|
|
'', #content Type Header
|
|
'', #content MD5 Header
|
|
path,
|
|
'', #alpha-sorted Query Params
|
|
timestamp,
|
|
apiSecret ]
|
|
|
|
#Build the request
|
|
requestMessage = '\n'.join( requestParts )
|
|
requestMessage = requestMessage.encode('ASCII')
|
|
print((requestMessage.__repr__()))
|
|
hmacObject = hmac.new(bytearray(apiSecret,'ASCII'), bytearray('','ASCII'), hashlib.sha256) #
|
|
hmacObject.update(requestMessage)
|
|
hmac_digest = hmacObject.digest()
|
|
sig = base64.b64encode(hmac_digest)
|
|
headerDict = {
|
|
'Authorization' : 'HMACAuth ' + apiKey + ':' + str(sig),
|
|
'Date' : timestamp
|
|
}
|
|
|
|
|
|
"""
|
|
|
|
# Don't know
|
|
def demo():
|
|
resp = do_request('/api/account/self/file/sync')
|
|
mylog.write(json.dumps(resp, indent=4))
|
|
sample_table = resp['files'][10]
|
|
filename = sample_table['filename']
|
|
print(sample_table['table'])
|
|
|
|
response = requests.request(method='GET', url=sample_table['url'], stream=True)
|
|
if(response.status_code != 200):
|
|
print(('Request response went bad. Got back a ', response.status_code, ' code, meaning the request was ', response.reason))
|
|
else:
|
|
#Use the downloaded data
|
|
with open(local_data_folder + filename, 'wb') as fd:
|
|
for chunk in response.iter_content(chunk_size=128):
|
|
fd.write(chunk)
|
|
print("Success")
|
|
if filename.split('.')[-1] == 'gz':
|
|
plain_filename = 'canvas_data/' + ".".join(filename.split('.')[:-1])
|
|
pf = open(plain_filename,'w')
|
|
with gzip.open('canvas_data/' + filename , 'rb') as f:
|
|
pf.write(f.read())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# How to drop columns
|
|
#columns = ['Col1', 'Col2', ...]
|
|
#df.drop(columns, inplace=True, axis=1)
|
|
|
|
# left join, one on column, one on index
|
|
#merged = pd.merge(result,users,left_index=True,right_on='id', how='left')
|
|
|
|
|
|
"""
|
|
You can call set_index on the result of the dataframe:
|
|
|
|
In [2]:
|
|
data=[['Australia',100],['France',200],['Germany',300],['America',400]]
|
|
pd.DataFrame(data,columns=['Country','Volume']).set_index('Country')
|
|
|
|
Out[2]:
|
|
Volume
|
|
Country
|
|
Australia 100
|
|
France 200
|
|
Germany 300
|
|
America 400
|
|
"""
|
|
|
|
|
|
|
|
def stats():
|
|
# nothing seems to happen here?
|
|
|
|
#input = csv.DictReader(codecs.open(schedfile,'r','utf-8'))
|
|
input = csv.DictReader(open(schedfile,'r'))
|
|
out2 = open('temp2.csv','w')
|
|
clean = {}
|
|
for r in input:
|
|
if r['crn']: clean[ r['crn'] ] = r
|
|
|
|
for c,r in list(clean.items()):
|
|
try:
|
|
if int(r['cap'])==0: continue
|
|
else: prct = (1.0 * int( r['act'] )) / int(r['cap'])
|
|
if prct < 0.01: continue
|
|
o_str = ''
|
|
if r['location'].strip()=='ONLINE': o_str = 'online'
|
|
#print r['location']
|
|
date_parts = r['date'].split('-')
|
|
start = strptime(date_parts[0], '%m/%d')
|
|
if start > semester_begin: o_str += "\tlatestart " + date_parts[0]
|
|
out2.write( "".join([c, "\t", r['sub'], "\t", r['crs'], "\t", str(round(prct,2)), "% full\t", o_str, "\n"]) )
|
|
except:
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
######### from curriculum. py
|
|
|
|
|
|
# open('cache/programs/programs_1.txt','r').read()
|
|
|
|
"""
|
|
SEE serve.py .... i mean ... interactive.py
|
|
def dict_generator(indict, pre=None):
|
|
pre = pre[:] if pre else []
|
|
if isinstance(indict, dict):
|
|
for key, value in indict.items():
|
|
if isinstance(value, dict):
|
|
for d in dict_generator(value, pre + [key]):
|
|
yield d
|
|
elif isinstance(value, list) or isinstance(value, tuple):
|
|
for v in value:
|
|
for d in dict_generator(v, pre + [key]):
|
|
yield d
|
|
else:
|
|
yield str(pre) + " " + str([key, value]) + "\n"
|
|
else:
|
|
yield pre + [indict]
|
|
yield str(pre) + " " + str([indict]) + "\n"
|
|
|
|
|
|
|
|
def print_dict(v, prefix='',indent=''):
|
|
if isinstance(v, dict):
|
|
return [ print_dict(v2, "{}['{}']".format(prefix, k) + "<br />", indent+" " ) for k, v2 in v.items() ]
|
|
elif isinstance(v, list):
|
|
return [ print_dict( v2, "{}[{}]".format(prefix , i) + "<br />", indent+" ") for i, v2 in enumerate(v) ]
|
|
else:
|
|
return '{} = {}'.format(prefix, repr(v)) + "\n"
|
|
|
|
|
|
def walk_file():
|
|
j = json.loads(open('cache/programs/programs_2.txt','r').read())
|
|
|
|
return print_dict(j)
|
|
|
|
from flask import Flask
|
|
from flask import request
|
|
|
|
def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
|
|
|
|
def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
|
|
|
|
def a(t,h): return '<a href="%s">%s</a>' % (h,t)
|
|
|
|
def server_save(key,value):
|
|
codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
|
|
|
|
def flask_thread(q):
|
|
app = Flask(__name__)
|
|
|
|
@app.route("/")
|
|
def home():
|
|
return tag('h1','This is my server.') + "<br />" + a('want to shut down?','/sd')
|
|
|
|
@app.route("/save/<key>/<val>")
|
|
def s(key,val):
|
|
server_save(key,val)
|
|
return tag('h1','Saved.') + "<br />" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
|
|
|
|
@app.route("/crazy")
|
|
def hello():
|
|
r = '<link rel="stylesheet" href="static/bootstrap.min.css">'
|
|
r += tag('style', 'textarea { white-space:nowrap; }')
|
|
r += tag('body', \
|
|
tagc('div','container-fluid', \
|
|
tagc('div','row', \
|
|
tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
|
|
tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
|
|
|
|
|
|
|
|
return r
|
|
|
|
@app.route("/sd")
|
|
def sd():
|
|
print('SIGINT or CTRL-C detected. Exiting gracefully')
|
|
func = request.environ.get('werkzeug.server.shutdown')
|
|
if func is None:
|
|
raise RuntimeError('Not running with the Werkzeug Server')
|
|
func()
|
|
return "Server has shut down."
|
|
app.run()
|
|
|
|
|
|
from queue import Queue
|
|
|
|
q = Queue()
|
|
|
|
def serve():
|
|
import webbrowser
|
|
import threading
|
|
x = threading.Thread(target=flask_thread, args=(q,))
|
|
x.start()
|
|
webbrowser.open_new_tab("http://localhost:5000")
|
|
|
|
|
|
|
|
|
|
#s = open('cache/programs/index.json','w')
|
|
#s.write( json.dumps({'departments':sorted(list(dept_index)), 'programs':prog_index}, indent=2) )
|
|
#s.close()
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
# Prompt for course id, return list of user dicts. TODO this duplicates courses.py ??
|
|
def getUsersInCourse(id=0): # returns list
|
|
if not id:
|
|
id = str(input("The Course ID? "))
|
|
id = str(id)
|
|
return fetch('/api/v1/courses/%s/users' % id, 0)
|
|
|
|
|
|
|
|
|
|
#### curriculum.py
|
|
|
|
|
|
def recur_look_for_leafs(item,indent=0,show=1):
|
|
global leafcount, displaynames
|
|
ii = indent * " "
|
|
is_leaf = am_i_a_leaf(item)
|
|
if type(item) == type({}):
|
|
status = ""
|
|
if show:
|
|
status = "Dict"
|
|
if is_leaf:
|
|
leafcount += 1
|
|
status = "Leaf Dict"
|
|
if status:
|
|
print("\n%s%s" % (ii,status))
|
|
indent += 1
|
|
ii = indent * " "
|
|
for K,V in list(item.items()):
|
|
if show or is_leaf:
|
|
print("%s%s:" % (ii, K), end="")
|
|
if K =='displayName': displaynames.append(V)
|
|
recur_look_for_leafs(V,indent+1,show or is_leaf)
|
|
|
|
elif type(item) == type([]):
|
|
status = ""
|
|
if show: status = "List (" + str( len(item) ) + ")"
|
|
if is_leaf: status = "Leaf List (" + str( len(item) ) + ")"
|
|
if status:
|
|
print("\n%s%s" % (ii,status))
|
|
indent += 1
|
|
ii = indent * " "
|
|
for V in item:
|
|
recur_look_for_leafs(V,indent+1, show or is_leaf)
|
|
|
|
elif type(item) == type("abc"):
|
|
if show: print("%s%s" % (' ', item))
|
|
elif type(item) == type(55):
|
|
if show: print("%s%i" % (' ', item))
|
|
elif type(item) == type(5.5):
|
|
if show: print("%s%f" % (' ', item))
|
|
elif type(item) == type(False):
|
|
if show: print("%s%s" % (' ', str(item)))
|
|
|
|
|
|
def am_i_a_leaf(item):
|
|
if type(item) == type({}):
|
|
for K,V in list(item.items()):
|
|
if type(V) == type({}) or type(V) == type([]):
|
|
return False
|
|
|
|
elif type(item) == type([]):
|
|
for V in item:
|
|
if type(V) == type({}) or type(V) == type([]):
|
|
return False
|
|
|
|
elif type(item) == type("abc"): return True
|
|
elif type(item) == type(55): return True
|
|
elif type(item) == type(5.5): return True
|
|
elif type(item) == type(False):
|
|
if item == False: return True
|
|
elif item == True: return True
|
|
return True
|
|
|
|
def sampleclass():
|
|
theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
|
|
#print(json.dumps(theclass,indent=2))
|
|
recur_look_for_leafs(theclass)
|
|
print(leafcount)
|
|
print(sorted(displaynames))
|
|
|
|
|
|
|
|
def recur_matcher(item, depth=0):
|
|
indent = depth * " "
|
|
my_result_lines = []
|
|
if type(item) == type({}):
|
|
if not match( item,
|
|
{'entityMetadata': {'entityTitle': _,'status': _, 'entityType':_, 'entityId':_ }},
|
|
lambda title,status,typ,id:
|
|
my_result_lines.append("%s%s: %s (id %s) status: %s" % (indent, str(typ), str(title), str(id), str(status))) ,
|
|
{'attributes': {'displayName': _}, 'lookUpDisplay': _, },
|
|
lambda x,y: my_result_lines.append("%s%s: %s" % (indent, clean(str(x)), clean(str(y)))) ,
|
|
{'attributes': {'displayName': _}, 'fieldValue': _, },
|
|
lambda x,y: my_result_lines.append("%s%s: %s" % (indent, clean(str(x)), clean(str(y)))) ,
|
|
{'sectionName': _},
|
|
lambda x: my_result_lines.append("%sSection: %s" % (indent, str(x))) ,
|
|
_, nothing
|
|
):
|
|
for K,V in list(item.items()):
|
|
my_result_lines.extend(recur_matcher(V,depth+1))
|
|
elif type(item) == type([]):
|
|
for V in item:
|
|
my_result_lines.extend(recur_matcher(V,depth+1))
|
|
return my_result_lines
|
|
|
|
|
|
|
|
|
|
|
|
def matchstyle():
|
|
theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
|
|
print("\n".join(recur_matcher(theclass)))
|
|
|
|
|
|
# 7: ['pattern matcher style', matchstyle],
|
|
# 8: ['pattern matcher - test on all classes', match_style_test],
|
|
|
|
|
|
|
|
##### from localcache
|
|
|
|
stem_course_id = '11015' # TODO
|
|
|
|
# NO LONGER USED - SEE COURSES
|
|
def enroll_stem_students():
|
|
depts = "MATH BIO CHEM PHYS ASTR GEOG".split(" ")
|
|
students = set()
|
|
for d in depts:
|
|
students.update(dept_classes(d))
|
|
print(students)
|
|
|
|
to_enroll = [ x for x in students if x not in already_enrolled ]
|
|
|
|
print(to_enroll)
|
|
print("prev line is people to enroll\nnext line is students already enrolled in stem")
|
|
print(already_enrolled)
|
|
|
|
for s in to_enroll:
|
|
t = url + '/api/v1/courses/%s/enrollments' % stem_course_id
|
|
data = { 'enrollment[user_id]': s[1], 'enrollment[type]':'StudentEnrollment',
|
|
'enrollment[enrollment_state]': 'active' }
|
|
print(data)
|
|
print(t)
|
|
if input('enter to enroll %s or q to quit: ' % s[0]) == 'q':
|
|
break
|
|
r3 = requests.post(t, headers=header, params=data)
|
|
print(r3.text)
|
|
|
|
|
|
#####
|
|
##### from users.py pretty much just use sql now
|
|
|
|
|
|
# unused?
|
|
def getAllTeachersInTerm(): # a list
|
|
# classes taught in last 3 semesters
|
|
# How many of them were published and used
|
|
# hits in last week/month/year
|
|
# most common department
|
|
# email addr
|
|
all_courses = {}
|
|
teachers = {} # keyed by goo
|
|
# { 'name':'', 'id':'', 'email':'', 'goo':'', 'classes':[ (#name,#id,#pubd,#hitsbyteacher) ... ] }
|
|
|
|
# This is a bit different from the 1 year schedule above, because it looks at
|
|
# people who were active in their shells in iLearn.
|
|
|
|
outfile = codecs.open('teacherdata/historical_shells_used.json','w', encoding='utf-8')
|
|
for term in last_4_semesters_ids: # [60,]:
|
|
print(("Fetching term: " + str(term)))
|
|
all_courses[term] = \
|
|
fetch('/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100')
|
|
i = 0
|
|
j = 0
|
|
for k,v in list(all_courses.items()): ##### term k, list v
|
|
for a_class in v:
|
|
print((a_class['name']))
|
|
published = 0
|
|
if a_class['workflow_state'] in ['available','completed']:
|
|
j += 1
|
|
published = 1
|
|
i += 1
|
|
#if i > 20: break
|
|
tch = fetch('/api/v1/courses/' + str(a_class['id']) + '/search_users?enrollment_type=teacher')
|
|
for r in tch: ##### TEACHER r of COURSE a_class
|
|
name = str(r['sortable_name'])
|
|
if not 'sis_import_id' in r:
|
|
print("This user wasn't available: " + name)
|
|
continue
|
|
goo = str(r['sis_import_id'])
|
|
print((r['sortable_name']))
|
|
if not name in teachers:
|
|
email = getEmail(r['id'])
|
|
teachers[name] = { 'name':r['sortable_name'], 'id':r['id'], 'email':email, 'goo':goo, 'classes':[] }
|
|
info = (a_class['name'],a_class['id'],published)
|
|
teachers[name]['classes'].append( info )
|
|
|
|
## TODO: hits in courses by teachers https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
|
|
|
|
for t,v in list(teachers.items()):
|
|
teachers[t]['num_courses'] = len(v['classes'])
|
|
teachers[t]['num_active_courses'] = sum( [x[2] for x in v['classes']] )
|
|
depts = [ dept_from_name(x[0]) for x in v['classes'] ]
|
|
teachers[t]['dept'] = most_common_item(depts)
|
|
|
|
#print(str(j), "/", str(i), " sections are published")
|
|
outfile.write(json.dumps(teachers))
|
|
|
|
|
|
"""
|
|
def teacherActivityLog(uid=1): ### Next: save results in a hash and return that....
|
|
global results, users, users_by_id
|
|
#get_users() # do this if you think 'teachers/users.json' is outdated.
|
|
|
|
load_users()
|
|
|
|
#for x in users_by_id.keys():
|
|
# if x < 20:
|
|
# print x
|
|
# print users_by_id[x]
|
|
|
|
|
|
teachers = csv.reader(open('teachers/current_semester.txt','r'), delimiter="\t")
|
|
for row in teachers:
|
|
print(row[0] + " is id: " + row[1])
|
|
uid = row[1]
|
|
print("Comes up as: " + str(users_by_id[int(uid)]))
|
|
info = users_by_id[int(uid)]
|
|
goo = info['login_id']
|
|
|
|
output_file = open('logs/users/byweek/'+ goo.lower() + '.csv', 'w')
|
|
|
|
|
|
# okay, actually, the first week here is the week before school IRL
|
|
start = isoweek.Week.withdate( datetime.date(2017,8,21))
|
|
end = isoweek.Week.thisweek()
|
|
byweek = []
|
|
|
|
i = 0
|
|
while(1):
|
|
results = []
|
|
start = start + 1
|
|
if start > end: break
|
|
|
|
myStart = start.day(0).isoformat() + 'T00:00-0700'
|
|
myEnd = start.day(6).isoformat() + 'T11:59:59-0700'
|
|
t = url + "/api/v1/users/" + str(uid) + "/page_views?start_time=" + myStart + '&end_time=' + myEnd + "&perpage=500"
|
|
print(t)
|
|
while(t):
|
|
print(".", end=' ')
|
|
t = fetch(t)
|
|
print("")
|
|
thisWeek = len(results)
|
|
print("Week # " + str(i) + "\t" + str(thisWeek))
|
|
byweek.append( "Week # " + str(i) + "\t" + str(thisWeek) )
|
|
output_file.write( start.isoformat() + "," + str(thisWeek) + "\n")
|
|
i += 1
|
|
for j in byweek:
|
|
print(j)
|
|
"""
|
|
|
|
"""
|
|
def summarize_student_teacher_role(u):
|
|
# u is a "group" from the groupby fxn
|
|
# term is sp18 now
|
|
t = 0
|
|
s = 0
|
|
for a in u:
|
|
if a=='TeacherEnrollment': t += 1
|
|
else: s += 1
|
|
if NUM_ONLY:
|
|
if t > s: return 'teacher'
|
|
return 'student'
|
|
else:
|
|
if t > s: return '1'
|
|
return '0'
|
|
"""
|
|
"""
|
|
def user_roles2():
|
|
# cross list users, classes enrolled, and their roles
|
|
global role_table, term_courses
|
|
|
|
role_table = enrollment_file()
|
|
user_table = users_file()
|
|
course_table = courses_file() # from canvas
|
|
term_table = term_file()
|
|
schedule = current_schedule() # from banner
|
|
|
|
# current semester
|
|
current = term_table[lambda d: d.course_section=='2018 Spring']
|
|
term_id = current['id'].values[0]
|
|
term_courses = course_table[lambda d: d.termid==term_id] # courses this semester
|
|
|
|
# add is_online flag (for courses listed in schedule as online-only)
|
|
term_courses['is_online'] = term_courses['code'].map( lambda x: course_is_online( get_crn_from_name(x) ) )
|
|
|
|
new_df = pd.DataFrame(columns=['type','oo','num'])
|
|
|
|
m = 0
|
|
data = []
|
|
for u in user_table.iterrows():
|
|
if m % 1000 == 0: print("on row " + str(m))
|
|
m += 1
|
|
data.append(categorize_user(u))
|
|
#if m > 1500: break
|
|
new_df = pd.DataFrame(data,columns=['i','type','onlineonly','numcls']).set_index('i')
|
|
print(new_df)
|
|
|
|
user_table = user_table.merge(new_df,left_index=True,right_index=True)
|
|
user_table.to_csv('canvas_data/users_online.csv')
|
|
"""
|
|
|
|
### IS THIS IN CANVAS_DATA.py?
|
|
|
|
|
|
|
|
|
|
""" Collate the raw logs into something more compact and useful. Version 1:
|
|
- # of accesses, user/day
|
|
- # of participations, user/day
|
|
-
|
|
|
|
- where day is the number of days into the semester. Classes shorter than 16 weeks should get a multiplier
|
|
-
|
|
|
|
- 2 initial goals:
|
|
a. data for statistics / clustering / regression / learning
|
|
b. data for visualization
|
|
"""
|
|
def req_to_db(fname_list):
|
|
fields = ','.join("id timestamp timestamp_year timestamp_month timestamp_day user_id course_id root_account_id course_account_id quiz_id discussion_id conversation_id assignment_id url user_agent http_method remote_ip interaction_micros web_application_controller web_applicaiton_action web_application_context_type web_application_context_id real_user_id session_id user_agent_id http_status http_version".split(" "))
|
|
sqlite_file = 'canvas_data/data.db'
|
|
conn = sqlite3.connect(sqlite_file)
|
|
c = conn.cursor()
|
|
# merge all requests into db
|
|
by_date_course = defaultdict( lambda: defaultdict(int) )
|
|
by_date_user = defaultdict( lambda: defaultdict(int) )
|
|
df_list = []
|
|
df_list_crs = []
|
|
users = defaultdict( lambda: defaultdict(int) )
|
|
i = 0
|
|
limit = 300
|
|
for fname in fname_list:
|
|
print((fname+"\n"))
|
|
for line in gzip.open('canvas_data/'+fname,'r'):
|
|
r = line.split('\t')
|
|
#tot = len(fields.split(','))
|
|
#i = 0
|
|
#for x in fields.split(','):
|
|
# print x + "\t" + r[i]
|
|
# i+= 1
|
|
|
|
qry = "insert into requests("+fields+") values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
conn.execute(qry, r)
|
|
|
|
|
|
# New method for below:
|
|
# read collated data from sqlite
|
|
# collate from more logs
|
|
# write back....?
|
|
|
|
"""
|
|
date = datetime.datetime.strptime( r['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
|
|
if r['userid'] in users:
|
|
users[r['userid']]['freq'] += 1
|
|
if users[r['userid']]['lastseen'] < date:
|
|
users[r['userid']]['lastseen'] = date
|
|
else:
|
|
users[r['userid']] = {"id":r['userid'], "lastseen":date, "freq":1}
|
|
by_date_course[ r['day'] ][ r['courseid'] ] += 1
|
|
by_date_user[ r['day'] ][ r['userid'] ] += 1
|
|
#if r['userid'] in by_user: by_user[r['userid']] += 1
|
|
#else: by_user[r['userid']] = 1
|
|
#if r['courseid'] in by_course: by_course[r['courseid']] += 1
|
|
#else: by_course[r['courseid']] = 1
|
|
#mylog.write("by_user = " + str(by_user))
|
|
df_list.append(pd.DataFrame(data=by_date_user))
|
|
df_list_crs.append(pd.DataFrame(data=by_date_course))
|
|
"""
|
|
i += 1
|
|
if i > limit: break
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
|
|
|
|
"""
|
|
Making columns:
|
|
table_data = [['a', 'b', 'c'], ['aaaaaaaaaa', 'b', 'c'], ['a', 'bbbbbbbbbb', 'c']]
|
|
for row in table_data:
|
|
print("{: >20} {: >20} {: >20}".format(*row))
|
|
|
|
Transpose a matrix:
|
|
rez = [[m[j][i] for j in range(len(m))] for i in range(len(m[0]))]
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
"""
|
|
ilearn_by_id = {}
|
|
ilearn_by_name = {}
|
|
for x in ilearn_list:
|
|
ilearn_by_id[x[3]] = x
|
|
ilearn_by_name[x[0]] = x
|
|
|
|
for ml in open('cache/teacher_manual_name_lookup.csv','r').readlines():
|
|
parts = ml.strip().split(',')
|
|
try:
|
|
manual_list[parts[0]] = ilearn_by_id[parts[1]]
|
|
except Exception as e:
|
|
print "Teacher missing: " + parts[0]
|
|
|
|
il_names = [ x[0] for x in ilearn_list ]
|
|
il_byname = {}
|
|
for x in ilearn_list: il_byname[x[0]] = x
|
|
sched_list_missed = [x for x in sched_list]
|
|
|
|
#
|
|
# key is long name (with middle name) from schedule, value is tuple with everything
|
|
name_lookup = manual_list
|
|
matches = []
|
|
|
|
#print ilearn_list
|
|
|
|
num_in_sched = len(sched_list)
|
|
num_in_ilearn = len(ilearn_list)
|
|
|
|
#for i in range(min(num_in_sched,num_in_ilearn)):
|
|
# print "|"+sched_list[i] + "|\t\t|" + ilearn_list[i][0] + "|"
|
|
|
|
print("Sched names: %i, iLearn names: %i" % (num_in_sched,num_in_ilearn))
|
|
|
|
for s in sched_list:
|
|
for t in il_names:
|
|
if first_last(s) == t:
|
|
#print ' MATCHED ' + s + ' to ' + t
|
|
sched_list_missed.remove(s)
|
|
try:
|
|
name_lookup[s] = ilearn_by_name[ first_last(s) ]
|
|
except Exception as e:
|
|
print "Teacher missing (2): " + s
|
|
il_names.remove(first_last(s))
|
|
matches.append(s)
|
|
|
|
|
|
print "Matched: " + str(matches)
|
|
|
|
print "\nDidn't match: " + str(len(sched_list_missed)) + " schedule names."
|
|
|
|
print "\nFinal results: "
|
|
print name_lookup
|
|
|
|
nlf = codecs.open('cache/sched_to_ilearn_names.json','w','utf-8')
|
|
nlf.write(json.dumps(name_lookup,indent=2))
|
|
# STRING DISTANCE
|
|
#sim = find_most_similar(s,i_names)
|
|
#print ' CLOSEST MATCHES to ' + s + ' are: ' + str(sim)
|
|
#mm.write(s+',\n')
|
|
"""
|
|
|
|
|
|
#ilearn_list = sorted(list(set(map(
|
|
# lambda x: #(tfi[x]['name'],tfi[x]['email'],tfi[x]['dept'],str(tfi[x]['id']),tfi[x]['goo']),
|
|
# tfi.keys()))))
|
|
#i_names = [ x[0] for x in ilearn_list ]
|
|
|
|
#print json.dumps(i_names,indent=2)
|
|
#return
|
|
|
|
|
|
|
|
# how to filter a dict based on values
|
|
# filtered = {k: v for k, v in course_combos.items() if v['dept'] == 'LIB' or v['dept'] == 'CSIS' }
|
|
|
|
# more pandas
|
|
# gapminder['continent'].unique()
|
|
|
|
|
|
|
|
|
|
|
|
#for name,group in bycode:
|
|
# #print name
|
|
# print name, " ", group['type']
|
|
|
|
#onl = gg.agg( lambda x: has_online(x) )
|
|
#ttl = gg.agg( lambda x: len(x) )
|
|
#ttl = ttl.rename(columns={'type':'total_sections'})
|
|
|
|
#onl.join(gg.agg( lambda x: has_hybrid(x) ),how='outer')
|
|
#onl.join(gg.agg( lambda x: has_lecture(x) ), how='outer')
|
|
|
|
#onl['num_sections'] = 0
|
|
#onl['num_lec'] = 0
|
|
#onl['num_online'] = 0
|
|
|
|
#all = pd.merge([onl,hyb,lec])
|
|
#print onl
|
|
#total=len, f2f=lambda x: ) set(x)
|
|
#{ 'num_sections': "count",
|
|
# 'num_lec': lambda x: 5,
|
|
# 'num_online': lambda x: 5 } )
|
|
#print gg
|
|
"""
|
|
def has_online(series):
|
|
# if any items of the series have the string 'online', return 1
|
|
for i in series:
|
|
if i == 'online': return 1
|
|
return 0
|
|
def has_lecture(series):
|
|
# if any items of the series have the string 'online', return 1
|
|
for i in series:
|
|
if i == 'online': return 1
|
|
return 0
|
|
def has_hybrid(series):
|
|
# if any items of the series have the string 'online', return 1
|
|
for i in series:
|
|
if i == 'hybrid': return 1
|
|
return 0
|
|
"""
|
|
#### RIGHT HERE IS WHERE I THINK... MAYBE THIS ISN'T THE RIGHT APPROACH. I DON'T SEEM
|
|
#### TO BE ABLE TO QUERY THE FACT BASE. IS THAT TRUE? SHOULD I JUST BE USING TABLES?
|
|
|
|
#### CHANGING COURSE... USE THE RULES TO UPDATE A DATABASE/TABLE/DATAFRAME
|
|
#### OR SET OF DICTS.
|
|
|
|
# ultimately i want this to be more flexible, so i can categorize degrees as 'available evening' etc
|
|
#
|
|
|
|
|
|
# Simple data structure. In this function, a degree is
|
|
""" degree = { 'name': 'History AA',
|
|
'blocks': [ { 'original_title':'xxx', 'rulecode':'u3',
|
|
'courses': [ {'code':'math1a', 'units': '3.0', 'wasonline':False },
|
|
{'code':'math2a', 'units': '3.0', 'wasonline':False },
|
|
{'code':'math3a', 'units': '3.0', 'wasonline':False } ] },
|
|
{ 'original_title':'xyz', 'rulecode':'a',
|
|
'courses': [ {'code':'math5a', 'units': '3.0', 'wasonline':False },
|
|
{'code':'math6a', 'units': '3.0', 'wasonline':False },
|
|
{'code':'math7a', 'units': '3.0', 'wasonline':False } ] } ] }
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Wrapper to get 2 schedules at once
|
|
def dl_sched():
|
|
global SEMESTER, semester_begin, filename, short_sem
|
|
SEMESTER = 'Fall 2019'
|
|
short_sem = 'fa19'
|
|
semester_begin = strptime('08/26', '%m/%d')
|
|
filename = 'fa19_sched.json'
|
|
|
|
txt = login()
|
|
codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
|
|
#stats()
|
|
#reg_nums()
|
|
|
|
#todo: these semesters
|
|
SEMESTER = 'Summer 2019'
|
|
short_sem = 'su19'
|
|
semester_begin = strptime('06/17', '%m/%d')
|
|
filename = 'su19_sched.json'
|
|
|
|
txt = login()
|
|
codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
|
|
#stats()
|
|
#reg_nums()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Send a personalized email regarding ZTC
|
|
def send_z_email(fullname, firstname, addr, courses_list):
|
|
FULLNAME = fullname #"Sabrina Lawrence"
|
|
FNAME = firstname # "Sabrina"
|
|
to_email = addr # "slawrence@gavilan.edu"
|
|
courses = courses_list # ["CSIS45", "CSIS85"]
|
|
|
|
course_template = "<a href='%s'>%s</a> "
|
|
url_template = "https://docs.google.com/forms/d/e/1FAIpQLSfZLQp6wHFEdqsmpZ7jz2Y8HtKLo8XTAhrE2fyvTDOEgquBDQ/viewform?usp=pp_url&entry.783353363=%s&entry.1130271051=%s" # % (FULLNAME, COURSE1)
|
|
|
|
bare_link = "https://forms.gle/pwZJHdWSkyvmH4L19"
|
|
|
|
COURSELINKS = ''
|
|
PLAINCOURSES = ''
|
|
for C in courses:
|
|
ut = url_template % (FULLNAME, C)
|
|
COURSELINKS += course_template % (ut, C)
|
|
PLAINCOURSES += C + " "
|
|
|
|
text_version = open('cache/ztc_mail1.txt','r').read()
|
|
html_version = open('cache/ztc_mail1_h.txt','r').read()
|
|
|
|
# replace these: $FNAME $COURSELINKS $LINK
|
|
|
|
email = re.sub( r'\$FNAME', FNAME, text_version )
|
|
email = re.sub( r'\$COURSELINKS', PLAINCOURSES, email )
|
|
email = re.sub( r'\$LINK', bare_link, email )
|
|
|
|
email_h = re.sub( r'\$FNAME', FNAME, html_version )
|
|
email_h = re.sub( r'\$COURSELINKS', COURSELINKS, email_h )
|
|
|
|
print(email_h+"\n\n"+email)
|
|
|
|
from O365 import Account
|
|
|
|
credentials = ('phowell@gavilan.edu', 'xxx')
|
|
client_secret = 'xxx' # expires 10/28/2020
|
|
tenant_id = "4ad609c3-9156-4b89-9496-0c0600aeb0bb"
|
|
# application client id: 29859402-fa55-4646-b717-752d90c61cde
|
|
|
|
account = Account(credentials, auth_flow_type='credentials', tenant_id=tenant_id)
|
|
if account.authenticate():
|
|
print('Authenticated!')
|
|
|
|
#account = Account(credentials)
|
|
#if account.authenticate(scopes=['message_all']):
|
|
# print('Authenticated!')
|
|
m = account.new_message()
|
|
m.to.add(addr)
|
|
m.subject = 'Quick question about your course textbook'
|
|
m.body = "email_h"
|
|
m.send()
|
|
|
|
"""
|
|
import smtplib
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
|
|
msg = MIMEMultipart('alternative')
|
|
msg['Subject'] = "Quick question about your course textbook"
|
|
msg['From'] = "gavdisted@gmail.com"
|
|
msg['To'] = to_email
|
|
|
|
msg.attach(MIMEText(email, 'plain'))
|
|
msg.attach(MIMEText(email_h, 'html'))
|
|
|
|
|
|
#s = smtplib.SMTP('smtp.gmail.com', 587)
|
|
#s.starttls()
|
|
#s.login("gavdisted", "xxx")
|
|
|
|
|
|
s = smtplib.SMTP_SSL('smtp.office365.com',587)
|
|
s.ehlo()
|
|
s.starttls()
|
|
s.login('phowell@gavilan.edu', 'xxx')
|
|
|
|
#s.sendmail(msg['From'], msg['To'], msg.as_string())
|
|
s.sendmail(msg['From'], msg['To'], "Testing")
|
|
s.quit()"""
|
|
|
|
|
|
|
|
def getInactiveTeachersInTerm(t=23): # a list
|
|
global results
|
|
teachers = {}
|
|
emails = {}
|
|
outfile = codecs.open('canvas/inactive_teachers.txt','w', encoding='utf-8')
|
|
efile = codecs.open('canvas/inactive_teachers_emails.txt','w', encoding='utf-8')
|
|
|
|
#yn = raw_input('All courses? y=all n=only active ')
|
|
#all = 0
|
|
#if yn=='y': all = 1
|
|
|
|
if not t:
|
|
t = askForTerms()
|
|
else: t = [ t, ]
|
|
for term in t:
|
|
r = url + '/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100'
|
|
while(r): r = fetch(r)
|
|
all_courses = results #json.loads(results)
|
|
#print "All unpublished courses: "
|
|
i = 0
|
|
j = 0
|
|
for k in all_courses:
|
|
j += 1
|
|
if k['workflow_state'] != 'available':
|
|
i += 1
|
|
print(str(i), "\t", k['name'], "\t", k['workflow_state'])
|
|
results = []
|
|
t2 = url + '/api/v1/courses/' + str(k['id']) + '/search_users?enrollment_type=teacher'
|
|
|
|
|
|
while(t2): t2 = fetch(t2)
|
|
#print results
|
|
for r in results:
|
|
key = r['sortable_name'] + "\t" + str(r['id'])
|
|
#if not 'email' in r: pdb.set_trace()
|
|
emails[key] = str(r['sis_user_id'])
|
|
#print r
|
|
if key in teachers:
|
|
teachers[key].append(k['name'])
|
|
else:
|
|
teachers[key] = [ k['name'], ]
|
|
#print json.dumps(results, indent=4, sort_keys=True)
|
|
#a = raw_input()
|
|
|
|
print(str(i), "/", str(j), " sections are unpublished")
|
|
for t in list(emails.keys()):
|
|
efile.write(emails[t] + ", ")
|
|
for t in list(teachers.keys()):
|
|
outfile.write(t + "\t")
|
|
for c in teachers[t]:
|
|
outfile.write(c + ",")
|
|
outfile.write("\n")
|
|
#f.write(json.dumps(teachers, indent=4, sort_keys=True))
|
|
print("Output file is in ./teachers/current_semester.txt")
|
|
#print json.dumps(all_courses, indent=4, sort_keys=True)
|
|
"""for x in all_courses:
|
|
qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
|
|
t = url + qry
|
|
while(t): t = fetch(t)
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
#for t,v in teachers.items():
|
|
# outfile.write( "|".join( [ v['goo'], v['name'], v['email'], v['dept'], str(v['num_courses']), str(v['num_active_courses']) ] ) + "\n" )
|
|
|
|
#{"goo": "G00275722", "name": "Agaliotis, Paul", "num_courses": 1, "num_active_courses": 1, "id": 5092, "dept": "AMT", "classes": [["AMT120 POWERPLANT TECH FA18 10958", 5322, 1]], "email": "PAgaliotis@gavilan.edu"},
|
|
|
|
#for t in teachers.keys():
|
|
# outfile.write(t + "\t")
|
|
# for c in teachers[t]:
|
|
# outfile.write(c + ",")
|
|
# outfile.write("\n")
|
|
#f.write(json.dumps(teachers, indent=4, sort_keys=True))
|
|
#print "Output file is in ./teachers/current_semester.txt"
|
|
#print json.dumps(all_courses, indent=4, sort_keys=True)
|
|
"""for x in all_courses:
|
|
qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
|
|
t = url + qry
|
|
while(t): t = fetch(t)
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
def course_location(course):
|
|
if len(course[0]) > 13:
|
|
period = Set( [course_location_raw(course[0][13])], )
|
|
else:
|
|
period = Set()
|
|
|
|
if len(course) > 1:
|
|
period.add(course_location_raw(course[1][13]))
|
|
|
|
if len(course) > 2:
|
|
period.add(course_location_raw(course[2][13]))
|
|
|
|
if len(course) > 3:
|
|
period.add(course_location_raw(course[3][13]))
|
|
|
|
if len(course) > 4:
|
|
period.add(course_location_raw(course[4][13]))
|
|
|
|
if len(course) > 5:
|
|
period.add(course_location_raw(course[5][13]))
|
|
|
|
|
|
if 'TBA' in period:
|
|
period.remove('TBA')
|
|
|
|
period = list(period)
|
|
|
|
if len(period)==0:
|
|
return ''
|
|
|
|
if len(period)==1:
|
|
return period[0]
|
|
|
|
if len(period)==2 and 'Online' in period:
|
|
period.remove('Online')
|
|
return 'Hybrid at ' + period[0]
|
|
return '/'.join(period)
|
|
|
|
|
|
def course_time(course):
|
|
# is it morning, mid, or evening?
|
|
|
|
period = Set( [raw_course_time(course[0][7])], )
|
|
|
|
if len(course) > 1:
|
|
#time += ", " + course[1][7]
|
|
period.add(raw_course_time(course[1][7]))
|
|
|
|
if len(course) > 2:
|
|
#time += ", " + course[2][7]
|
|
period.add(raw_course_time(course[2][7]))
|
|
|
|
if len(course) > 3:
|
|
#time += ", " + course[3][7]
|
|
period.add(raw_course_time(course[3][7]))
|
|
|
|
if len(course) > 4:
|
|
#time += ", " + course[4][7]
|
|
period.add(raw_course_time(course[4][7]))
|
|
|
|
if len(course) > 5:
|
|
#time += ", " + course[5][7]
|
|
period.add(raw_course_time(course[5][7]))
|
|
|
|
#print raw_course_time(course[0][7]),
|
|
|
|
if 'TBA' in period:
|
|
period.remove('TBA')
|
|
|
|
period = list(period)
|
|
|
|
if len(period)==0:
|
|
return ''
|
|
|
|
if len(period)==1:
|
|
return period[0]
|
|
|
|
return '/'.join(period)
|
|
|
|
|
|
|
|
def course_teacher(course):
|
|
t = Set()
|
|
for c in course:
|
|
t.add(c[11])
|
|
return " / ".join(list(t))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def reg_nums():
|
|
courses = []
|
|
dates = []
|
|
sections = categorize()
|
|
|
|
today = todays_date_filename()
|
|
|
|
out = open(today+'.csv','w')
|
|
dates = {'loc':{}, 'time':{}, 'start':{}, 'teacher':{}}
|
|
i = 1
|
|
for f in os.listdir('.'):
|
|
m = re.search('reg_'+short_sem+'_(\d+)\.csv',f)
|
|
if m:
|
|
filein = open(f,'r').readlines()[1:]
|
|
d = m.group(1)
|
|
dates[d] = {}
|
|
for L in filein:
|
|
parts = L.split(',') # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
|
|
if not re.search('(\d+)',parts[0]): continue
|
|
if len(parts)<8: continue
|
|
if not parts[8]: continue
|
|
if float(parts[8])==0: continue
|
|
|
|
dates[d][parts[0] + " " + parts[1]] = (1.0* float(parts[9])) / float(parts[8])
|
|
|
|
if i == 1 and parts[0] in sections:
|
|
dates['loc'][parts[0] + " " + parts[1]] = course_location( sections[parts[0]] )
|
|
dates['time'][parts[0] + " " + parts[1]] = course_time(sections[parts[0]] )
|
|
dates['start'][parts[0] + " " + parts[1]] = course_start( sections[parts[0]] )
|
|
dates['teacher'][parts[0] + " " + parts[1]] = course_teacher( sections[parts[0]] )
|
|
|
|
#dates[d]['act'] = parts[9]
|
|
#dates[d]['nam'] = parts[5]
|
|
#dates[d]['onl'] = ''
|
|
#print parts
|
|
#if len(parts)>13 and parts[13]=='ONLINE': dates[d]['onl'] = 'online'
|
|
i += 1
|
|
"""for d in sorted(dates.keys()):
|
|
for c in d:
|
|
print d
|
|
print dates[d]['crs']"""
|
|
|
|
df = pd.DataFrame(dates)
|
|
df.to_csv(out)
|
|
|
|
# In the schedule, is this a class or a continuation of the class above?
|
|
def categorize():
|
|
# todo: must we open all these files?
|
|
dates = {}
|
|
|
|
files = sorted(os.listdir('.'))
|
|
files = list( filter( lambda x: re.search('reg(\d+)\.csv',x), files) )
|
|
files.reverse()
|
|
|
|
f = files[0]
|
|
filein = codecs.open(f,'r','utf-8').readlines()[1:]
|
|
sections = {}
|
|
this_section = []
|
|
|
|
for L in filein:
|
|
parts = L.strip().split(',') # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
|
|
parts = list( map( lambda x: clean_funny3(x), parts ) )
|
|
|
|
if not re.search('(\d+)',parts[0]): # This is a continuation
|
|
this_section.append(parts)
|
|
else: # this is a new section or the first line
|
|
if this_section:
|
|
sections[ this_section[0][0] ] = this_section
|
|
#print "Section: " + this_section[0][0] + " is: " + str(this_section) + "\n"
|
|
#print this_section[0][0] + "\t", course_start(this_section)
|
|
#print this_section[0][0] + "\t", course_time(this_section)
|
|
#print this_section[0][0] + "\t", course_location(this_section)
|
|
this_section = [ parts, ]
|
|
return sections
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Deprecated. call perl.
|
|
def constructSchedule():
|
|
term = raw_input("Name of html file? (ex: sp18.html) ")
|
|
os.chdir('make-web-sched')
|
|
cmd = 'perl make.pl ' + term
|
|
print "command: " + cmd
|
|
os.system(cmd)
|
|
|
|
|
|
|
|
"""
|
|
def fetch_dict(target,params={}):
|
|
# if there are more results, return the url for more fetching.
|
|
# else return false
|
|
#print target
|
|
global results_dict
|
|
r2 = requests.get(target, headers = header, params=params)
|
|
output = r2.text
|
|
if output.startswith('while('):
|
|
output = output[9:]
|
|
#print output
|
|
mycopy = results_dict.copy()
|
|
results_dict = {}
|
|
results_dict.update(json.loads(output))
|
|
results_dict.update(mycopy)
|
|
f.write(json.dumps(results_dict, indent=2))
|
|
#print "\n"
|
|
if ('link' in r2.headers):
|
|
links = r2.headers['link'].split(',')
|
|
for L in links:
|
|
ll = L.split(';')
|
|
link = ll[0].replace("<","")
|
|
link = link.replace(">","")
|
|
if re.search(r'next', ll[1]):
|
|
#print ll[1] + ":\t" + link
|
|
return link
|
|
return ""
|
|
"""
|
|
|
|
def get_schedule(term='201870', sem='fall'):
|
|
"""
|
|
sched_data = { 'term_in':term, 'sel_subj':'dummy', 'sel_day':'dummy',
|
|
'sel_schd':'dummy', 'sel_insm':'dummy', 'sel_camp':'dummy', 'sel_levl':'dummy', 'sel_sess':'dummy',
|
|
'sel_instr':'dummy', 'sel_ptrm':'dummy', 'sel_attr':'dummy', 'sel_subj':'%', 'sel_crse':'', 'sel_title':'',
|
|
'sel_schd':'%', 'sel_from_cred':'', 'sel_to_cred':'', 'sel_camp':'%', 'sel_ptrm':'%', 'sel_sess':'%',
|
|
'sel_attr':'%', 'begin_hh':'0', 'begin_mi':'0', 'begin_ap':'a', 'end_hh':'0', 'end_mi':'0', 'end_ap':'a' }
|
|
initial_headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
'Accept-Encoding':'gzip, deflate, sdch, br',
|
|
'Accept-Language':'en-US,en;q=0.8',
|
|
'Connection':'keep-alive',
|
|
'Host':'ssb.gavilan.edu',
|
|
'Upgrade-Insecure-Requests':'1',
|
|
} #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
|
|
headers = { 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
'Accept-Encoding':'gzip, deflate, br',
|
|
'Accept-Language':'en-US,en;q=0.8',
|
|
'Cache-Control':'max-age=0',
|
|
'Connection':'keep-alive',
|
|
'Content-Type':'application/x-www-form-urlencoded',
|
|
'Host':'ssb.gavilan.edu',
|
|
'Origin':'https://ssb.gavilan.edu',
|
|
'Referer':'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term='+term,
|
|
'Upgrade-Insecure-Requests':'1',
|
|
} #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
|
|
initial_url = 'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=' + term
|
|
sesh = requests.Session()
|
|
#r1 = sesh.get(initial_url,headers=initial_headers)
|
|
#sesh.headers.update(headers)
|
|
url = 'https://ssb.gavilan.edu/prod/bwckschd.p_get_crse_unsec'
|
|
r1 = sesh.get(initial_url)
|
|
r = sesh.post(url, data=sched_data)
|
|
print r.headers
|
|
data = r.text
|
|
out = open('data/temp/'+term+'.html','w')
|
|
out.write(data)
|
|
out.close()"""
|
|
os.system('perl parse_schedule.pl data/temp/' + term + '.html' + ' ' + sem)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#####
|
|
#####
|
|
##### conf.py ?
|
|
|
|
|
|
str="""355 985 1296
|
|
354 730 1295
|
|
353 319 1290
|
|
352 985 1289
|
|
351 813 1285
|
|
350 281 1285
|
|
349 267 1279
|
|
348 981 1252
|
|
347 994 1252
|
|
346 26 1250
|
|
345 757 1288
|
|
344 368 1288
|
|
343 1 1286
|
|
259 703 1295
|
|
256 693 1293
|
|
255 660 1292
|
|
254 1 1291
|
|
250 482 1287
|
|
246 2 1284
|
|
245 333 1283
|
|
244 27 1282
|
|
243 703 1281
|
|
242 730 1281
|
|
241 482 1280
|
|
239 211 1278
|
|
238 794 1278
|
|
237 2 1277
|
|
236 297 1276
|
|
235 831 1276
|
|
233 482 1251"""
|
|
|
|
for L in str.split("\n"):
|
|
(id,host,session) = L.split("\t")
|
|
qry = "INSERT INTO conf_signups (user,session,timestamp) VALUES (%s,%s,'2022-08-08 17:20:00');" % (host,session)
|
|
print(qry)
|
|
|
|
|
|
|