commit 09fb62577258d23701710ee2dc71dd55abcc6951
Author: Coding with Peter <coding.with.peter@gmail.com>
Date:   Wed Mar 22 09:29:52 2023 -0700

    initial commit from canvasapp HDD

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..55dc7c6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,15 @@
+secrets.py
+*.bak
+.ipynb_checkpoints
+104ab42f11
+__pycache__
+cache
+mergeme
+qanda
+qanda_student
+sftp
+static
+ipython_log.*
+completer.hist
+*.zip
+*.un~
\ No newline at end of file
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apphelp.py b/apphelp.py
new file mode 100644
index 0000000..0d117af
--- /dev/null
+++ b/apphelp.py
@@ -0,0 +1,1619 @@
+
+
+import os,re
+
+output = ''
+todos = 0
+
+todos_d = {}
+
+# TODO: Make a second pass, and look for fxn calls inside of 
+#       each function. Draw a graphviz.
+#
+
+def fxns(fname):
+    global output, todos
+    lines = open(fname).readlines()
+    prev_L = ""
+    for L in lines:
+        
+        # is it a todo
+        a = re.search(r'(TODO|todo)\s*:?\s*(.*)$',L)
+        if a:
+            output += "\t\ttodo: " + a.group(2) + "\n"
+            todos += 1
+            if fname in todos_d:
+                todos_d[fname] += 1
+            else:
+                todos_d[fname] = 1
+    
+        # is it a function def?
+        if re.search('^\s*def\s',L):
+            output += "\n"
+            if re.search('^#',prev_L): output += "\t"+prev_L
+            output += "\t"+L+""
+        prev_L = L
+        
+files = os.listdir('.')
+files.sort()
+
+
+for F in files:
+    if F=='apphelp.py': continue
+    if re.search('\.py$',F):
+        output +=  "\n" + F + "\n"
+        fxns(F)
+       
+prog_in = open('apphelp.py','r')
+prog_out = ''
+td = '# Total TODOs remaining: %i' % todos   
+
+td2 = '# TODOs per file:  \n#\n'
+for k in sorted(todos_d.keys()):
+    td2 += "#\t%i - %s \n" % (todos_d[k],k)
+    
+
+for R in prog_in.readlines():
+    if re.search('^##\sF',R):
+        prog_out += "## Functions\n#\n%s\n#\n%s\n#\n" % (td,td2)
+        break
+    prog_out += R
+
+prog_out += '\n"""\n\n' + output + '\n\n"""\n\n'
+prog_in.close()
+prog_write = open('apphelp.py','w')
+prog_write.write(prog_out)
+prog_write.close()
+
+
+
+
+
+## Functions
+#
+# Total TODOs remaining: 57
+#
+# TODOs per file:  
+#
+#	1 - checker.py 
+#	1 - content.py 
+#	6 - courses.py 
+#	3 - curriculum.py 
+#	5 - depricated.py 
+#	6 - localcache.py 
+#	2 - outcomes.py 
+#	20 - pipelines.py 
+#	2 - server.py 
+#	2 - tasks.py 
+#	1 - tempget.py 
+#	8 - users.py 
+
+#
+
+"""
+
+
+__init__.py
+
+checker.py
+		todo: make this sweet
+
+	def safe_html(html):
+
+	def _attr_name_whitelisted(attr_name):
+
+	def safe_css(attr, css):
+
+	def plaintext(input):
+
+	    def _unescape(text):
+
+	        def fixup(m):
+
+	def check_folder(fname,path):
+
+	def check_class(folder):
+
+	def check_all():
+
+content.py
+
+	def d(s):
+
+	def stripper(s):
+
+	def mycleaner(s):
+
+	def freshdesk():
+
+	# Build a master file with the entire class content
+	def accessible_check(id=""):
+		todo: include linked pages even if they aren't in module
+
+	def pan_testing():
+
+	# Given course, page url, and new content, upload the new revision of a page
+	def create_page(course_num,new_title,new_content):
+
+	def md_to_course():
+
+	# DL pages only
+	def grab_course_pages(course_num=-1):
+
+	# Appears to not be used
+	def put_course_pages():
+
+	# Also not used
+	def put_revised_pages():
+
+	# Download, clean html, and reupload page
+	def update_page():
+
+	# Given course, page url, and new content, upload the new revision of a page
+	def upload_page(course_num,pageurl,new_content):
+
+	# Use template to build html page with homegrown subtitles    
+	def build_srt_embed_php(data):
+
+	def yt_title(code):
+
+	def swap_youtube_subtitles():
+
+	def test_swap():
+
+	def multiple_downloads():
+
+courses.py
+		todo: 
+
+	def int_or_zero(x):
+
+	def float_or_zero(x):
+
+	# Gott 1 Bootcamp - report on who completed it.
+	def get_gott1_passers():
+
+	# Plagiarism Module - report on who completed it.
+	def get_plague_passers():
+
+	# Who, in a class, passed?
+	def get_course_passers(course, min_passing, passers_filename, still_active_filename):
+
+	# Change courses to show 2 announcements
+	def change_course_ann_homepage(id="10458"):
+
+	def scrape_bookstore():
+		todo: where does the most recent schedule come from?
+
+	# Input: xxxx_sched.json. Output: xxxx_latestarts.txt    
+	def list_latestarts():
+
+	# All students enrolled in a class in the given semester. Simpler verson of below. Return SET of course_ids.
+	def users_in_semester():
+		todo: 
+
+	# All students in STEM (or any list of depts.. match the course_code). Return SET of canvas ids.
+	def users_in_depts_live(depts=[], termid='171'):
+
+	def course_enrollment(id=''):
+
+	def askForTerms():
+
+	# Return a list of term names and IDs. Also store in cache/courses/terms.txt
+	def getTerms(printme=1, ask=1):
+		todo: unsafe overwrite
+
+	def getCourses():   # a dict
+
+	# Relevant stuff trying to see if its even being used or not
+	def course_term_summary():
+
+	# Fetch all courses in a given term
+	def getCoursesInTerm(term=0,show=1,active=0):        # a list
+
+	def getCoursesTermSearch(term=0,search='',v=0):
+
+	def courseLineSummary(c,sections={}):
+
+	def xlistLineSummary(c,sections={}):
+
+	def eslCrosslister():
+
+	def xlist(parasite='', host=''):        # section id     ,    new course id 
+		todo: need to get the section id from each course:
+
+	def unenroll_student(courseid,enrolid):
+
+	def enroll_stem_students_live():
+
+	def enroll_orientation_students():
+
+	def summarize_proportion_online_classes(u):
+
+	def summarize_num_term_classes(u):
+
+	def make_ztc_list(sem='sp20'):
+
+	def course_search_by_sis():
+
+	def add_evals(section=0):
+		todo: wanted: group shell for each GP (guided pathway) as a basic student services gateway....
+
+curriculum.py
+		todo: These secrets
+
+	def another_request(url,startat):
+
+	def fetch_all_classes():
+
+	def fetch_all_programs():
+
+	def sortable_class(li):
+
+	def c_name(c):
+
+	def show_classes(createoutput=1):
+
+	def clean_d_name(d):
+
+	def show_programs():
+
+	def dd(): return defaultdict(dd)
+
+	def organize_courses():
+
+	def check_de():
+
+	def clean_programs():
+
+	def course_lil_format(s):
+
+	def header_lil_format(s):
+
+	def organize_programs():
+
+	def divide_courses_list(li,rwd,online):
+
+	def organize_programs2():
+
+	# sorting by order key of dict
+	def cmp_2(a):
+
+	def cmp_order(a,b):
+
+	# decipher the grouped up courses line
+	def split_course(st):
+
+	# Any number gets an X (checked). Blank or zero gets no check.
+	def units_to_x(u):
+
+	def p_block_rule(r,printme,doc,out=0):
+
+	def p_cert_header(type,doc,r='',out=0):
+
+	def p_block_header(r,doc,out=0):
+
+	def p_cert_course_missing(cd,doc,out=0):
+
+	def p_cert_course(cd,history,doc,out=0):
+
+	def p_end_block(out=0):
+
+	def p_end_cert(bigdoc, out=0):
+
+	def ask_for_rule(r):
+
+	def action_to_english(a):
+
+	# Return True if the courses satisfy the rule
+	def check_a_block(b, courses, verbose=False):
+
+	def read_block_english_to_code():
+
+	def read_section_online_history():
+		todo: this file depends on other fxns. which?
+
+	# This is the 3rd attempt.
+	def simple_find_online_programs():
+		todo: courses with a HYPHEN in NAME get parsed wrong.
+
+	def check_a_block_a(b,verbose=False):
+
+	def smart_find_online_programs():
+
+	        def show_contained_class(c):
+
+	        def show_block(c):
+
+	        def show_block(c):
+
+	        def is_online(c):
+
+	        def is_online(c):
+
+	        def is_online_inblock(c):
+
+	        def is_online_inblock(c):
+
+	# of all the programs, what can be accomplished online?    
+	def find_online_programs():
+
+	# take a string of all the types of classes offered, return a vector of [tot,lec,hyb,onl]
+	def string_to_types(st):
+
+	def my_default_counter():
+
+	# Of the recent schedules, what was actually offered online?
+	def summarize_online_sections():
+
+	def fibonacci(n):
+
+	def test_pampy():
+
+	def cq_parse_experiment(root=0, indent=''):
+
+	def cq_start():
+
+	def cq_pattern_backup1(root=0, indent=''):
+
+	def found(*x): 
+
+	def lookForMatch(rules,item):
+
+	def cq_pattern(root=0, indent=''):
+
+	def myprinter(item, indent=''):
+
+	def cq_pattern_start():
+
+	def baby_int(j):
+
+	def find_deg_in_cluster( clusters, deg ):
+
+	def try_match_deg_programs():
+
+	def dict_generator(indict, pre=None):
+
+	def print_dict(v, prefix='',indent=''):
+
+	def walk_file():
+
+	def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+	def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+	def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+	def server_save(key,value):
+
+	def flask_thread(q):
+
+	    def home():
+
+	    def s(key,val):
+
+	    def hello():
+
+	    def sd():
+
+	def serve():
+
+	def attempt_match8020(rules,item):
+
+	def clever_printer(item, indent=''):
+
+	def print_return(x):
+
+	def cq_8020(root=0, indent=''):
+
+	def cq_8020_start():
+
+curriculum2020.py
+
+	def to_md(s):
+
+	def print_return(x):
+
+	def cq_8020(root,indent=0):
+
+	def cq_8021(root,indent=0):
+
+	def cq_8021_start():
+
+	def cq_8022(root,indent=0):
+
+	def cq_8022_start():
+
+	def sortable_class(li):
+
+	def c_name(c):
+
+	def show_classes2020():
+
+	def show_classes2020_start():
+
+curriculum_patterns.py
+
+	def div1(a,b): 
+
+	def d2(a,b): 
+
+	def d3(a,b): 
+
+	def pp1(a,b): 
+
+	def pp2(a,b):
+
+	def pp3(a,b,c,d,e,f,g):
+
+	def pp4(a,b):
+
+	def pp5(a,b,c):
+
+	def pp6(a,b):
+
+	def pp7(a,b):
+
+	def pp8(a,b):
+
+	def jj1(a,b,c,d,e):
+
+	def jj2(a,b,c,d,e,f):
+
+	def jj3(a,b,c,d,e):
+
+	def jj4(a,b,c,d):
+
+	def jj5(a,b,c,d,e,f):
+
+	def jj6(a,b,c,d):
+
+	def jj2(a,b,c,d):
+
+	def jj2(a,b,c,d):
+
+depricated.py
+
+	# Don't know
+	def demo():
+
+	def stats():
+
+	def dict_generator(indict, pre=None):
+
+	def print_dict(v, prefix='',indent=''):
+
+	def walk_file():
+
+	def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+	def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+	def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+	def server_save(key,value):
+
+	def flask_thread(q):
+
+	    def home():
+
+	    def s(key,val):
+
+	    def hello():
+
+	    def sd():
+
+	def serve():
+		todo: this duplicates courses.py  ??
+
+	# Prompt for course id, return list of user dicts.   TODO this duplicates courses.py  ??
+	def getUsersInCourse(id=0):   # returns list
+		todo: 
+
+	# NO LONGER USED - SEE COURSES
+	def enroll_stem_students():
+
+	# unused?    
+	def getAllTeachersInTerm():        # a list
+		todo: hits in courses by teachers    https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
+
+	def teacherActivityLog(uid=1):   ### Next: save results in a hash and return that....
+
+	def summarize_student_teacher_role(u):
+
+	def user_roles2():
+
+	def req_to_db(fname_list):
+
+	def has_online(series):
+
+	def has_lecture(series):
+
+	def has_hybrid(series):
+
+	# Wrapper to get 2 schedules at once
+	def dl_sched():
+		todo: these semesters
+
+	# Send a personalized email regarding ZTC    
+	def send_z_email(fullname, firstname, addr, courses_list):
+
+	def getInactiveTeachersInTerm(t=23):        # a list
+
+	def course_location(course):
+
+	def course_time(course):
+
+	def course_teacher(course):
+
+	def reg_nums():
+
+	# In the schedule, is this a class or a continuation of the class above?    
+	def categorize():
+		todo: must we open all these files?
+
+	# Deprecated. call perl.
+	def constructSchedule():
+
+	def fetch_dict(target,params={}):
+
+	def get_schedule(term='201870', sem='fall'):
+
+interactive.py
+
+	def dict_generator(indict, pre=None):
+
+	def print_dict(v, prefix='',indent=''):
+
+	def walk_file():
+
+	def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+	def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+	def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+	def server_save(key,value):
+
+	def flask_thread(q):
+
+	    def before_request():
+
+	    def save_post():
+
+	    def restart():
+
+	    def dispatch3(func,arg,arrg):
+
+	    def dispatch2(func,arg):
+
+	    def dispatch(func):
+
+	    def dispatch3j(func,arg,arrg):
+
+	    def dispatch2j(func,arg):
+
+	    def dispatch1j(func):
+
+	    def home():
+
+	    def send_jslib(path):
+
+	    def send_cachedata(path):
+
+	    def send_js(path):
+
+	    def s(key,val):
+
+	    def do_sample():
+
+	    def media(file_id):
+
+	    def podcast():
+
+	    def weblec():
+
+	    def hello():
+
+	    def sd():
+
+	    def test_message(message):
+
+	def serve():
+
+	    def make_teacher_rel(self, tchr, clss):
+
+	    def __init__(self, uri, user, password):
+
+	    def close(self):
+
+	    def print_greeting(self, message):
+
+	    def _create_and_return_greeting(tx, message):
+
+	def make_teacher_rel(g, tchr, clss):
+
+	def testgraph():
+
+	def Memoize( func):
+
+	    def wrapper(*args):
+
+	    def startup(self, outfile):
+
+	    def set_my_dict(self,d):
+
+	    def cycle_color(self, s):
+
+	    def ascii_art(self, text):
+
+	    def close_window(self, ):
+
+	    def suggest(self, word):
+
+	    def curses_print_word(self, word,color_pair_code):
+
+	    def curses_print_line(self, line,color_pair_code):
+
+	    def redraw(self, start_y,end_y,fallback_y,fallback_x):
+
+	    def scroll_down(self, noredraw,fallback_y,fallback_x):
+
+	    def clear_upside(self, n,y,x):
+
+	    def display_suggest(self, y,x,word):
+
+	    def inputloop(self, ):
+
+	    def setup_command(self,outfile):
+
+	    def cleanup_command(self):
+
+	    def handle_command(self, cmd):
+
+	def repl_staff():
+
+	def repl_degs():
+
+	def repl():
+
+	def repl():
+
+interactivex.py
+
+	def dict_generator(indict, pre=None):
+
+	def print_dict(v, prefix='',indent=''):
+
+	def walk_file():
+
+	def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+	def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+	def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+	def server_save(key,value):
+
+	def flask_thread(q):
+
+	    def before_request():
+
+	    def restart():
+
+	    def dispatch3(func,arg,arrg):
+
+	    def dispatch2(func,arg):
+
+	    def dispatch(func):
+
+	    def dispatch3j(func,arg,arrg):
+
+	    def dispatch2j(func,arg):
+
+	    def dispatch1j(func):
+
+	    def home():
+
+	    def send_jslib(path):
+
+	    def send_cachedata(path):
+
+	    def send_js(path):
+
+	    def s(key,val):
+
+	    def do_sample():
+
+	    def hello():
+
+	    def sd():
+
+	def serve():
+
+	    def make_teacher_rel(self, tchr, clss):
+
+	    def __init__(self, uri, user, password):
+
+	    def close(self):
+
+	    def print_greeting(self, message):
+
+	    def _create_and_return_greeting(tx, message):
+
+	def make_teacher_rel(g, tchr, clss):
+
+	def Memoize( func):
+
+	    def wrapper(*args):
+
+	    def startup(self, outfile):
+
+	    def set_my_dict(self,d):
+
+	    def cycle_color(self, s):
+
+	    def ascii_art(self, text):
+
+	    def close_window(self, ):
+
+	    def suggest(self, word):
+
+	    def curses_print_word(self, word,color_pair_code):
+
+	    def curses_print_line(self, line,color_pair_code):
+
+	    def redraw(self, start_y,end_y,fallback_y,fallback_x):
+
+	    def scroll_down(self, noredraw,fallback_y,fallback_x):
+
+	    def clear_upside(self, n,y,x):
+
+	    def display_suggest(self, y,x,word):
+
+	    def inputloop(self, ):
+
+	    def setup_command(self,outfile):
+
+	    def cleanup_command(self):
+
+	    def handle_command(self, cmd):
+
+	def repl_staff():
+
+	def repl_degs():
+
+	def repl():
+
+	def repl():
+
+ipython_log.py
+
+localcache.py
+
+	def db():
+
+	def setup_table(table='requests'):
+
+	# Help the next function to upload new users directly to conf database on gavilan.
+	def employees_refresh_flex(data):
+
+	# Everyone in iLearn DB with an   xyz@gavilan.edu   email address. 
+	def all_gav_employees():
+
+	#
+	def teachers_courses_semester():
+
+	# 
+	def teachers_by_term():
+
+	# Report for AEC
+	def aec_su20_report():
+
+	# Return the most up do date version of the given file. Useful for 'dimensions'.
+	def most_recent_file_of( target ):
+
+	    def finder(st):
+
+	# Given a table schema, parse log file, return a list of dicts. Optionally remove some columns.
+	def parse_file_with(  file, format, with_gid=0  ):
+
+	# I return a list of the read lines if the log dates in the file are within dates (top of this file), or FALSE
+	def is_requestfile_interesting(fname):
+		todo: more robust here
+		todo: - investigate pearson, developer key: 170000000000376 and their ridiculous amounts of hits.
+
+	# Return a 'timeblock'. An integer number of 15 minute blocks from my epoch. Expects a datetime object in PST timezone.
+	def timeblock_from_dt(dt_obj):
+
+	# Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+	def dt_from_timeblock(tb):
+
+	# Twenty Four hour timeblocks
+	def timeblock_24hr_from_dt(dt_obj):
+
+	# Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+	def dt_from_24hr_timeblock(tb):
+
+	# Four hour timeblocks
+	def timeblock_4hr_from_dt(dt_obj):
+
+	# Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+	def dt_from_4hr_timeblock(tb):
+
+	# I make the line into a dict, erase keys with no data, make a DT field called date, make a time_block (int) field.
+	def requests_line(line,i=0):
+
+	# Bulk insert of requests logs. Too much data to be useful. 
+	def requests_file(fname_list):
+		todo: select if timeblock exists
+
+	# Insert or update a request line.
+	def upsert_request(line, vals):
+
+	# Generic insert of a dict into a table. Keys of dict must match table columns.
+	def dict_to_insert(thisline,table):  # a dict
+
+	# This now does tallying by timeblock. 
+	def merge_requests():
+
+	def merge_comm_channel():
+
+	def merge_pseudonym():
+
+	def merge_users():
+
+	def merge_courses():
+
+	def merge_enrollment():
+
+	def merge_term():
+
+	def merge_roles():
+
+	def merge_convos():
+
+	# For returning sqlite results as dicts
+	def dict_factory(cursor, row):
+		todo: ... approaches to all this data... list requests in order descending time, unique users, and just 
+
+	# Attempt to do tallying
+	def make_views_summarys():
+
+	# original without time_blocks info.
+	def make_views_summarys_v1():
+
+	# Setup my basic db stats base from scratch
+	def full_reload():
+
+	def guess_dept(t):
+
+	# Main view of all class / all user overview...
+	def dept_with_studentviews(dept="", sem=''):
+
+	    def f(x):
+
+	# get student count and teacher name from local db
+	def course_quick_stats(canvasid):
+
+	# What a student has taken / teacher has taught
+	def user_enrolled_in(userid):
+
+	# All students in this semester ...
+	def users_this_semester_db(sem=''):
+
+	# Everyone whose first semester is .....
+	def users_new_this_semester(sem=''):
+
+	# All student users in STEM - from local db
+	def user_in_stem():
+
+	# Get all the classes in one dept
+	def dept_classes(dept,sem=''):
+		todo: 
+
+	def depts_with_classcounts(sem=''):
+		todo: 
+
+	    def f(x):
+
+	    def name_with_count(name,li):
+
+	def arrange_data_for_web(dept='', sem=''):
+
+	    def f(x):
+
+	# Get enrollments. (Best to freshly run pipelines/get_rosters) and put them into DB
+	def build_tables(headers,name):
+
+	def load_tables(table,headers,row,verbose=0):
+
+	def semester_enrollments(verbose=0):
+
+	    def qstrip(txt): return txt.strip('"')
+
+	def more_unused_xreferencing():
+
+	def user_role_and_online():
+
+	def comm_channel_file():
+
+	def pseudonym_file():
+
+	def users_p_file():
+
+	    def com_channel_dim():
+
+	def abcd():
+
+	def crns_to_teachers():
+
+main.py
+
+outcomes.py
+
+	def outcome_overview(term=21):
+
+	def create_acct_lvl_outcomes(src,dept,makefolder='',folder=0):
+
+	def connect_acct_oc_to_course(course_id,oc_group_id):
+
+	def outcome_groups():
+
+	def outcome_groups_backup():
+
+	def x_ref_dept_names():
+
+	def create_course_group(short,parent):
+
+	def create_dept_group(short):
+
+	def outcomes_attached_to_courses(term=65,limitdept=''):
+		todo: Handle this:     CSIS/DM85 WEB DESIGN 40823/24
+
+	def summarize_course_online_slo(outcome_list):
+
+	def fetch_outcome_details(id):
+
+	# Report on the actual evaluation data?        
+	def outcome_report1():
+		todo: 
+
+	# For the given course, get all outcome measurements, and display scores and stats.
+	def outcome_report2():
+
+	def fix_joined_class(str):
+
+	def split_slo_name(str):
+
+	def outcome_report3():
+
+	def read_slo_source():
+
+	def slo_source_by_dept():
+
+patterns_8020.py
+
+patterns_topdown.py
+
+	def pp0(a,b,c,d,e):
+
+	def pp1(a,b,c,d,e,f):
+
+	def pp2(a,b,c,d,e):
+
+	def pp3(a,b,c,d):
+
+	def pp4(a,b,c):
+
+	def pp5(a,b,c):
+
+	def pp6(a,b,c):
+
+	def div1(a,b): 
+
+	def d2(a,b): 
+
+	def d3(a,b): 
+
+	def pp1(a,b): 
+
+	def pp2(a,b):
+
+	def pp3(a,b,c,d,e,f,g):
+
+	def pp4(a,b):
+
+	def pp5(a,b,c):
+
+	def pp6(a,b):
+
+	def pp7(a,b):
+
+	def pp8(a,b):
+
+	def jj3(a,b,c,d,e):
+
+	def jj5(a,b,c,d,e,f):
+
+	def jj2(a,b,c,d):
+
+	def jj2(a,b,c,d):
+
+pipelines.py
+		todo: secrets
+		todo: all these constants for SSB  --  line 1008
+		todo: secrets
+
+	def d(s): 
+
+	# Main canvas querying fxn    
+	def fetch(target,verbose=0):
+
+	# Main canvas querying fxn - stream version - don't die on big requests
+	def fetch_stream(target,verbose=0): 
+
+	# paging makes problems... example: enrollment_terms    
+	def fetch_collapse(target,collapse='',verbose=0):
+
+	# Teacher name format changed. Remove commas and switch first to last   
+	def fix_t_name(str):
+
+	# Separate dept and code
+	def split_class_dept(c):
+
+	def split_class_code(c):
+
+	def split_class_code_letter(c):
+
+	# go from sp20 to 2020spring
+	def shortToLongSem(s):
+
+	# Go to the semesters folder and read the schedule. Return dataframe    
+	def getSemesterSchedule(short='sp21'):                # I used to be current_schedule
+		todo: Some semesters have a different format.... partofday   type   site  xxx i just dL'd them again
+
+	def prep_online_courses_df():
+
+	def course_is_online(crn):
+
+	def get_crn_from_name(name):
+
+	def get_enrlmts_for_user(user,enrollments):
+
+	# Get something from Canvas Data
+	def do_request(path):  
+
+	# Canvas data, download all new files
+	def sync_non_interactive():
+
+	# list files in canvas_data (online) and choose one or some to download.
+	def interactive():
+
+	def todays_date_filename():       # helper    
+
+	def nowAsStr():     # possible duplicate
+
+	def row_has_data(r):      # helper    
+
+	def row_text(r):   # helper  
+
+	# Take banner's html and make a csv(?) file  
+	def ssb_to_csv(src):
+
+	def clean_funny(str):
+
+	def clean_funny2(str):
+
+	def clean_funny3(str):
+
+	### course is a list of 1-3 lists, each one being a line in the schedule's output. First one has section     
+	def course_start(course):
+		todo: use this to make a early/late/short field and store semester dates w/ other constants
+
+	def time_to_partofday(t):
+		todo: account for multiple sites/rows
+
+	# Deduce a 'site' field, based on room name and known offsite locations    
+	def room_to_site(room,verbose=0):
+		todo: account for multiple sites/rows
+		todo: better way to store these offsite labels
+
+	# take text lines and condense them to one dict per section    
+	def to_section_list(input_text,verbose=0):
+		todo: no output files
+		todo: if extra line is different type?
+
+	# Log the history of enrollments per course during registration
+	def log_section_filling(current_sched_list):
+
+	# Same as above, but compressed, act only
+	def log_section_filling2(current_sched_list):
+
+	# Use Firefox and log in to ssb and get full schedule. Only works where selenium is installed
+	def scrape_schedule():
+		todo: my data here.... secret
+		todo: 
+
+	# recreate schedule json files with most current online schedule format. 
+	def recent_schedules():
+		todo: sems is a global in this file. Is that the right thing to do?   
+		todo: the pipeline is disorganized. Organize it to have 
+		todo: where does this belong in the pipeline? compare with recent_schedules()
+
+	# Take the generically named rosters uploads files and move them to a semester folder and give them a date.    
+	def move_to_folder(sem,year,folder):
+
+	# This relates to enrollment files, not schedule. 
+	def convert_roster_files(semester="",year="",folder=""):
+
+	# From instructure sftp site
+	def fetch_current_rosters():
+		todo: secret
+
+	def fetch_current_rosters_auto():
+
+	# read schedule file with an eye toward watching what's filling up
+	def schedule_filling():
+		todo: hardcoded
+
+	# Upload a json file to www
+	def put_file(remotepath,localpath, localfile,prompt=1):
+		todo: remove this secret
+		todo: these paths
+
+	def sec(t): return "<h3>"+t+"</h3>\n"
+
+	def para(t): return "<p>"+t+"</p>\n"
+
+	def ul(t): return "<ul>"+t+"</ul>\n"
+
+	def li(t): return "<li>"+t+"</li>\n"
+
+	def question(t,bracket=1): 
+
+	def answer(t): 
+
+	def read_paragraph_element(element,type="NORMAL_TEXT"):
+
+	def get_doc(docid, bracket=1, verbose=0):
+		todo: x link, x bold, list, image.
+
+	def read_paragraph_element_2(element,type="NORMAL_TEXT"):
+
+	# t is a string that begins with "Icons: " ... and contains comma(space) separated list
+	def handle_icons(t):
+
+	# t is a string that begins with "Tags: " ... and contains comma(space) separated list
+	def handle_tags(t):
+
+	def handle_question(t,bracket=1):
+
+	def handle_answer(t):
+
+	def handle_sec(t): return ('section',t)
+
+	def handle_para(t): return ('paragraph',t)
+
+	def handle_ul(t): return ('unorderdedlist',t)
+
+	def handle_li(t): return ('listitem',t)
+
+	def fetch_doc_image(k,value):
+
+	def get_doc_generic(docid, bracket=1, verbose=0):
+
+	def scrape_schedule_py():
+
+server.py
+
+	def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+	def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+	def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+	def homepage():
+
+	def orgline(L):
+		todo: \s\[\#A\](.*)$', L)
+
+	def editor(src):
+
+	def in_form(txt,path):
+
+	def mytime(fname):
+
+	def index():
+
+	def writing(fname):
+
+	def dashboard():
+
+	def dash():
+
+	def mycalendar():
+
+	def most_recent_file_of( target, folder ):
+
+	    def finder(st):
+
+	def news():
+
+	def randPic():
+
+	def sample():
+
+	def sample2(a=""):
+
+	# Filter a stream of loglines for those that match a course's url / id
+	def has_course(stream,courseid):
+
+	def js(s):
+
+	def sem_from_array_crn(crn):
+
+	def user_courses(uid):
+
+	def user_course_history_summary(usr_id):
+
+	def roster(crn):
+
+	def user_course_hits(usr,courseid):
+
+	def profiles(id=1,b=2,c=3):
+
+	# Departments, classes in each, and students (with hits) in each of those.
+	def enrollment(a):
+
+	# All the classes in this dept, w/ all the students in each, with count of their views.
+	def dept(d=''):
+
+	def user(canvas_id=None):
+
+	def lectures():
+
+	def web_lectures():
+		todo: update: dept, title, any of the other fields. 
+
+	# update a value: dept id of a personnel id
+	def update_pers_title(pid, tid):
+
+	# update a value: dept id of a personnel id
+	def update_pers_dept(pid, did):
+
+	def user_edit(canvas_id='2'):
+
+	def staff_dir(search=''):
+
+	def server_save(key,value):
+
+	def server_dispatch_json(function_name,arg='', arg2=''):
+
+	def server_dispatch(function_name,arg='', arg2=''):
+
+stats.py
+
+	def grades_rundown():
+
+	def class_logs():
+
+	def user_logs():
+
+	def recent_logins():
+
+	def userHitsThisSemester(uid=2):
+
+	def getCurrentActivity():  # a dict
+
+	def externaltool():        # a list
+
+tasks.py
+
+	def survey_answer(q=0):
+
+	def survey_organize():
+
+	def build_quiz(filename=""):
+
+	# Send an email
+	def send_email(fullname, firstname, addr, subj, content):
+
+	def convert_to_pdf(name1, name2):
+
+	# Build (docx/pdf) certificates for gott graduates
+	def certificates_gott_build():
+
+	# Email experiment
+	def mail_test():
+
+	# Change LTI Settings. Experimental
+	def modify_x_tool():   
+
+	# Upload with sftp to www website folder:   student/online/srt/classfoldername
+	def put_file(classfoldername):
+		todo: ',cnopts=cnopts) as sftp:
+
+	# Switch everyone in a class to a teacher
+	def switch_enrol():
+
+	# Change dates & term of a class to unrestrict enrollment
+	def unrestrict_course():
+
+	# Bulk enroll users into a course    
+	def enroll_accred():
+
+	# Calculate attendance stats based on enrollment/participation at 20% of term progressed, then 60% of term progressed.
+	def twenty_sixty_stats(li):
+
+	# Older positive attendance hours calculation.
+	def hours_calc():
+
+	def course_2060_dates(crn=""):
+		todo: 
+
+	def course_update_all_users_locallogs(course_id=''):
+
+	def hours_calc_pulldata(course_id=''):
+
+	def xlist_cwe():
+
+	def pos_atten():
+
+temp.py
+
+tempget.py
+
+	# Use Firefox and log in to ssb and get full schedule
+	def login():
+		todo: my data here.... secret
+
+	def filename_friendly(str):
+
+	def otter():
+
+templates.py
+
+	def item_to_masonry(item):
+
+	def try_untemplate():
+
+	        def php_remove(m):
+
+	        def php_add(m):
+
+	def do_template(temp,source,side):
+
+	def remove_filetype(f):
+
+	def make():
+
+	def txt_2_table():
+
+	def studenttech_faq():
+
+	# https://docs.google.com/document/d/1tI_b-q75Lzu25HcA0GCx9bGfUt9ccM8m2YrrioDFZcA/edit?usp=sharing
+	def de_faq():
+
+	def degwork_faq():
+
+	def vrc_faq():
+
+	def counseling_faq():
+
+	def finaid_faq():
+
+	def coun_loc():
+
+	def tutor_faq():
+
+	def test_repl():
+
+timer.py
+
+	def func(a, b):
+
+users.py
+		todo: these constants
+
+	# All users to a cache file     cache/allusers.json
+	def fetchAllUsers():
+
+	# Fetch teacher users objects from local cache
+	def teacherRolesCache():               # I used to be load_users
+
+	# Canvas:   Fetch all people with gavilan.edu email address        
+	def teacherRolesUpdateCache():    # I used to be get_users
+
+	# Fetch preferred email address for a given user id.   ( Canvas )
+	def getEmail(user_id):
+
+	# All teachers in a particular course
+	def getAllTeachers(course_id=59):  # a list 
+
+	#
+	def classType(t):
+		todo: fix bug in schedule parser so non-online classes have a type field
+
+	def my_blank_string(): return "no data"
+
+	def my_blank_dict(): return {'name':'NoName','email':'noemail@gavilan.edu'}
+
+	def my_empty_dict(): return defaultdict(my_blank_string)
+
+	def get_email_from_rec(name,name_to_record):
+
+	# Pull the staff directory on the webpage. Convert to pandas dataframe
+	def staff_dir(get_fresh=False):
+		todo: lol get fresh again... 
+
+	def schedForTeacherOverview(long,short):
+
+	# Return a dataframe of the last 4 semester schedules put together
+	def oneYearSchedule():
+
+	def num_sections_last_year(line):
+
+	def sec_type_stats(line):
+
+	def prct_online(line):
+
+	def prct_lecture(line):
+
+	def prct_hybrid(line):
+
+	# Given the names of teachers in last year's schedules, fill in email, etc. from ilearn files
+	def teacher_basic_info(sched, from_ilearn, names):
+
+	    def find_that_name(x):
+
+	# Outputs:   cache/teacher_by_semester.csv, 
+	def teacherModalityHistory(sched=[],names=[]):
+
+	def teacherCourseHistory(a,names):
+		todo: sort by dept also
+
+	# Outputs:   cache/course_teacher_combos.csv, 
+	def teacherSharedCourses(a=[]):
+
+	# How many courses in each department were taught in the last year?
+	def departmentCountCourses(a=[]):
+
+	def clean_nonprint(s):
+
+	def read_cmte(names):
+
+	def read_training_records():
+
+	# open a file and mark the people with their ids given. Return a dataframe
+	def read_bootcamp1(filename):
+
+	# open a file and mark the people with their ids given. Return a dataframe
+	def read_bootcamp2(filename):
+
+	def not_blank_or_pound(L):
+
+	def temp1(x):
+
+	def add_realnames(df,names):    # the surveys. raw name is in 2nd column
+
+	def compareToughNames(a,b):
+
+	def compareNames(a,b,verbose=0):
+
+	def find_ilearn_record(ilearn_records,manual_records, othername,verbose=0):
+
+	def manualNamesAndDept():
+
+	def manualNames():
+
+	# given a list of class codes, return the most common (academic) department
+	def guessDept(d_list):
+
+	# Make one big csv file of everything I know about a teacher    
+	def getTeachersInfoMain():
+
+	def enroll_staff_shell():
+
+	# take a list of raw hits. 
+	def activity_summary(hits):
+		todo: month is hardcoded here
+
+	# Get views counts on current teachers. todo: month is hardcoded here
+	def get_recent_views(id=1):
+
+	# Have they taught online or hybrid classes?        
+	def categorize_user(u):
+		todo: threaded
+
+	# Doest the account have a photo loaded?
+	def checkForAvatar(id=2):
+
+	# Grab em. Change the first if when continuing after problems....
+	def downloadPhoto():
+
+	def mergePhotoFolders():
+
+	def mergePhotoFolders2():
+
+	# Go through my local profile pics, upload any that are missing.
+	def uploadPhoto():
+
+	def test_email():
+
+	def create_ztc_list():
+
+	def get_user_info(id):
+
+	# these are any messages that get pushed out to their email
+	def comm_mssgs_for_user(uid=0):
+
+	#
+	def convos_for_user(uid=0):
+
+	# single q sub
+	def quiz_get_sub(courseid, quizid, subid=0):
+
+	# quiz submissions for quiz id x, in course id y
+	def quiz_submissions(courseid=9768, quizid=32580):
+
+	# return (timeblock, course, read=0,write=1)
+	def requests_line(line,i=0):
+
+	# 
+	def report_logs(id=0):
+
+	def track_users_in_sem():
+
+	def track_users_in_class(L=[]):
+
+	def track_user_q(id, q):
+
+	# Maintain local logs. Look to see if we have some, download logs since then for a user.
+	def track_user(id=0,qid=0):
+		todo: set up this info file if it isn't there. check any changes too. it
+		todo: 
+
+	#
+	def track_users_by_teacherclass():
+
+	def nlp_sample():
+
+	def nlp_sample2():
+
+	def one_course_enrol():
+
+util.py
+
+	def print_table(table):
+
+	def remove_nl(str):
+
+	def UnicodeDictReader(utf8_data, **kwargs):
+
+	def minimal_string(s):
+
+	def to_file_friendly(st):
+
+	def clean_title(st):    
+
+	def match59(x):
+
+	def item_2(x): return x[2]
+
+	def unix_time_millis(dt):
+
+	# ENGL250 returns ENGL    
+	def dept_from_name(n):
+
+	def most_common_item(li):
+
+	def srt_times(a,b):
+
+	def how_long_ago(a):    # number of hours ago 'a' was...
+
+	def partition(times_list):
+
+
+"""
+
diff --git a/checker.py b/checker.py
new file mode 100644
index 0000000..53a6d2d
--- /dev/null
+++ b/checker.py
@@ -0,0 +1,225 @@
+# Common functions for checking web and canvas for accessibility
+
+import os, sys, glob, codecs
+import subprocess, re, pdb, html
+from bs4 import BeautifulSoup, Comment
+import html.entities
+from datetime import datetime
+import pdb
+#from html.parser import HTMLParseError
+
+
+# the following from: https://chase-seibert.github.io/blog/2011/01/28/sanitize-html-with-beautiful-soup.html#
+# hasnt been tested yet
+
+
+def safe_html(html):
+
+    if not html:
+        return None
+
+    # remove these tags, complete with contents.
+    blacklist = ["script", "style" ]
+
+    whitelist = [ 
+        "div", "span", "p", "br", "pre","a",
+        "blockquote",
+        "ul", "li", "ol",
+        "b", "em", "i", "strong", "u", "iframe","img",
+        "h1","h2","h3","h4","h5","h6"
+        ]
+
+    try:
+        # BeautifulSoup is catching out-of-order and unclosed tags, so markup
+        # can't leak out of comments and break the rest of the page.
+        soup = BeautifulSoup(html,'lxml')
+    except Exception as e:
+        # special handling?
+        raise e
+
+    removelist = ['table','tbody','thead','th','tr','td']
+
+    # now strip HTML we don't like.
+    for tag in soup.findAll():
+        if tag.name.lower()=='iframe': continue
+        if tag.name.lower()=='img': continue
+        if tag.name.lower() in blacklist:
+            # blacklisted tags are removed in their entirety
+            tag.extract()
+        elif tag.name.lower() in whitelist:
+            # tag is allowed. Make sure all the attributes are allowed.
+            #print tag
+            #print tag.attrs
+            #pdb.set_trace()
+            #tag.attrs = [(a[0], safe_css(a[0], a[1])) for a in tag.attrs if _attr_name_whitelisted(a[0])]
+            for k,v in list(tag.attrs.items()):
+                #print 'attr: ' + str(k) + ' = ' + str(v) + '.... ',
+                if not _attr_name_whitelisted(k):
+                    tag.attrs.pop(k)
+                    #print ' removed'
+                else:
+                    tag.attrs[k] = v
+                    #print ' kept'
+        elif tag.name.lower() in removelist:
+            tag.unwrap()
+        else:
+            # not a whitelisted tag. I'd like to remove it from the tree
+            # and replace it with its children. But that's hard. It's much
+            # easier to just replace it with an empty span tag.
+            
+            #tag.name = "span"
+            #tag.attrs = []
+            tag.unwrap()
+
+    # scripts can be executed from comments in some cases
+    comments = soup.findAll(text=lambda text:isinstance(text, Comment))
+    for comment in comments:
+        comment.extract()
+
+    safe_html = str(soup)
+
+    if safe_html == ", -":
+        return None
+
+    return safe_html
+
+def _attr_name_whitelisted(attr_name):
+    return attr_name.lower() in ["href", "src","width","height","alt","target","title","class","id"]
+
+def safe_css(attr, css):
+    if attr == "style":
+        return re.sub("(width|height):[^;]+;", "", css)
+    return css
+
+def plaintext(input):
+    """Converts HTML to plaintext, preserving whitespace."""
+
+    # from http://effbot.org/zone/re-sub.htm#unescape-html
+    def _unescape(text):
+        def fixup(m):
+            text = m.group(0)
+            if text[:2] == "&#":
+                # character reference
+                try:
+                    if text[:3] == "&#x":
+                        return chr(int(text[3:-1], 16))
+                    else:
+                        return chr(int(text[2:-1]))
+                except ValueError:
+                    pass
+            else:
+                # named entity
+                try:
+                    text = chr(html.entities.name2codepoint[text[1:-1]])
+                except KeyError:
+                    pass
+            return text # leave as is
+        return re.sub("&#?\w+;", fixup, text)
+
+    input = safe_html(input) # basic sanitation first
+    text = "".join(BeautifulSoup("<body>%s</body>" % input).body(text=True))
+    text = text.replace("xml version='1.0' encoding='%SOUP-ENCODING%'", "") # strip BS meta-data
+    return _unescape(text)
+
+
+#os.system("node node_modules/pa11y/bin/pa11y.js --standard Section508 http://www.gavilan.edu/student/online")
+
+
+def check_folder(fname,path):
+    report = '<h2>' + fname + '</h2>\n'
+    number = -1
+    count = 0
+    try:
+        for F in os.listdir(path+fname):    #'assignments'):A
+            cmd = "/usr/bin/node " + \
+                "/home/phowell/Documents/access/node_modules/pa11y/bin/pa11y.js --standard Section508 " + \
+                path + fname + "/" + F
+            print(("" + path + fname + "/" + F))
+            output = subprocess.run(cmd, stdout=subprocess.PIPE, 
+                universal_newlines=True, shell=True, check=False)
+
+            report += "<h3>" + F + "</h3>\n"
+            line = output.stdout.split('\n')[-3]
+            if re.search('No\sissues',line): 
+                pass
+                #print("Got zero")
+            else:
+                m = re.search('(\d+)\sErr',line)
+                if m:
+                    count += int(m.group(1))
+                lines = output.stdout.split("\n")
+                #pdb.set_trace()
+                lines = lines[4:]
+                report += "<pre>" + html.escape("\n".join(lines)) + "</pre>\n\n\n"
+    except Exception as e:
+        print('finished with error or folder missing')
+        print(e)
+    return int(count), report
+
+def check_class(folder):
+    path = "/home/phowell/hdd/SCRIPTS/everything-json/course_temps/" + folder + "/"         
+    class_report = "<h1>Report on course: " + folder + "</h1>\n\n"
+    (cnt_a,rep_a) = check_folder('assignments',path) 
+    (cnt_p,rep_p) = check_folder('pages',path)
+    class_report += rep_a
+    class_report += rep_p
+
+    #oo = open(path+'report.html','w')
+    #oo.write(class_report)
+    #oo.close()
+    #print(class_report)
+    return cnt_a+cnt_p, class_report
+
+def check_all():
+    hd_path = '/home/phowell/hdd/SCRIPTS/everything-json/'
+
+    rep_f = open(hd_path+'report.html','w')
+    rep_s = open(hd_path+'summary.html','w')
+
+    rep_f.write('<meta charset="utf-8"/>\n')
+
+    listt = os.listdir('/home/phowell/hdd/SCRIPTS/everything-json/course_temps')
+    #listt = ['course_4341',]       # for testing
+    for L in listt:
+        print(('Directory is: '+L))
+        m = glob.glob('/home/phowell/hdd/SCRIPTS/everything-json/course_temps/' +L+'/*.txt')
+        if m: name = m[0]
+        else: name = 'unknown.txt'
+        name = name.split('.')[0]
+        name = name.split('/')[-1]
+
+        print(('name is: ' + name))
+        (cnt,rep) = check_class(L)
+        rep_f.write("<a name='"+L+"'><h1>"+name+"</h1>\n"+rep+"\n\n<br /><br />\n\n")
+        rep_f.flush()
+        rep_s.write("("+str(cnt)+") Class: <a href='report.html#"+L+"'>"+name+"</a><br />\n")
+        rep_s.flush()
+
+if __name__ == "__main__":
+    check_all()
+
+    #print(('arguments: '+str(sys.argv)))
+
+    # test
+    """
+    file = 'course_temps/course_6862/pages/choose-the-right-browser.html'
+    dir = 'course_temps/course_6862/pages/'
+    #ff = open(file,'r').read()
+    #print safe_html(ff)
+
+    for file in os.listdir(dir):
+        if re.search('_cleaned\.html',file):
+            os.remove(dir+file)
+
+    for file in os.listdir(dir):
+        if file.endswith(".html"):
+            newfname = re.sub('\.html','_cleaned.html',file)
+            ff = codecs.open(dir+file,'r','utf-8').read()
+            print(file)
+            print(newfname)
+            newf = codecs.open(dir+newfname,'w','utf-8')
+            newf.write(safe_html(ff))
+            newf.close()
+    """
+
+
diff --git a/content.py b/content.py
new file mode 100644
index 0000000..be5553d
--- /dev/null
+++ b/content.py
@@ -0,0 +1,860 @@
+
+
+#saved_titles = json.loads( codecs.open('cache/saved_youtube_titles.json','r','utf-8').read() )
+import requests, codecs, os, re, json
+from pipelines import header, fetch, url
+from util import clean_title, to_file_friendly
+from bs4 import BeautifulSoup as bs
+from html.parser import HTMLParser
+import tomd, checker 
+import html2markdown as h2m
+import pypandoc
+h = HTMLParser()
+
+
+DBG = 1
+
+def d(s):
+    global DBG
+    if DBG: print(s)
+    
+def stripper(s):
+    REMOVE_ATTRIBUTES = [
+    'lang','language','onmouseover','onmouseout','script','style','font',
+    'dir','face','size','color','style','class','width','height','hspace',
+    'border','valign','align','background','bgcolor','text','link','vlink',
+    'alink','cellpadding','cellspacing']
+
+    #doc = '''<html><head><title>Page title</title></head><body><p id="firstpara" align="center">This is <i>paragraph</i> <a onmouseout="">one</a>.<p id="secondpara" align="blah">This is <i>paragraph</i> <b>two</b>.</html>'''
+    soup = bs(s, features='lxml')
+    for tag in soup.recursiveChildGenerator():
+        try:
+            tag.attrs = {key:value for key,value in tag.attrs.iteritems() 
+                         if key not in REMOVE_ATTRIBUTES}
+        except AttributeError: 
+            # 'NavigableString' object has no attribute 'attrs'
+            pass
+    return soup.prettify()
+
+def mycleaner(s):
+    s = re.sub(r'<br\s?\/>','\n',s)
+    s = re.sub(r'<\/?b>','',s)
+    s = re.sub(r' +',' ',s)
+    s = re.sub(r'^[\s\t\r\n]+$','',s,flags=re.MULTILINE)
+    s = re.sub('^ ','',s)
+    return s
+
+def freshdesk():
+    path = "C:\\Users\\peter\\Downloads\\freshdesk\\Solutions.xml"
+    soup = bs( codecs.open(path,'r','utf-8').read() ,features="lxml")
+    
+    outpt = codecs.open('cache/faqs.txt','w')
+    out = ""
+    for a in soup.find_all('solution-article'):
+    
+        print("TITLE\n"+a.find('title').get_text())
+        out += a.find('title').get_text()
+        
+        """for d in a.find_all('description'):
+            #print(d)
+            if d:
+                d = h.unescape(d.get_text())
+                e = stripper(d)
+                m = tomd.convert( e )
+                m = mycleaner(m)
+                print("\nDESCRIPTION\n"+m)"""
+        
+        #print("\nWHAT IS THIS?\n" + 
+        hh = a.find('desc-un-html').get_text()
+        d = h.unescape(hh)
+        e = stripper(d)
+        m = tomd.convert( e )
+        m = mycleaner(m)
+        print("\nDESCRIPTION\n"+m)
+        out += "\n\n" + m + "\n\n"
+        
+        print("-----------\n\n")
+    outpt.write(out)
+
+# Download everything interesting in a course to a local folder
+# Build a master file with the entire class content
+def accessible_check(id=""):
+    if not id:
+        id = input("ID of course to check?  ")
+    pagebreak = '\n\n<!-- BREAK -->\n\n'
+    verbose = 1
+    
+    save_file_types = ['application/pdf','application/docx','image/jpg','image/png','image/gif','image/webp','application/vnd.openxmlformats-officedocument.wordprocessingml.document']
+
+    courseinfo = fetch('/api/v1/courses/' + str(id), verbose )
+    
+    item_id_to_index = {}
+    items_inorder = ["<font size='24'>" + courseinfo['name'] + "</font>\n\n" + pagebreak,]
+    running_index = 1
+    
+    modules = fetch('/api/v1/courses/' + str(id) + '/modules',verbose)
+    
+    items = []
+    for x in range(9000): items.append(0)
+    
+    video_link_list = []
+    
+    for m in modules:
+        items[running_index] = '<h2>%s</h2>%s\n' % ( m['name'], pagebreak )
+        running_index += 1
+        
+        mod_items = fetch('/api/v1/courses/' + str(id) + '/modules/'+str(m['id'])+'/items', verbose)
+        
+        for I in mod_items:
+            
+            if I['type'] in ['SubHeader', 'Page', 'Quiz', 'Discussion', 'ExternalUrl' ] or 'content_id' in I:
+                running_index += 1
+                
+                if I['type'] == 'SubHeader': 
+                    #print('subheader: ' + str(I))
+                    items[running_index] = '<h3>%s</h3>\n' % str(json.dumps(I,indent=2))
+                    
+                if I['type'] == 'Page': 
+                    item_id_to_index[ I['page_url'] ] = running_index
+                    
+                if I['type'] == 'Quiz':  
+                    item_id_to_index[ I['content_id'] ] = running_index
+                
+                if I['type'] == 'Discussion':  
+                    item_id_to_index[ I['content_id'] ] = running_index
+                
+                if I['type'] == 'ExternalUrl':
+                    items[running_index] = "<a href='%s'>%s</a><br />\n\n" % (I['external_url'], I['title'])
+                    
+                # ?
+                #if 'content_id' in I:
+                #    item_id_to_index[ I['content_id'] ] = running_index
+            else:
+                print("What is this item? " + str(I))
+                    
+            
+            #items_inorder.append('<i>Not included: '+ I['title'] + '(a ' + I['type'] + ')</i>\n\n\n' )
+            
+            # I['title']
+            # I['content_id']
+            # I['page_url']
+            # I['type']
+            # I['published']
+    # assignments and files have content_id, pages have page_url
+    
+    course_folder = '../course_temps/course_'+id
+    index = []
+    try:
+        os.mkdir(course_folder)
+    except:
+        print("Course folder exists.")
+    ###
+    ### FILES
+    ###
+    files_f = course_folder + '/files'
+    headered = 0
+    print("\nFILES")
+    try:
+        os.mkdir(files_f)
+    except:
+        print(" * Files folder already exists.")
+        
+    files = fetch('/api/v1/courses/' + str(id) + '/files', verbose)
+    print("LISTING COURSE FILES")
+    for f in files:
+        for arg in 'filename,content-type,size,url'.split(','):
+            if arg=='size':
+                f['size'] = str(int(f['size']) / 1000) + 'k'
+        
+        if f['content-type'] in save_file_types:
+            d(' - %s' % f['filename'])
+            
+            if not os.path.exists(files_f + '/' + f['filename']):
+                r = requests.get(f['url'],headers=header, stream=True)
+                with open(files_f + '/' + f['filename'], 'wb') as fd:
+                    for chunk in r.iter_content(chunk_size=128):
+                        fd.write(chunk)
+            else:
+                d(" - already downloaded %s" % files_f + '/' + f['filename'])
+                
+            if not headered: 
+                index.append( ('<br /><b>Files</b><br />') )
+                headered = 1
+            index.append( ('files/' + f['filename'], f['filename']) )
+        
+    ###
+    ### PAGES
+    ###
+    pages_f = course_folder + '/pages'
+    headered = 0
+    image_count = 0
+    print("\nPAGES")
+    try:
+        os.mkdir(pages_f)
+    except:
+        print(" * Pages folder already exists.")
+        
+        
+    pages = fetch('/api/v1/courses/' + str(id) + '/pages', verbose)
+    for p in pages:
+        d(' - %s' % p['title'])
+        
+        p['title'] = clean_title(p['title'])
+        easier_filename = clean_title(p['url'])
+        this_page_filename = "%s/%s.html" % (pages_f, easier_filename)
+        #for a in 'title,updated_at,published'.split(','):
+        #    print(str(p[a]), "\t", end=' ')
+
+        if not headered: 
+            index.append( ('<br /><b>Pages</b><br />') )
+            headered = 1
+        index.append( ( 'pages/' + easier_filename + '.html', p['title'] ) )
+
+
+        if os.path.exists(this_page_filename):
+            d(" - already downloaded %s" % this_page_filename)
+            this_page_content = open(this_page_filename,'r').read()
+        elif re.search(r'eis-prod',p['url']) or re.search(r'gavilan\.ins',p['url']):
+            d('   * skipping file behind passwords')
+        else:    
+            t2 = fetch('/api/v1/courses/' + str(id) + '/pages/'+p['url'], verbose)
+            if t2 and 'body' in t2 and t2['body']:
+                bb = bs(t2['body'],features="lxml")
+                a_links = bb.find_all('a')
+                for A in a_links:
+                    if re.search( r'youtu', A['href']):
+                        video_link_list.append( (A['href'], A.text, 'pages/'+easier_filename + ".html") )
+                        
+
+                page_images = bb.find_all('img') 
+                for I in page_images:
+                    d('   - %s' % I['src'])
+                    if re.search(r'eis-prod',I['src']) or re.search(r'gavilan\.ins',I['src']):
+                        d('   * skipping file behind passwords')
+                    else:
+                        try:
+                            r = requests.get(I['src'],headers=header, stream=True)
+                            mytype = r.headers['content-type']
+                            #print("Response is type: " + str(mytype))
+                            r_parts = mytype.split("/")
+                            ending = r_parts[-1]
+
+                            with open(pages_f + '/' + str(image_count) + "." + ending, 'wb') as fd:
+                                for chunk in r.iter_content(chunk_size=128):
+                                    fd.write(chunk)
+                            image_count += 1
+                        except Exception as e:
+                            d( ' * Error downloading page image, %s' % str(e) )
+                
+                try:
+                    with codecs.open(this_page_filename, 'w','utf-8') as fd:
+                        this_page_content = "<h2>%s</h2>\n%s" % ( t2['title'], t2['body'] )
+                        fd.write(this_page_content)
+                except:
+                    d(' * problem writing page content')
+                    ## TODO include linked pages even if they aren't in module
+            else:
+                d('   * nothing returned or bad fetch')
+        # write to running log of content in order of module
+        if p and p['url'] in item_id_to_index:
+            items[  item_id_to_index[ p['url'] ]  ] = this_page_content +'\n\n'+pagebreak
+        else:
+            d(' -- This page didnt seem to be in the modules list.')
+    
+    
+    ###
+    ### ASSIGNMENTS
+    ###
+    headered = 0
+    asm_f = course_folder + '/assignments'
+    print("\nASSIGNMENTS")
+    try:
+        os.mkdir(asm_f)
+    except:
+        d(" - Assignments dir exists")
+        
+    asm = fetch('/api/v1/courses/' + str(id) + '/assignments', verbose)
+    for p in asm:
+        d(' - %s' % p['name'])
+
+        
+        try:
+            friendlyfile = to_file_friendly(p['name'])
+            this_assmt_filename = asm_f + '/' + str(p['id'])+"_"+ friendlyfile + '.html'
+            if os.path.exists(this_assmt_filename):
+                d(" - already downloaded %s" % this_assmt_filename)
+                this_assmt_content = open(this_assmt_filename,'r').read()
+            else:
+                t2 = fetch('/api/v1/courses/' + str(id) + '/assignments/'+str(p['id']), verbose)
+                with codecs.open(this_assmt_filename, 'w','utf-8') as fd:
+                    this_assmt_content = "<h2>%s</h2>\n%s\n\n" % (t2['name'], t2['description'])
+                    fd.write(this_assmt_content)
+                if not headered: 
+                    index.append( ('<br /><b>Assignments</b><br />') )
+                    headered = 1
+                index.append( ('assignments/' + str(p['id'])+"_"+friendlyfile + '.html', p['name']) )
+                
+            # write to running log of content in order of module
+            if p['id'] in item_id_to_index:
+                items[  item_id_to_index[ p['url'] ]  ] = this_assmt_content+'\n\n'+pagebreak
+        except Exception as e:
+            d(' * Problem %s' % str(e))
+    
+    ###
+    ### FORUMS
+    ###
+    """forum_f = course_folder + '/forums'
+    headered = 0
+    image_count = 0
+    print("\nFORUMS")
+    try:
+        os.mkdir(forum_f)
+        forums = fetch('/api/v1/courses/' + str(id) + '/discussion_topics', verbose)
+        for p in forums:
+            p['title'] = clean_title(p['title'])
+            forum_id = p['id']
+            easier_filename = p['title']
+            for a in 'title,posted_at,published'.split(','):
+                print(str(p[a]), "\t", end=' ')
+            print("")
+            t2 = fetch('/api/v1/courses/' + str(id) + '/discussion_topics/'+str(forum_id), verbose)
+            
+            
+            #### REMOVED
+            bb = bs(t2['body'],features="lxml")
+            print("IMAGES IN THIS PAGE")
+            page_images = bb.find_all('img') 
+            for I in page_images:
+                r = requests.get(I['src'],headers=header, stream=True)
+                mytype = r.headers['content-type']
+                print("Response is type: " + str(mytype))
+                r_parts = mytype.split("/")
+                ending = r_parts[-1]
+
+                with open(pages_f + '/' + str(image_count) + "." + ending, 'wb') as fd:
+                    for chunk in r.iter_content(chunk_size=128):
+                        fd.write(chunk)
+                image_count += 1
+            #### END REMOVED
+            
+            try:
+                with codecs.open(forum_f + '/' + easier_filename + '.html', 'w','utf-8') as fd:
+                    fd.write("<h1>"+t2['title']+"</h1>\n")
+                    fd.write(t2['message'])
+                if not headered: index.append( ('<br /><b>Discussion Forums</b><br />') )
+                headered = 1
+                index.append( ( 'forums/' + easier_filename + '.html', p['title'] ) )
+                
+                # write to running log of content in order of module
+                if p['id'] in item_id_to_index:
+                    items_inorder[  item_id_to_index[ p['id'] ]  ] = '<h1>'+t2['title']+'</h1>\n\n'+t2['message']+'\n\n'+pagebreak
+                else:
+                    print('  This forum didnt seem to be in the modules list.')
+            except Exception as e:
+                print("Error here:", e)
+                #print p
+                #print results_dict
+    except Exception as e:
+        print("** Forum folder seems to exist. Skipping those.")
+        print(e)
+        
+        
+        
+    
+    
+    
+    ###
+    ### QUIZZES
+    ###
+    
+    
+    # get a list external urls
+    headered = 0
+    t = url + '/api/v1/courses/' + str(id) + '/modules'
+    while t: t = fetch(t)
+    mods = results
+    results = []
+    for m in mods:
+        results = []
+        t2 = url + '/api/v1/courses/' + str(id) + '/modules/' + str(m['id']) + '/items'
+        while t2: t2 = fetch(t2)
+        items = results
+        for i in items:
+            #print i
+            if i['type'] == "ExternalUrl":
+                #print i
+                for j in 'id,title,external_url'.split(','):
+                    print unicode(i[j]), "\t",
+                print ""
+                if not headered: index.append( ('<br /><b>External Links</b><br />') )
+                headered = 1
+                index.append( (i['external_url'], i['title']) )
+    """
+    
+    
+    
+    # Create index page of all gathered items
+    myindex = codecs.open(course_folder+'/index.html','w','utf-8')
+    for i in index:
+        if len(i)==2: myindex.write("<a href='"+i[0]+"'>"+i[1]+"</a><br />\n")
+        else: myindex.write(i)
+    
+
+    
+    # Full course content in single file    
+    print("Writing main course files...")
+    mycourse = codecs.open(course_folder+'/fullcourse.raw.html','w','utf-8')
+    
+    for I in items:
+        if I:
+            mycourse.write(  I  )
+            
+    
+    
+    temp = open('cache/coursedump.txt','w')
+    temp.write( "items:  " +  json.dumps(items,indent=2) )
+    temp.write("\n\n\n")
+    temp.write( "index:  " + json.dumps(index,indent=2) )
+    temp.write("\n\n\n")
+    temp.write( "items_inorder:  " + json.dumps(items_inorder,indent=2) )
+    temp.write("\n\n\n")
+    temp.write( "item_id_to_index:  " + json.dumps(item_id_to_index,indent=2) )
+    
+    
+    
+    
+    
+    
+    
+    if video_link_list:
+        mycourse.write('\n<h1>Videos Linked in Pages</h1>\n<table>')
+        for V in video_link_list:
+            (url, txt, pg) = V
+            mycourse.write("<tr><td><a target='_blank' href='"+url+"'>"+txt+"</a></td><td> on <a target='_blank' href='" + pg + "'>" + pg + "</a></td></tr>\n")
+        mycourse.write("</table>\n")
+            
+    mycourse.close()
+    output = pypandoc.convert_file(course_folder+'/fullcourse.raw.html', 'html', outputfile=course_folder+"/fullcourse.html")
+    output1 = pypandoc.convert_file(course_folder+'/fullcourse.html', 'md', outputfile=course_folder+"/fullcourse.md")
+    output2 = pypandoc.convert_file(course_folder+'/fullcourse.html', 'docx', outputfile=course_folder+"/fullcourse.docx")
+
+
+def pan_testing():
+    course_folder = '../course_temps/course_6862'
+    output3 = pypandoc.convert_file(course_folder+'/fullcourse.md', 'html', outputfile=course_folder+"/fullcourse.v2.html")
+
+# Given course, page url, and new content, upload the new revision of a page
+def create_page(course_num,new_title,new_content):
+    t3 = url + '/api/v1/courses/' + str(course_num) + '/pages'
+    #xyz = raw_input('Enter 1 to continue and send back to: ' + t3 + ': ')
+    #print("Creating page: %s\nwith content:%s\n\n\n" % (new_title,new_content))
+    print("Creating page: %s" % new_title)
+    xyz = input('type 1 to confirm: ')   #'1'
+    if xyz=='1':
+        data = {'wiki_page[title]':new_title, 'wiki_page[body]':new_content}
+        r3 = requests.post(t3, headers=header, params=data)
+        print(r3)
+        print('ok')
+
+
+def md_to_course():
+    #input = 'C:/Users/peter/Nextcloud/Documents/gavilan/student_orientation.txt'
+    #output = 'C:/Users/peter/Nextcloud/Documents/gavilan/stu_orientation/student_orientation.html'
+    id = "11214"
+    infile = 'cache/pages/course_%s.md' % id
+    output = 'cache/pages/course_%s_fixed.html' % id
+    output3 = pypandoc.convert_file(infile, 'html', format='md', outputfile=output)
+    
+    xx = codecs.open(output,'r','utf-8').read()
+    soup = bs(  xx, features="lxml" )
+    soup.encode("utf-8")
+    
+    current_page = ""
+    current_title = ""
+    
+    for child in soup.body.children:
+        if child.name == "h1" and not current_title:
+            current_title = child.get_text()
+        elif child.name == "h1":
+            upload_page(id,current_title,current_page)
+            current_title = child.get_text()
+            current_page = ""
+            print( "Next page: %s" % current_title )
+        else:
+            #print(dir(child))
+            if 'prettify' in dir(child):
+                current_page += child.prettify(formatter="html")
+            else:
+                current_page += child.string 
+            
+    upload_page(id,current_title,current_page)
+    print("Done")
+
+
+# DL pages only
+def grab_course_pages(course_num=-1):
+    global results, results_dict, url, header
+    # course_num = raw_input("What is the course id? ")
+    if course_num<0:
+        course_num = input("Id of course? ")
+    else:
+        course_num = str(course_num)
+    modpagelist = []
+    modurllist = []
+    # We want things in the order of the modules
+    t4 = url + '/api/v1/courses/'+str(course_num)+'/modules?include[]=items'
+    results = fetch(t4)
+    i = 1
+    pageout = codecs.open('cache/pages/course_'+str(course_num)+'.html','w','utf-8')
+    pageoutm = codecs.open('cache/pages/course_'+str(course_num)+'.md','w','utf-8')
+    divider = "\n### "
+    for M in results:
+        print("Module Name: " + M['name'])
+        for I in M['items']:
+            if I['type']=='Page':
+                modpagelist.append(I['title'])
+                modurllist.append(I['page_url'])
+                pageout.write(divider+I['title']+'### '+I['page_url']+'\n')
+                easier_filename = clean_title(I['page_url'])
+                print("  " + str(i) + ". " + I['title'])
+                t2 = url + '/api/v1/courses/' + str(course_num) + '/pages/'+I['page_url']
+                print('Getting: ' + t2)
+                mypage = fetch(t2)
+                fixed = checker.safe_html(mypage['body'])
+                if fixed:
+                    #markdown = h2m.convert(fixed)
+                    #p_data = pandoc.read(mypage['body'])
+                    markdown = pypandoc.convert_text("\n<h1>" + I['title'] + "</h1>\n" + mypage['body'], 'md', format='html')
+                    pageout.write(fixed+'\n')
+                    pageoutm.write(markdown+'\n')
+                    pageout.flush()
+                i += 1
+    pageout.close()
+    pageoutm.close()
+
+# Upload pages. Local copy has a particular format.
+# Appears to not be used
+def put_course_pages():
+    course_num = '6862'
+    filein = codecs.open('cache/pages/course_'+str(course_num)+'.html','r','utf-8')
+    my_titles = []
+    my_urls = []
+    my_bodys = []
+    started = 0
+    current_body = ""
+    for L in filein.readlines():
+        ma = re.search('^###\s(.*)###\s(.*)$',L)
+        if ma:
+            my_titles.append(ma.group(1))
+            my_urls.append(ma.group(2))
+            if started:
+                my_bodys.append(current_body)
+                current_body = ""
+            started = 1
+        else:
+            current_body += "\n" + L
+    my_bodys.append(current_body)
+
+    i = 0
+    for U in my_urls:
+        # and now upload it....lol
+        upload_page(course_num,U,my_bodys[i])
+        i += 1
+
+# Also not used
+def put_revised_pages():
+    course_num = '6862'
+    course_folder = '../course_temps/course_6862'
+    filein = codecs.open(course_folder+'/fullcourse.v2.html','r','utf-8')
+    my_titles = []
+    my_urls = []
+    my_bodys = []
+    started = 0
+    current_body = ""
+    for L in filein.readlines():
+        ma = re.search('^<h1>(.*)</h1>.*$',L)
+        if ma:
+            my_titles.append(ma.group(1))
+            my_urls.append(ma.group(2))
+            if started:
+                my_bodys.append(current_body)
+                current_body = ""
+            started = 1
+        else:
+            current_body += "\n" + L
+    my_bodys.append(current_body)
+
+    i = 0
+    for U in my_urls:
+        # and now upload it....lol
+        upload_page(course_num,U,my_bodys[i])
+        i += 1
+        
+# Download, clean html, and reupload page
+def update_page():
+    global results, results_dict, url, header
+    # course_num = raw_input("What is the course id? ")
+    course_num = '6862'
+    t = url + '/api/v1/courses/' + str(course_num) + '/pages'
+    while t: t = fetch(t)
+    pages = results
+    results = []
+    mypagelist = []
+    myurllist = []
+    modpagelist = []
+    modurllist = []
+    for p in pages:
+        p['title'] = clean_title(p['title'])
+        mypagelist.append(p['title'])
+        myurllist.append(p['url'])
+        easier_filename = clean_title(p['url'])
+        #for a in 'title,updated_at,published'.split(','):
+        #    print unicode(p[a]), "\t",
+        #print ""
+
+    # We want things in the order of the modules
+    t4 = url + '/api/v1/courses/'+str(course_num)+'/modules?include[]=items'
+    while t4: t4 = fetch(t4)
+    mods = results
+    results = []
+    i = 1
+    print("\nWhat page do you want to repair?")
+    for M in mods:
+        print("Module Name: " + M['name'])
+        for I in M['items']:
+            if I['type']=='Page':
+                modpagelist.append(I['title'])
+                modurllist.append(I['page_url'])
+                print("  " + str(i) + ". " + I['title'])
+                i += 1
+
+    choice = input("\n> ")
+    choice = int(choice) - 1
+    chosen_url = modurllist[choice]
+    print('Fetching: ' + modpagelist[choice])
+    t2 = url + '/api/v1/courses/' + str(course_num) + '/pages/'+chosen_url
+    print('From: ' + t2)
+
+    results_dict = {}
+    while(t2): t2 = fetch_dict(t2)
+    mypage = results_dict
+    fixed_page = checker.safe_html(mypage['body'])
+    upload_page(course_num,chosen_url,fixed_page)
+
+# Given course, page url, and new content, upload the new revision of a page
+def upload_page(course_num,pageurl,new_content):
+    print("Repaired page:\n\n")
+    #print new_content
+    print(pageurl)
+    t3 = url + '/api/v1/courses/' + str(course_num) + '/pages/' + pageurl
+    xyz = input('Enter 1 to continue and send back to: ' + t3 + ': ')
+    #xyz = '1'
+    if xyz=='1':
+        data = {'wiki_page[body]':new_content}
+        r3 = requests.put(t3, headers=header, params=data)
+        print(r3)
+        print('ok')
+
+# Use template to build html page with homegrown subtitles    
+def build_srt_embed_php(data):
+    template = codecs.open('template_srt_and_video.txt','r','utf-8').readlines()
+    result = ''
+    for L in template:
+        L = re.sub('FRAMEID',data['frameid'],L)
+        L = re.sub('TITLE',data['title'],L)
+        L = re.sub('EMBEDLINK',data['embedlink'],L)
+        L = re.sub('SRTFOLDERFILE',data['srtfolderfile'],L)
+        result += L
+    return result
+ 
+
+
+
+def yt_title(code):
+    global saved_titles
+    if code in saved_titles:
+        return saved_titles[code]
+    a = requests.get('https://www.youtube.com/watch?v=%s' % code)
+    bbb = bs(a.content,"lxml")
+    ccc = bbb.find('title').text
+    ccc = re.sub(r'\s\-\sYouTube','',ccc)
+    saved_titles[code] = ccc
+    codecs.open('saved_youtube_titles.json','w','utf-8').write(json.dumps(saved_titles))
+    return ccc
+ 
+def swap_youtube_subtitles():
+    # example here:  http://siloor.github.io/youtube.external.subtitle/examples/srt/
+    
+    # srt folder, look at all filenames
+    srtlist = os.listdir('video_srt')
+    i = 0
+    for V in srtlist:
+        print(str(i) + '.  ' + V)
+        i += 1
+    choice = input("Which SRT folder? ")
+    choice = srtlist[int(choice)]
+    srt_folder = 'video_srt/'+choice
+    class_srt_folder = choice
+    srt_files = os.listdir(srt_folder)
+    srt_shorts = {}
+    print("\nThese are the subtitle files: " + str(srt_files))
+    for V in srt_files:
+        if V.endswith('srt'):
+            V1 = re.sub(r'(\.\w+$)','',V)
+            srt_shorts[V] = minimal_string(V1)
+    
+    crs_id = input("What is the id of the course?  ")
+    grab_course_pages(crs_id)
+    v1_pages = codecs.open('page_revisions/course_'+str(crs_id)+'.html','r','utf-8')
+    v1_content = v1_pages.read()
+    
+    # a temporary page of all youtube links
+    tp = codecs.open('page_revisions/links_' + str(crs_id) + '.html', 'w','utf-8')
+    
+    # course pages, get them all and look for youtube embeds
+    title_shorts = {}
+    title_embedlink = {}
+    title_list = []
+    print("I'm looking for iframes and youtube links.")
+    for L in v1_content.split('\n'):
+        if re.search('<a.*?href="https:\/\/youtu',L):
+            print("Possibly there's a linked video instead of embedded:" + L)
+        if re.search('iframe',L):
+            ma = re.compile('(\w+)=(".*?")')
+            #print "\n"
+            this_title = ''
+            for g in ma.findall(L):
+                print(g)
+                if g[0]=='title':
+                    this_title = g[1].replace('"','')
+                if g[0]=='src':
+                    this_src = g[1].replace('"','')
+                #print g
+            if not this_title:
+                tmp = re.search(r'embed\/(.*?)\?',this_src)
+                if not tmp: tmp = re.search(r'embed\/(.*?)$',this_src)
+                if tmp:
+                    this_title = yt_title(tmp.groups()[0])
+            title_shorts[this_title] = minimal_string(this_title)
+            title_list.append(this_title)
+            title_embedlink[this_title] = this_src    
+            print("%s\n" % this_title.encode('ascii','ignore'))
+            tp.write(  "%s<br><a target='_blank' href='%s'>%s</a><br /><br />" % (this_title, this_src, this_src) )
+    # match them
+    # lowercase, non alpha or num chars become a single space, try to match
+    # if any srts remain unmatched, ask. 
+    tp.close()
+    webbrowser.open_new_tab('file://C:/SCRIPTS/everything-json/page_revisions/links_'+str(crs_id)+'.html')
+    
+    matches = {}                    # key is Title, value is srt file
+    for S,v in list(srt_shorts.items()):
+        found_match = 0
+        print(v, end=' ')
+        for T, Tv in list(title_shorts.items()):
+            if v == Tv:
+                print(' \tMatches: ' + T, end=' ')
+                found_match = 1
+                matches[T] = S
+                break
+        #print "\n"
+    
+    print("\nThese are the srt files: ")
+    print(json.dumps(srt_shorts,indent=2))
+    print("\nThese are the titles: ")
+    print(json.dumps(title_shorts,indent=2))
+    print("\nThese are the matches: ")
+    print(json.dumps(matches,indent=2))
+    
+    print(("There are %d SRT files and %d VIDEOS found. " % ( len(list(srt_shorts.keys())), len(list(title_shorts.keys())) ) ))
+    
+    for S,v in list(srt_shorts.items()):
+        if not S in list(matches.values()):
+            print("\nDidn't find a match for: " + S)
+            i = 0
+            for T in title_list:
+                if not T in list(matches.keys()): print(str(i+1) + ". " + T.encode('ascii', 'ignore'))
+                i += 1
+            print("Here's the first few lines of the SRT:")
+            print((  re.sub(r'\s+',' ', '\n'.join(open(srt_folder+"/"+S,'r').readlines()[0:10]))+"\n\n"))
+            choice = input("Which one should I match it to? (zero for no match)  ")
+            if int(choice)>0:
+                matches[ title_list[ int(choice)-1 ] ] = S
+                print("SRT clean name was: %s, and TITLE clean name was: %s" % (v,title_shorts[title_list[ int(choice)-1 ]] ))
+    print("ok, here are the matches:")
+    print(json.dumps(matches,indent=2))
+    
+    # construct subsidiary pages, upload them
+    i = 0
+    for m,v in list(matches.items()):
+        # open template
+        # do replacement
+        i += 1
+        data = {'frameid':'videoframe'+str(i), 'title':m, 'embedlink':title_embedlink[m], 'srtfolderfile':v  }
+        print(json.dumps(data,indent=2))
+        file_part = v.split('.')[0]
+        new_php = codecs.open(srt_folder + '/' + file_part + '.php','w','utf-8')
+        new_php.write(build_srt_embed_php(data))
+        new_php.close()
+    #srt_files = os.listdir(srt_folder)
+    put_file(class_srt_folder)
+    
+    
+def test_swap():
+    crs_id = '6923'
+    # swap in embed code and re-upload canvas pages
+    v2_pages = codecs.open('page_revisions/course_'+str(crs_id)+'.html','r','utf-8')
+    v2_content = v2_pages.read()
+    ma = re.compile('(\w+)=(".*?")')
+    
+    for L in v2_content.split('\n'):
+        find = re.findall('<iframe(.*?)>',L)
+        if find:
+            print("Found: ", find)
+            for each in find:
+                #print "\n"
+                this_title = ''
+                this_src = ''
+                for g in ma.findall(each):
+                    #print g
+                    if g[0]=='title':
+                        this_title = g[1].replace('"','')
+                    if g[0]=='src':
+                        this_src = g[1].replace('"','')
+                    #print g
+                if not this_title:
+                    tmp = re.search(r'embed\/(.*?)\?',this_src)
+                    if not tmp: tmp = re.search(r'embed\/(.*?)$',this_src)
+                    if tmp:
+                        this_title = yt_title(tmp.groups()[0])
+                print("Found embed link: %s\n and title: %s\n" % (this_src,this_title.encode('ascii','ignore')))
+
+ 
+def multiple_downloads():
+    
+    x = input("What IDs? Separate with one space: ")
+    for id in x.split(" "):
+        accessible_check(id)
+
+        
+if __name__ == "__main__":
+    
+    print ('')
+    options = { 1: ['download a class into a folder / word file', accessible_check] ,
+                2: ['download multiple classes', multiple_downloads ],
+                3: ['convert stuff', pan_testing ],
+                4: ['convert md to html', md_to_course ],
+                5: ['import freshdesk content', freshdesk ],
+                6: ['download all a courses pages', grab_course_pages],
+              }
+    
+    for key in options:
+        print(str(key) + '.\t' + options[key][0])
+    
+    print('')
+    resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+    
+    
+    
diff --git a/courses.py b/courses.py
new file mode 100644
index 0000000..d5f641c
--- /dev/null
+++ b/courses.py
@@ -0,0 +1,1446 @@
+
+import json, re, requests, codecs, sys, time, funcy, os
+import pandas as pd
+#from tabulate import tabulate
+from dateutil import parser
+from datetime import datetime
+from util import print_table
+
+from pipelines import fetch, fetch_stream, getSemesterSchedule, fetch_collapse, header, url, shortToLongSem
+from pipelines import sems
+from localcache import users_new_this_semester, db, course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
+from collections import defaultdict
+
+
+
+stem_course_id = '11015'   # TODO
+
+
+#########
+#########  GET FACTS FROM INDIVIDUAL COURSES
+#########
+#########
+
+def int_or_zero(x):
+    if x == None: return 0
+    else: return int(x)
+
+def float_or_zero(x):
+    if x == None: return 0
+    else: return float(x)
+
+# Gott 1 Bootcamp - report on who completed it.
+def get_gott1_passers():
+    course = '1561'
+                          
+    min_passing = 85
+    passers_filename = 'cache/teacherdata/bootcamp_passed.csv'
+    still_active_filename = 'cache/teacherdata/bootcamp_active.csv'
+    get_course_passers(course, min_passing, passers_filename, still_active_filename)
+
+# Plagiarism Module - report on who completed it.
+def get_plague_passers():
+    course = '11633'
+    min_passing = 85
+    passers_filename = 'cache/teacherdata/plagiarism_passed.csv'
+    still_active_filename = 'cache/teacherdata/plagiarism_active.csv'
+    (passed, didnt) = get_course_passers(course, min_passing, passers_filename, still_active_filename)
+    passed = set( [z[2] for z in passed] )
+    didnt = set( [z[2] for z in didnt] )
+    enrol = [ [ str(z) for z in list(course_enrollment(cr)) ] for cr in ['11677','11698'] ]
+    
+    print(enrol)
+    
+    enrol = set(funcy.cat(enrol))
+    everyone = passed.union(didnt,enrol)
+    
+    reportable = passed.intersection(enrol)
+    outputfile = open('cache/plagcheck.txt','w').write( json.dumps( [ list(reportable), list(enrol), list(passed), list(didnt), list(everyone) ],indent=2))
+    return 1
+    
+    #enrol = { cr: [ str(z) for z in list(course_enrollment(cr).keys()) ] for cr in ['11677','11698',] }           
+    #   #  [x['user_id'] for x in course_enrollment(cr)]
+    outputfile = open('cache/plagcheck.txt','w').write( json.dumps( [ [z[2] for z in passed],[z[2] for z in didnt],enrol],indent=2))
+    return 1
+    
+    passed_d = {}
+    didnt_d = {}
+    
+    output_by_course = {}
+    course_s = {}
+    
+    for p in passed: passed_d[str(p[2])] = p
+    for p in didnt: didnt_d[str(p[2])] = p
+    
+    passed_s = [ str(k) for k in passed_d.keys() ]
+    didnt_s = [ str(k) for k in didnt_d.keys() ]
+    
+    
+    crossref = ['11677','11698',]
+    
+    outputfile = open('cache/plagcheck.txt','w')
+    oo = { 'passed': passed_d, 'didnt': didnt_d }
+    
+    for cr in crossref:
+        student_int = course_enrollment(cr)
+        student_d = { str(k): v for k,v in student_int.items() }
+        oo[cr] = student_d
+        
+        output_by_course[cr] = { 'passed':{}, 'didnt':{}, 'missing':{} }
+        
+        course_s[cr] = set( [ str(k) for k in student_d.keys() ])
+        
+        for k,v in student_d.items():
+            key_s = str(k)
+            
+            if key_s in passed_d:
+                output_by_course[cr]['passed'][key_s] = passed_d[key_s]
+            elif key_s in didnt_d:
+                output_by_course[cr]['didnt'][key_s] = didnt_d[key_s]
+            else:
+                output_by_course[cr]['missing'][key_s] = v['user']
+    
+    oo['final_output'] = output_by_course
+    oo['passed_s'] = list(passed_s)
+    oo['didnt_s'] = list(didnt_s)
+    
+    course_sd = {k: list(v) for k,v in course_s.items() }
+    
+    oo['course_s'] = course_sd
+    
+    outputfile.write(json.dumps(oo,indent=2))
+        
+
+# Who, in a class, passed?
+def get_course_passers(course, min_passing, passers_filename, still_active_filename):
+    path = url + '/api/v1/courses/%s/enrollments' % str(course)
+    
+    tempout = open('cache/passers_temp.txt','w')
+    
+    enrl = fetch( path, 0)
+    passed = []
+    didnt  = []
+    for E in enrl:
+        try:
+            n = E['user']['name']
+            oo = E['user']['sis_user_id']
+            i = str(E['user_id'])
+            r = E['role']
+            g = E['grades']['current_score']
+            l = E['last_activity_at']
+            p = float_or_zero(g) > min_passing
+            print( "%s: a %s, grade of %s. Passed? %s. Last seen: %s" % (n,r,str(g),str(p),l)  )
+            
+            tempout.write(json.dumps(E['user']['name']) + "\n")
+            tempout.write(json.dumps(E['grades'],indent=2) + "\n\n-----\n\n")
+            
+            if p:
+                passed.append( [n, oo, i, r, g, l ] )
+            else:
+                didnt.append( [n, oo, i, r, g, l ] )
+        except:
+            pass
+            
+    columns = ['name', 'goo','canvas_id','role','grade','last_activity']
+    pp = pd.DataFrame(passed, columns=columns)
+    pp.sort_values(by='last_activity',inplace=True)
+    pp.to_csv(passers_filename, index=False)
+    dd = pd.DataFrame(didnt, columns=columns)
+    dd.sort_values(by='last_activity',inplace=True)
+    dd.to_csv(still_active_filename, index=False)
+    
+    print("Saved output to \n - passed: %s\n - not passed: %s\n" % (passers_filename, still_active_filename))
+    return (passed,didnt)
+
+    
+    # Gott 1A
+    """course = '2908'
+    quiz = '15250'
+    pass_grade = 0.90
+    
+    path = url + '/api/v1/courses/%s/quizzes/%s/submissions' % (course,quiz)
+    q_subs = fetch_collapse(path, 'quiz_submissions')
+    for Q in q_subs:
+        prct = float_or_zero(Q['score']) / float_or_zero( Q['quiz_points_possible'] )
+        print( 'Passed: %s\t Score: %s,\tUser: %s' % \
+               ( str(prct>0.9), str(int_or_zero(Q['score'])), Q['user_id'] ))"""
+
+
+
+# Who, in a class and a quiz, passed?
+def get_quiz_passers():
+    # Gott 1 Bootcamp
+    course = '1561'
+    path = url + '/api/v1/courses/%s/enrollments' % course
+    enrl = fetch( path, 0)
+    min_passing = 85
+    passed = []
+    didnt  = []
+    for E in enrl:
+        try:
+            n = E['user']['name']
+            i = E['user_id']
+            r = E['role']
+            g = E['grades']['current_score']
+            l = E['last_activity_at']
+            p = float_or_zero(g) > min_passing
+            print( "%s: a %s, grade of %s. Passed? %s. Last seen: %s" % (n,r,str(g),str(p),l)  )
+            if p:
+                passed.append( [n, i, r, g, l ] )
+            else:
+                didnt.append( [n, i, r, g, l ] )
+        except:
+            pass
+            
+    columns = ['name','canvas_id','role','grade','last_activity']
+    pp = pd.DataFrame(passed, columns=columns)
+    pp.sort_values(by='last_activity',inplace=True)
+    pp.to_csv('cache/teacherdata/bootcamp_passed.csv', index=False)
+    dd = pd.DataFrame(didnt, columns=columns)
+    dd.sort_values(by='last_activity',inplace=True)
+    dd.to_csv('cache/teacherdata/bootcamp_active.csv', index=False)
+    
+    print("Saved output to ./teachers/bootcamp_*")
+            
+    # Gott 1A
+    """course = '2908'
+    quiz = '15250'
+    pass_grade = 0.90
+    
+    path = url + '/api/v1/courses/%s/quizzes/%s/submissions' % (course,quiz)
+    q_subs = fetch_collapse(path, 'quiz_submissions')
+    for Q in q_subs:
+        prct = float_or_zero(Q['score']) / float_or_zero( Q['quiz_points_possible'] )
+        print( 'Passed: %s\t Score: %s,\tUser: %s' % \
+               ( str(prct>0.9), str(int_or_zero(Q['score'])), Q['user_id'] ))"""
+
+
+
+
+# Change courses to show 2 announcements
+def change_course_ann_homepage(id="10458"):
+    u = url + "/api/v1/courses/%s/settings" % id
+    data = { 'show_announcements_on_home_page':'true', \
+            'home_page_announcement_limit':'2'}
+    r = requests.put(u, data=data, headers=header)
+    print(r.text)
+
+
+#########
+#########  BOOKSTORE
+#########
+#########
+
+def scrape_bookstore():
+    big_courselist_url = "https://svc.bkstr.com/courseMaterial/courses?storeId=10190&termId=100058761"
+    bcu_cached = json.loads( open('cache/bookstore_courses.json','r').read() )
+    
+    one_section = "https://svc.bkstr.com/courseMaterial/results?storeId=10190&langId=-1&catalogId=11077&requestType=DDCSBrowse"   # NO TEXT
+    
+    another_section = "https://svc.bkstr.com/courseMaterial/results?storeId=10190&langId=-1&catalogId=11077&requestType=DDCSBrowse"   #  3 REQUIRED at: 
+    # [""0""].courseSectionDTO[""0""].courseMaterialResultsList
+    #
+    # and also:
+    #
+    # [""0""].courseSectionDTO[""0""].sectionAdoptionDTO.materialAdoptions
+    
+# todo: where does the most recent schedule come from?
+
+# Input: xxxx_sched.json. Output: xxxx_latestarts.txt    
+def list_latestarts():
+    #term = input("Name of current semester file? (ex: sp18)  ")
+    term = "sp23"   # sems[0]
+    
+    term_in = "cache/" + term + "_sched.json"                 
+    term_out = "cache/" + term + "_latestarts.txt"
+    print("Writing output to " + term_out)
+    infile = open(term_in, "r")
+    outfile = open(term_out, "w")
+    sched = json.loads(infile.read())
+    #print sched
+    by_date = {}
+    for C in sched:
+        parts = C['date'].split("-")
+        start = parts[0]
+        codes = C['code'].split(' ')
+        dept = codes[0]
+        if dept in ['JLE','JFT','CWE']:
+            continue
+        if re.search('TBA',start): continue
+        try:
+            startd = parser.parse(start)
+        except Exception as e:
+            print(e, "\nproblem parsing ", start)
+        #print startd
+        if not startd in by_date:
+            by_date[startd] = []
+        by_date[startd].append(C)
+    for X in sorted(by_date.keys()):
+        #print "Start: " + str(X)
+        if len(by_date[X]) < 200:
+            prettydate = X.strftime("%A, %B %d")
+            print(prettydate + ": " + str(len(by_date[X])) + " courses")
+            outfile.write(prettydate + ": " + str(len(by_date[X])) + " courses" + "\n")
+            for Y in by_date[X]:
+                #print "\t" + Y['code'] + " " + Y['crn'] + "\t" + Y['teacher']
+                print(Y)
+                #outfile.write("\t" + Y['code'] + " " + Y['crn'] + "\t" + Y['teacher'] + "\t" + Y['type'] +"\n")
+                outfile.write("\t" + Y['code'] + " " + Y['crn'] + "\t" + Y['teacher'] + "\t" + Y['type'] + "\t" + "\n")
+
+
+# All students enrolled in a class in the given semester. Simpler verson of below. Return SET of course_ids.
+def users_in_semester():
+    all_c = getCoursesInTerm('65',0,0)   # fall 2020 TODO
+    all_s = set()
+    for c in all_c:
+        for u in course_enrollment(c['id']).values():
+            if u['type'] != "StudentEnrollment": continue
+            all_s.add(u['id'])
+    return all_s
+    
+
+#
+# All students in STEM (or any list of depts.. match the course_code). Return SET of canvas ids.
+def users_in_depts_live(depts=[], termid='171'):
+    courses_by_dept = {}
+    students_by_dept = {}
+
+    all_c = getCoursesInTerm(termid,0,0)  
+    codecs.open('cache/courses_in_term_%s.json' % termid,'w','utf-8').write( json.dumps(all_c,indent=2) )
+    for c in all_c:
+        #print(c['course_code'])
+        for d in depts:
+            #print("Dept: %s" % d)
+            match = re.search('^(%s)' % d, c['course_code'])
+            if match:
+                print("Getting enrollments for %s" % c['course_code'])
+                if d in courses_by_dept: courses_by_dept[d].append(c)
+                else: courses_by_dept[d] = [ c, ]
+                for u in course_enrollment(c['id']).values():
+                    if u['type'] != "StudentEnrollment": continue
+                    if not (d in students_by_dept):
+                        students_by_dept[d] = set() 
+                    students_by_dept[d].add(u['user_id'])
+                continue
+    print(students_by_dept)
+    codecs.open('cache/students_by_dept_in_term_%s.json' % termid,'w','utf-8').write( str(students_by_dept) )
+    all_students = set()
+    for dd in students_by_dept.values(): all_students.update(dd)
+    codecs.open('cache/all_students_in_depts_in_term_%s.json' % termid,'w','utf-8').write( str(all_students) )
+    return all_students
+
+
+                
+def course_enrollment(id=''):
+    print("Getting enrollments for course id %s" % str(id))
+    if not id:
+        id = input('Course id? ')
+    t = url + '/api/v1/courses/%s/enrollments?role[]=StudentEnrollment' % str(id)
+    print(t)
+    emts = fetch(t,0)
+    print(emts)
+    emt_by_id = {}
+    for E in emts:
+        print(E)
+        try:
+            emt_by_id[E['user_id']] = E
+        except Exception as exp:
+            print("Skipped that class with this exception: %s" % str(exp))
+    ff = codecs.open('cache/courses/%s.json' % str(id), 'w', 'utf-8')
+    ff.write(json.dumps(emt_by_id, indent=2))
+    print( "  %i results" % len(emts) )
+    return emt_by_id
+
+
+def askForTerms():
+    user_input = input("The term id? (separate multiples with commas) ")
+    return user_input.split(",")
+
+"""
+    names = []
+    if not term:
+        s = url + '/api/v1/accounts/1/terms?workflow_state[]=all'
+        s = fetch_collapse(s,"enrollment_terms",1)
+        print(json.dumps(s,indent=2))
+        print("Terms: ")
+        for u in s:
+            print(str(u['id']) + "\t" + u['name'])
+        #print json.dumps(results_dict,indent=2)
+        term = input("The term id? ")
+"""
+
+
+
+# Return a list of term names and IDs. Also store in cache/courses/terms.txt
+def getTerms(printme=1, ask=1):
+    s = url + '/api/v1/accounts/1/terms' #?workflow_state[]=all'
+    terms = fetch_collapse(s,'enrollment_terms')
+    ff = codecs.open('cache/courses/terms.txt', 'w', 'utf-8')  # TODO unsafe overwrite
+    #print(terms)
+    ff.write(json.dumps(terms, indent=2))
+    ff.close()
+
+    if printme:
+        print("Terms: ")
+        for u in terms:
+            print(str(u['id']) + "\t" + u['name'])
+    if ask:
+        return input("The term id? ")
+    return terms
+
+def getCourses(x=0):   # a dict
+    if not x:
+        user_input = input("The Course IDs to get? (separate with spaces: ")
+        courselist = list(map(int, user_input.split()))
+    else:
+        courselist = [x, ]
+        
+    for id in courselist:
+        t = url + '/api/v1/courses/' + str(id) # + '?perpage=100'
+        t = fetch(t,1)
+    print(t)
+    return t
+
+
+def update_course_conclude(courseid="13590",enddate='2021-12-23T01:00Z'):
+    (connection,cursor) = db()
+    q = "SELECT * FROM courses AS c WHERE c.code LIKE '%FA21%' AND c.conclude='2021-08-29 07:00:00.000'"
+    result = cursor.execute(q)
+    for R in result:
+        try:
+            #print(R)
+            print('doing course: %s' % R[6])
+            courseid = R[1]
+            #d = getCourses(courseid)
+            #print("\tconclude on: %s" % d['end_at'])
+            
+            data = { 'course[end_at]': enddate } 
+            t = url + '/api/v1/courses/' + str(courseid)
+            r3 = requests.put(t, headers=header, params=data)
+            #print("   " + r3.text)
+        except Exception as e:
+            print('****%s' % str(e))
+        
+# Relevant stuff trying to see if its even being used or not
+def course_term_summary_local(term="176",term_label="FA22"):
+    O = "\t<li>Course: <a href='%s' target='_blank' class='%s'>%s</a><br />Status: <b>%s</b><br />Teacher: %s<br />Number students: %s</li>\n"
+    courses = get_courses_in_term_local(term)
+    oo = codecs.open('cache/semester_summary.html','w','utf-8')
+    oo.write('<style>.a{background-color:yellow;}.b{background-color:pink;}</style><ul>\n')
+    
+    for C in sorted(courses):
+        style = ''
+        info = course_quick_stats(C[3])
+        sinfo = course_student_stats(C[3])
+        D = list(C)
+        D.append(info)
+        D.append(sinfo)
+        #print(D)
+        if D[6][0][0] == 0: continue
+        if D[2] == 'claimed': style="a"
+        mystr = O % ( "https://ilearn.gavilan.edu/courses/"+str(D[3]), style, D[1], D[2], str(', '.join(D[5])), str(D[6][0][0]))
+        print(D[1])
+        oo.write(mystr )
+        oo.flush()
+        #print(info)
+    oo.write('\n</ul>\n')
+
+# Relevant stuff trying to see if its even being used or not
+def course_term_summary(term="176",term_label="FA22"):
+    print("Summary of %s" % term_label)
+    courses = getCoursesInTerm(term,0,0)
+
+    print("output to cache/term_summary.txt")
+    outp = codecs.open('cache/term_summary.txt','w','utf-8')
+    
+    tup = tuple("id course_code default_view workflow_state".split(" "))
+    smaller = [ funcy.project(x , tup) for x in courses ]
+    #print(json.dumps(smaller, indent=2))
+    by_code = {}
+    (connection,cursor) = db()
+    (pub, not_pub) = funcy.split( lambda x: x['workflow_state'] == "available", smaller)
+
+    for S in smaller:
+        print(S)
+        by_code[ S['course_code'] ] = str(S) + "\n"
+        outp.write( str(S) + "\n" )
+        q = """SELECT c.id AS courseid, c.code, tt.name, c.state, COUNT(u.id) AS student_count  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+JOIN ( SELECT c.id AS courseid, u.id AS userid, c.code, u.name  FROM courses AS c
+	JOIN enrollment AS e ON e.course_id=c.id
+	JOIN users AS u ON u.id=e.user_id
+	WHERE c.canvasid=%s
+	AND e."type"="TeacherEnrollment" ) AS tt ON c.id=tt.courseid
+WHERE c.canvasid=%s
+AND e."type"="StudentEnrollment"
+GROUP BY c.code ORDER BY c.state, c.code""" % (S['id'],S['id'])
+        result = cursor.execute(q)
+        for R in result:
+            print(R)
+            by_code[ S['course_code'] ] += str(R) + "\n"
+            outp.write( str(R) + "\n\n" )
+            pages = fetch(url + "/api/v1/courses/%s/pages" % S['id'])
+            by_code[ S['course_code'] ] += json.dumps(pages, indent=2) + "\n\n"
+            modules = fetch(url + "/api/v1/courses/%s/modules" % S['id'])
+            by_code[ S['course_code'] ] += json.dumps(modules, indent=2) + "\n\n"
+
+        print()
+
+    out2 = codecs.open('cache/summary2.txt','w', 'utf-8')
+
+    for K in sorted(by_code.keys()):
+        out2.write('\n------ ' + K + '\n' + by_code[K])
+        out2.flush()
+
+    return
+
+    #published = list(funcy.where( smaller, workflow_state="available" ))
+    #notpub = list(filter( lambda x: x['workflow_state'] != "available", smaller))
+    notpub_ids = [ x['id'] for x in notpub ]
+
+    #for ix in notpub_ids:
+    #    # print(course_quick_stats(ix))
+
+
+    outp.write(json.dumps(courses, indent=2))
+
+    outp2 = codecs.open('cache/term_summary_pub.txt','w','utf-8')
+    outp2.write("PUBLISHED\n\n" + json.dumps(published, indent=2))
+    outp2.write("\n\n---------\nNOT PUBLISHED\n\n" + json.dumps(notpub, indent=2))
+
+# Fetch all courses in a given term
+def getCoursesInTerm(term=0,get_fresh=1,show=0,active=0):        # a list
+    if not term:
+        term = getTerms(1,1)
+    ff = 'cache/courses_in_term_%s.json' % str(term)
+    if not get_fresh:
+        if os.path.isfile(ff):
+            return json.loads(  codecs.open(ff,'r','utf-8').read()  )
+        else:
+            print(" -> couldn't find cached classes at: %s" % ff)
+
+    # https://gavilan.instructure.com:443/api/v1/accounts/1/courses?published=true&enrollment_term_id=11
+    names = []
+    if active:
+        active = "published=true&"
+    else:
+        active = ""
+    t = url + '/api/v1/accounts/1/courses?' + active + 'enrollment_term_id=' + str(term) #+ '&perpage=30'
+    results = fetch(t,show)
+    if show:
+        for R in results:
+            try:
+                print(str(R['id']) + "\t" + R['name'])
+            except Exception as e:
+                print("Caused a problem: ") 
+                print(R)
+        #print json.dumps(results,indent=2)
+    info = []
+    for a in results:
+        names.append(a['name'])
+        info.append( [a['id'], a['name'], a['workflow_state'] ] )
+    if show: print_table(info) 
+    codecs.open(ff, 'w', 'utf-8').write(json.dumps(results,indent=2))
+    return results
+
+
+def getCoursesTermSearch(term=0,search='',v=0):
+    term =  term or input("term id? ")
+    search =  search or input("What to search for? ")
+    
+    s = url + '/api/v1/accounts/1/courses?enrollment_term_id=%s&search_term=%s' % (  str(term) , search )
+    if v: print(s)
+    
+    courses = fetch(s)
+    if v: print(json.dumps(courses,indent=2))
+    return courses
+
+def courseLineSummary(c,sections={}):
+    ss = "\t"
+    crn = "\t"
+    host = ""
+    if 'crn' in c:
+        crn = "crn: %s\t" % c['crn']
+        
+    if c['id'] in sections:
+        ss = "section: %s\t" % str(sections[c['id']])
+    
+    if 'host' in c:
+        host = "send to crn: %s\t" % c['host']
+        
+    out = "%i\t%s%s%s%s" % (c['id'], ss ,crn, host, c['name'])
+    return out
+    
+def xlistLineSummary(c,sections={}):
+    # can_id  incoming_sec_id  crn  name
+    
+    new_sec = "missing"
+    if 'partner' in c and 'sectionid' in c['partner']:
+        new_sec = c['partner']['sectionid']
+    
+    out = "can_id:%i\t new_sec_id:%s\t crn:%s\t %s" % (c['id'], new_sec ,c['crn'], c['name'])
+    return out
+
+def numbers_in_common(L):
+    # how many leading numbers do the strings in L share?
+    for i in [0,1,2,3,4]:
+        number = L[0][i]
+        for s in L:
+            #print("%s -> %s" % (number,s[i]))
+            if s[i] != number: return i
+    return 5
+
+def combined_name(nic,L):
+    # string with prettier section numbers combined
+    if len(L) < 2:
+        return L[0]
+    if nic < 2:
+        return "/".join(L)
+    L_mod = [ x[nic:6] for x in L]
+    L_mod[0] = L[0]
+    new_name = "/".join(L_mod)
+    #print(nic, " ", L_mod)
+    return new_name
+    
+
+def semester_cross_lister():
+    checkfile = codecs.open('cache/xlist_check.html','w','utf-8')
+    checkfile.write('<html><body><table>\n')
+    
+    current_term = '178'
+    xlistfile = codecs.open('cache/sp23_crosslist.csv','r','utf-8').readlines()[1:]
+    by_section = {}
+    by_group = defaultdict( list )
+    crn_to_canvasid = {}
+    crn_to_canvasname = {}
+    crn_to_canvascode = {}
+    
+    get_fresh = 0
+    if get_fresh:
+        c = getCoursesInTerm(178,0,0)    # sp23
+        codecs.open('cache/courses_in_term_178.json','w','utf-8').write(json.dumps(c,indent=2))
+    else:
+        c = json.loads(  codecs.open('cache/courses_in_term_178.json','r','utf-8').read()  )
+    
+    for C in c:
+        if 'sis_course_id' in C and C['sis_course_id']:
+            crn_to_canvasid[C['sis_course_id'][7:13]] = str(C['id'])
+            crn_to_canvasname[C['sis_course_id'][7:13]] = str(C['name'])
+            crn_to_canvascode[C['sis_course_id'][7:13]] = str(C['course_code'])
+    # "Term","PrtTerm","xlstGroup","Subject","CrseNo","EffectCrseTitle","CRN","Session","SecSchdType","AttnMeth","MtgSchdType","MtgType","MaxEnroll","TotalEnroll","SeatsAvail","Bldg","Room","Units","LecHrs","LabHrs","HrsPerDay","HrsPerWk","TotalHrs","Days","D/E","Wks","BegTime","EndTime","StartDate","EndDate","LastName","FirstName","PercentResp"
+    for xc in xlistfile:
+        parts = xc.split(r',')   
+        course = parts[3] + " " + parts[4]
+        group = parts[2]
+        crn = parts[6]
+        
+        if crn in crn_to_canvasid:
+            cid = crn_to_canvasid[crn]
+            oldname = crn_to_canvasname[crn]
+            oldcode = crn_to_canvascode[crn]
+        else:
+            print("! Not seeing crn %s in canvas semester" % crn)
+            cid = ''
+            oldname = ''
+            oldcode = ''
+        
+        if crn in by_section: continue
+        by_section[crn] = [crn, course, group, cid, oldname, oldcode]
+        by_group[group].append( [crn, course, group, cid, oldname, oldcode] )
+    
+    for x in by_section.values():
+        print(x)
+        href = '<a target="_blank" href="%s">%s</a>' % ('https://ilearn.gavilan.edu/courses/'+x[3]+'/settings#tab-details', x[3])
+        checkfile.write('<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % (x[0],x[2],x[1],href) )
+    checkfile.write('</table></body></html>')
+    
+    print("GROUPS")
+    for y in by_group.keys():
+        sects = [ z[0] for z in by_group[y] ]
+        sects.sort()
+        nic = numbers_in_common(sects)
+        new_sec = combined_name(nic,sects)
+        new_name = by_group[y][0][4][0:-5] + new_sec
+        new_code = by_group[y][0][5][0:-5] + new_sec
+        print(y)
+        print("\t", sects)
+        #print("\tThey share %i leading numbers" % nic)
+        print("\t", by_group[y])
+        print("\t", new_name)
+        print()
+        
+        host_id = by_group[y][0][3]
+        sections = by_group[y][1:]
+        
+        for target_section in sections:
+            xlist_ii(target_section[3],host_id,new_name,new_code)
+
+
+def xlist_ii(parasite_id,host_id,new_name,new_code):
+    print("Parasite id: ",parasite_id,"  Host id: ", host_id)
+    print("New name: ", new_name)
+    xyz = input("Perform cross list? Enter for yes, n for no: ")
+    if xyz != 'n':
+        uu = url + '/api/v1/courses/%s/sections' % parasite_id
+        c_sect = fetch(uu)
+        #print(json.dumps(c_sect,indent=2))
+        if len(c_sect) > 1:
+            print("* * * * Already Crosslisted!!")
+            return
+        if not c_sect:
+            print("* * * * Already Crosslisted!!")
+            return
+        else:
+            parasite_sxn_id = str(c_sect[0]['id'])
+            print("Parasite section id: ", parasite_sxn_id)
+
+        u = url + "/api/v1/sections/%s/crosslist/%s" % (parasite_sxn_id,host_id)
+        print(u)
+        res = requests.post(u, headers = header)
+        print(res.text)
+
+        u3 = url + "/api/v1/courses/%s" % host_id
+        data = {'course[name]': new_name, 'course[course_code]': new_code}
+        print(data)
+        print(u3)
+        r3 = requests.put(u3, headers=header, params=data)
+        print(r3.text)
+        print("\n\n")
+
+def all_semester_course_sanity_check():
+    c = getCoursesInTerm(178,0,0)    # sp23
+    codecs.open('cache/courses_in_term_178.json','w','utf-8').write(json.dumps(c,indent=2))
+    output = codecs.open('cache/courses_w_sections.csv','w','utf-8')
+    output.write(  ",".join(['what','id','parent_course_id','sis_course_id','name']) + "\n" )
+    output2 = codecs.open('cache/courses_checker.csv','w','utf-8')
+    output2.write(  ",".join(['id','sis_course_id','name','state','students']) + "\n" )
+    i = 0
+    for course in c:
+        u2 = url + '/api/v1/courses/%s?include[]=total_students' % str(course['id'])
+        course['info'] = fetch(u2)
+        #print(json.dumps(course['info'],indent=2))
+        ts = '?'
+        try:
+            ts = course['info']['total_students']
+        except Exception as e:
+            pass
+        info = [ 'course', course['id'], '', course['sis_course_id'], course['name'], course['workflow_state'], ts  ]
+        info = list(map(str,info))
+        info2 = [ course['id'], course['sis_course_id'], course['name'], course['workflow_state'], ts ]
+        info2 = list(map(str,info2))
+        output2.write( ",".join(info2) + "\n" )
+        output2.flush()
+        print(info2)
+        output.write( ",".join(info) + "\n" )
+        #uu = url + '/api/v1/courses/%s/sections' % str(course['id'])
+        #course['sections'] = fetch(uu)
+        #s_info = [ [ 'section', y['id'], y['course_id'], y['sis_course_id'], y['name'], y['total_students'] ] for y in course['sections'] ]
+        #for row in s_info:
+        #    print(row)
+        #    output.write( ",".join( map(str,row) ) + "\n" )
+        output.flush()
+        i += 1
+        if i % 5 == 0:
+            codecs.open('cache/courses_w_sections.json','w','utf-8').write(json.dumps(c,indent=2))
+    codecs.open('cache/courses_w_sections.json','w','utf-8').write(json.dumps(c,indent=2))
+
+
+def eslCrosslister():
+    fives = []
+    sevens = []
+    others = []
+    
+    course_by_crn = {}
+    
+    sections = {}
+    
+    combos =   [ [y.strip() for y in x.split(',') ] for x in open('cache/xcombos.txt','r').readlines() ]
+    
+    combo_checklist = [ 0 for i in range(len(combos)) ]
+                
+    #print("\n\nCombos:")
+    #[ print("%s - %s" % (x[0],x[1])) for x in combos]
+    
+    #return
+    
+    courses = getCoursesTermSearch(62,"ESL",0)
+    
+    for C in courses:
+        ma = re.search( r'(\d{5})', C['name'])
+        if ma:
+            #print("Found Section: %s from course %s" % (ma.group(1), C['name']))
+            C['crn'] = ma.group(1)
+            course_by_crn[C['crn']] = C
+            
+        if C['name'].startswith("ESL5"): fives.append(C)
+        elif C['name'].startswith("ESL7"): sevens.append(C)
+        else: others.append(C)
+    
+    for S in sevens:
+        uu = url + '/api/v1/courses/%i/sections' % S['id']
+        #print(uu)
+        c_sect = fetch(uu)
+        print(".",end='')
+        #print(json.dumps(c_sect,indent=2))
+        if len(c_sect) > 1:
+            print("* * * * Already Crosslisted!!")
+        if c_sect:
+            sections[ S['id'] ] = c_sect[0]['id']
+            S['sectionid'] = c_sect[0]['id']
+        
+        if S['crn']:
+            for i,co in enumerate(combos):
+                if S['crn'] == co[0]:
+                    S['partner'] = co[1]
+                    combo_checklist[i] = 1
+                    course_by_crn[co[1]]['partner'] = S
+                elif S['crn'] == co[1]:
+                    S['partner'] = co[0]
+                    combo_checklist[i] = 1
+                    course_by_crn[co[0]]['partner'] = S
+                    
+        
+    print("Others:")
+    for F in sorted(others, key=lambda x: x['name']):
+        print(courseLineSummary(F))
+    
+    print("\n\nFive hundreds")
+    for F in sorted(fives, key=lambda x: x['name']):
+        print(courseLineSummary(F))
+    
+    print("\n\nSeven hundreds")
+    for F in sorted(sevens, key=lambda x: x['name']):
+        print(courseLineSummary(F,sections))
+    
+    
+    print("\n\nMake a x-list: ")
+    for F in sorted(fives, key=lambda x: x['name']):
+        if 'partner' in F:
+            print(xlistLineSummary(F,sections))
+            if 'partner' in F and 'sectionid' in F['partner']:
+                if not input('ready to crosslist. Are you? Enter "q" to quit. ') == 'q':
+                    xlist( F['partner']['sectionid'], F['id'] )
+                else:
+                    break
+    for i,c in enumerate(combo_checklist):
+        if not c:
+            print("Didn't catch: "+ str(combos[i]))
+            
+def xlist(parasite='', host=''):        # section id     ,    new course id 
+
+    host = host or input("ID number of the HOSTING COURSE? ")
+    if not parasite:
+        parasite = input("ID number of the SECTION to add to above? (or 'q' to quit) ")
+
+    while parasite != 'q':
+        #h_sections = fetch( url + "/api/v1/courses/%s/sections" % str(host))
+        #print(h_sections)
+        
+        p_sections = fetch( url + "/api/v1/courses/%s/sections" % str(parasite))
+        #print(p_sections)
+        parasite_section = p_sections[0]['id']
+        # TODO need to get the section id from each course:
+        # GET /api/v1/courses/:course_id/sections
+        
+        # POST /api/v1/sections/:id/crosslist/:new_course_id
+        #                 SECTION ID (to move)       NEW __COURSE__ ID
+        
+        u = url + "/api/v1/sections/%s/crosslist/%s" % (str(parasite_section),str(host))
+        print(u)
+        res = requests.post(u, headers = header)
+        print(res.text)
+        parasite = input("ID number of the SECTION to add to above? ")
+
+def unenroll_student(courseid,enrolid):
+    t = url + "/api/v1/courses/%s/enrollments/%s" % ( str(courseid), str(enrolid) )
+    data = {"task": "delete" }
+    r4 = requests.delete(t, headers=header, params=data)
+    print(data)
+    
+#def get_enrollments(courseid):
+#    t = url + "/api/v1/courses/%s/enrollments?type=StudentEnrollment" % courseid
+#    return fetch(t,1)
+    
+    
+def enroll_stem_students_live():
+    the_term = '178'   
+    do_removes = 0
+    depts = "MATH BIO CHEM CSIS PHYS PSCI GEOG ASTR ECOL ENVS ENGR".split(" ")
+    users_to_enroll = users_in_depts_live(depts, the_term)      # term id
+    
+    stem_enrollments = course_enrollment(stem_course_id)     # by user_id
+
+    users_in_stem_shell = set( [ x['user_id'] for x in stem_enrollments.values() ])
+
+    print("ALL STEM STUDENTS %s" % str(users_to_enroll))
+    print("\n\nALREADY IN STEM SHELL %s" % str(users_in_stem_shell))
+
+    enroll_us = users_to_enroll.difference(users_in_stem_shell)
+    #enroll_us = users_to_enroll
+    remove_us = users_in_stem_shell.difference(users_to_enroll)
+
+    print("\n\nTO ENROLL %s" % str(enroll_us))
+    (connection,cursor) = db()
+                                    
+    #xyz = input('enter to continue')
+
+    
+    
+    eee = 0
+    uuu = 0
+
+    if do_removes:
+        print("\n\nTO REMOVE %s" % str(remove_us))
+        for j in remove_us:
+            try:
+                q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+                cursor.execute(q)
+                s = cursor.fetchall()
+                if s:
+                    s = s[0]
+                    print("Removing: %s" % s[0])
+                r1 = unenroll_student(str(stem_course_id), stem_enrollments[j]['id'])
+                print(r1)
+                uuu += 1
+                time.sleep(0.600)
+            except Exception as e:
+                print("Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
+
+    for j in enroll_us:
+        try:
+            q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+            cursor.execute(q)
+            s = cursor.fetchall()
+            if s:
+                s = s[0]
+                print("Enrolling: %s" % s[0])
+            enrollment = { }
+            #print(s)
+            t = url + '/api/v1/courses/%s/enrollments' % stem_course_id
+            data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
+                     'enrollment[enrollment_state]': 'active' }
+            #print(data)
+            #if input('enter to enroll %s or q to quit: ' % s[0]) == 'q':
+            #break
+            r3 = requests.post(t, headers=header, params=data)
+            print(data)
+            eee += 0
+            time.sleep(0.600)
+        except Exception as e:
+            print("Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
+        #print(r3.text)
+    print("\n\nTO ENROLL %s" % str(enroll_us))
+    #print("\n\nTO REMOVE %s" % str(remove_us))
+    return (eee,uuu)
+
+
+
+###########################
+
+def enroll_bulk_students_bydept(course_id, depts, the_term="172", cautious=1):     #  a string,  a list of strings
+    users_to_enroll = users_in_depts_live(depts, the_term)                         #  term id
+    
+    targeted_enrollments = course_enrollment(course_id)                            #  by user_id.. (live, uses api)
+
+    current_enrollments = set( [ x['user_id'] for x in targeted_enrollments.values() ])
+
+    print("ALL TARGET STUDENTS %s" % str(users_to_enroll))
+    print("\nALREADY IN SHELL %s" % str(current_enrollments))
+
+    enroll_us = users_to_enroll.difference(current_enrollments)
+    remove_us = current_enrollments.difference(users_to_enroll)
+
+    print("\n\nTO ENROLL %s" % str(enroll_us))
+    xyz = input('enter to continue')
+    print("\n\nTO REMOVE %s" % str(remove_us))
+
+    (connection,cursor) = db()
+
+
+    for j in remove_us:
+        try:
+            q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+            cursor.execute(q)
+            s = cursor.fetchall()
+            if s:
+                s = s[0]
+                print("Removing: %s" % s[0])
+            r1 = unenroll_student(str(course_id), stem_enrollments[j]['id'])
+            #print(r1)
+            time.sleep(0.600)
+        except Exception as e:
+            print("Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
+
+    for j in enroll_us:
+        try:
+            q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+            cursor.execute(q)
+            s = cursor.fetchall()
+            if s:
+                s = s[0]
+                print("Enrolling: %s" % s[0])
+            enrollment = { }
+            #print(s)
+            t = url + '/api/v1/courses/%s/enrollments' % course_id
+            data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
+                     'enrollment[enrollment_state]': 'active' }
+            
+            if cautious:
+                print(t)
+                print(data)
+                prompt = input('enter to enroll %s, k to go ahead with everyone, or q to quit: ' % s[0]) 
+                if prompt == 'q':
+                    break
+                elif prompt == 'k':
+                    cautious = 0
+            r3 = requests.post(t, headers=header, params=data)
+            if cautious:
+                print(data)
+            time.sleep(0.600)
+        except Exception as e:
+            print("Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
+        #print(r3.text)
+
+
+
+def enroll_art_students_live():
+    depts = "THEA ART DM MUS MCTV".split(" ")
+    course_id = "13717"
+    enroll_bulk_students_bydept(course_id,depts)
+    print("done.")
+
+def enroll_orientation_students():
+    ori_shell_id = "15924"    #   2023 orientation shell   #      2022: "9768"
+    the_semester = "202330"
+
+    users_to_enroll = users_new_this_semester(the_semester)             ###               #####   USES LOCAL DB
+    users_in_ori_shell = set( \
+            [ str(x['user_id']) for x in course_enrollment(ori_shell_id).values() ])
+
+    print("ALL ORIENTATION STUDENTS %s" % str(users_to_enroll))
+    print("\n\nALREADY IN ORI SHELL %s" % str(users_in_ori_shell))
+
+    enroll_us = users_to_enroll.difference(users_in_ori_shell)
+
+    print("\n\nTO ENROLL %s" % str(enroll_us))
+    print("%i new users to enroll." % len(enroll_us))
+
+    eee = 0
+    uuu = 0
+    
+    (connection,cursor) = db()
+
+    for j in enroll_us:
+        s = ""
+        try:
+            q = "SELECT name,canvasid FROM users WHERE canvasid=%s" % j
+            cursor.execute(q)
+            s = cursor.fetchall()
+            if s:
+                s = s[0]
+                print(" + Enrolling: %s" % s[0])
+            t = url + '/api/v1/courses/%s/enrollments' % ori_shell_id
+            data = { 'enrollment[user_id]': j, 'enrollment[type]':'StudentEnrollment',
+                     'enrollment[enrollment_state]': 'active' }
+            #print(data)
+            r3 = requests.post(t, headers=header, params=data)
+            eee += 1
+            #print(r3.text)
+            time.sleep(0.600)
+        except Exception as e:
+            print(" - Something went wrong with id %s, %s, %s" % (j, str(s), str(e)))
+    return (eee,uuu)
+
+def enroll_o_s_students():
+    #full_reload()
+    
+    (es,us) = enroll_stem_students_live()
+    (eo, uo) = enroll_orientation_students()
+    
+    print("Enrolled %i and unenrolled %i students in STEM shell" % (es,us))
+    print("Enrolled %i students in Orientation shell" % eo)
+    
+    
+##########
+########## CALCULATING SEMESTER STUFF
+##########
+
+
+def summarize_proportion_online_classes(u):
+    # u is a "group" from the groupby fxn
+    #print u
+    if NUM_ONLY:
+        if ((1.0 * u.sum()) / u.size) > 0.85: return '2'
+        if ((1.0 * u.sum()) / u.size) < 0.15: return '0'
+        return '1'
+    else:
+        if ((1.0 * u.sum()) / u.size) > 0.85: return 'online-only'
+        if ((1.0 * u.sum()) / u.size) < 0.15: return 'f2f-only'
+        return 'mixed'
+    
+def summarize_num_term_classes(u):
+    # u is a "group" from the groupby fxn
+    # term is sp18 now
+    #print u
+    return u.size
+
+
+
+
+def make_ztc_list(sem='sp20'):
+    sched = json.loads(open('output/semesters/2020spring/sp20_sched.json','r').read())
+    responses = open('cache/ztc_responses_sp20.csv','r').readlines()[1:]
+    
+    result = open('cache/ztc_crossref.csv','w')
+    result.write('Course,Section,Name,Teacher,ZTC teacher\n')
+    
+    ztc_dict = {}
+    for R in responses:
+        R = re.sub(',Yes','',R)
+        R = re.sub('\s\s+',',',R)
+        
+        parts = R.split(r',')     #name   courselist    yes
+        #print(parts[1])
+        name = parts[0]
+        
+        for C in parts[1:] :
+            C = C.strip()
+            #print(C)
+            if C in ztc_dict:
+                ztc_dict[C] += ', ' + parts[0]
+            else:
+                ztc_dict[C] = parts[0]
+    print(ztc_dict)
+    for CO in sched:
+        #if re.match(r'CWE',CO['code']):
+            #print(CO)
+            
+        if CO['code'] in ztc_dict:
+            print(('Possible match, ' + CO['code'] + ' ' + ztc_dict[CO['code']] + ' is ztc, this section taught by: ' + CO['teacher'] ))
+            result.write(   ','.join( [CO['code']  ,CO['crn']  ,  CO['name']  ,  CO['teacher']  ,  ztc_dict[CO['code']]  ])  + "\n" )
+
+def course_search_by_sis():
+    term = 65    
+    all_courses = getCoursesInTerm(term)
+    all = []
+    for course in all_courses: 
+        #u = "/api/v1/accounts/1/courses/%s" % course_id
+        #i = fetch( url + u)
+        all.append([ course['name'], course['sis_course_id'] ])
+    print_table(all)
+        # print(json.dumps(x, indent=2))
+
+
+def mod_eval_visibility( shell_id, visible=True ):
+    evals_hidden = True
+    if (visible): evals_hidden = False
+    data = {'position':2, 'hidden':evals_hidden}
+    u2 = "https://gavilan.instructure.com:443/api/v1/courses/%s/tabs/context_external_tool_1953" % shell_id
+    r3 = requests.put(u2, headers=header, params=data)
+    #print("   " + r3.text)
+    
+
+
+def instructor_list_to_activate_evals():
+    courses = all_sem_courses_teachers()
+    
+    mylist = codecs.open('cache/fa21_eval_teachers.txt','r','utf-8').readlines()
+    mylist = [ x.split(',')[2].strip() for x in mylist ]
+    
+    count = 0
+    limit = 5000
+    
+    for c in courses:
+        shell_id = c[1]
+        teacher_id = c[6]
+        teacher_name = c[5]
+        course_name = c[3]
+        
+        if teacher_id in mylist:
+            print("Teacher: %s \t course: %s" % (teacher_name,course_name))
+            mod_eval_visibility( shell_id, False)
+            count += 1
+            if count > limit: return
+            
+            
+    #print(mylist)
+
+
+    
+def add_evals(section=0):
+    # show or hide?
+    hidden = True
+    #s = [ x.strip() for x in codecs.open('cache/sp21_eval_sections.txt','r').readlines()]
+    #s = [ x.split(',')[4].split('::') for x in codecs.open('cache/fa22_eval_sections.csv','r').readlines()]
+    s = [ x.strip() for x in codecs.open('cache/fa22_eval_sections.csv','r').readlines()]
+    print(s)
+    s = list(funcy.flatten(s))
+    s.sort()
+    xyz = input('hit return to continue')
+    
+    #c = getCoursesInTerm(168,0,1)
+    #c = getCoursesInTerm(174,0,1)   # sp22
+    c = getCoursesInTerm(176,0,1)   # fa22
+    print(c)
+    ids = []
+    courses = {}
+    for C in c:
+        if C and 'sis_course_id' in C and C['sis_course_id']:
+            parts = C['sis_course_id'].split('-')
+            if parts[1] in s:
+                print(C['name'])
+                courses[str(C['id'])] = C
+                ids.append(str(C['id']))
+    
+    ask = 0
+    data = {'position':2, 'hidden':hidden}
+    
+    for i in ids:
+        if ask:
+            a = input("Hit q to quit, a to do all, or enter to activate eval for: " + str(courses[i]))
+            if a == 'a': ask = 0
+            if a == 'q': return
+        u2 = "https://gavilan.instructure.com:443/api/v1/courses/%s/tabs/context_external_tool_1953" % i
+        r3 = requests.put(u2, headers=header, params=data)
+        print(r3)
+        time.sleep(0.600)
+        
+        
+    return 1
+    
+    u2 = "https://gavilan.instructure.com:443/api/v1/courses/12001/tabs"
+    r = fetch(u2)
+    print(json.dumps(r,indent=2))
+    
+    
+    
+    # PUT /api/v1/courses/:course_id/tabs/:tab_id
+
+def course_dates_terms(section=0):
+    """s = [ x.strip() for x in codecs.open('cache/fa22_eval_sections.csv','r').readlines()]
+    s = list(funcy.flatten(s))
+    s.sort()
+    xyz = input('hit return to continue')
+    """
+    
+    #c = getCoursesInTerm(168,0,1)
+    #c = getCoursesInTerm(174,0,1)   # sp22
+    #c = getCoursesInTerm(176,0,1)   # fa22
+    
+    get_fresh = 0
+    
+    if get_fresh:
+        c = getCoursesInTerm(178,0,0)    # sp23
+        codecs.open('cache/courses_in_term_178.json','w','utf-8').write(json.dumps(c,indent=2))
+    else:
+        c = json.loads(  codecs.open('cache/courses_in_term_178.json','r','utf-8').read()  )
+    
+    crn_to_canvasid = {}
+    for C in c:
+        #print(C['name'])
+        if 'sis_course_id' in C and C['sis_course_id']:
+            crn_to_canvasid[C['sis_course_id'][7:13]] = str(C['id'])
+    
+    #print(crn_to_canvasid)
+    #return
+    
+    s = json.loads(  codecs.open('cache/sp23_sched_expanded.json','r','utf-8').read()  )
+    for S in s:
+        start = re.sub( r'\-','/', S['start']) + '/2023'
+        d_start = datetime.strptime(start,"%m/%d/%Y")
+        
+        if d_start.month > 5:
+            print("Ignoring ", d_start, " starting too late...")
+            continue
+        
+        if d_start.month == 1 and d_start.day == 12:
+            print("- Aviation ", start, d_start, " - ", S['code'], " ", S['crn'] )
+            continue
+            
+        if d_start.month == 1 and d_start.day ==3:
+            print("+ winter session: ", d_start, " - ", S['code'])
+            winter_term = '177'
+            data = {'course[term_id]':winter_term}
+            u2 = "https://gavilan.instructure.com:443/api/v1/courses/%s" % crn_to_canvasid[S['crn']]
+            r3 = requests.put(u2, headers=header, params=data)
+            print(u2, "   OK")
+            #print(r3.text)
+            continue
+            
+        if d_start.month == 1 and d_start.day == 30:
+            # normal class
+            continue
+            
+        print("- Late start? ", start, d_start, " - ", S['code'], " ", S['crn'] )
+        data = {'course[start_at]':d_start.isoformat(), 'course[restrict_enrollments_to_course_dates]': True}
+        u2 = "https://gavilan.instructure.com:443/api/v1/courses/%s" % crn_to_canvasid[S['crn']]
+        r3 = requests.put(u2, headers=header, params=data)
+        print(u2, "   OK")
+    
+    return
+    
+    
+
+def remove_n_analytics(section=0):
+    print("Fetching list of all active courses")
+    
+    c = getCoursesInTerm(172,1,0)
+    print(c)
+    ids = []
+    courses = {}
+    data = {'hidden':True}
+    
+    pause = 1
+    
+    for C in c:
+        #print( json.dumps(C,indent=2) ) 
+        parts = C['sis_course_id'].split('-')
+        #print("\n")
+        print(C['name'])
+        courses[str(C['id'])] = C
+        ids.append(str(C['id']))
+        
+        u3 = url + '/api/v1/courses/%s/tabs' % str(C['id'])
+        tabs = fetch(u3)
+        for T in tabs:
+            if T['label'] == "New Analytics":
+                print( "\tVisibility is: " + T["visibility"] )    # json.dumps(tabs,indent=2) ) 
+                if "hidden" in T:
+                    print( "\tHidden is: " + str(T["hidden"]) )    # json.dumps(tabs,indent=2) ) 
+                if 1: # T["visibility"] != "admins":
+                    u4 = url + "/api/v1/courses/%s/tabs/%s"    %    ( str(C['id']), str(T['id']) )
+                    print( "\tChanging visiblity of a. tab" )
+                    r4 = requests.put(u4, headers=header, params=data)
+                    print("\t" + r4.text)
+                    if pause:
+                        xyz = input('\n\nenter for next one or [y] to do all: ')
+                        if xyz == 'y': pause = 0
+    
+    
+    exit()
+    
+    
+    """ask = 1
+    
+    evals_hidden = True
+    
+    
+    data = {'position':2, 'hidden':evals_hidden}
+    
+    for i in ids:
+        if ask:
+            a = input("Hit q to quit, a to do all, or enter to activate eval for: \n " + str(courses[i]) + "\n> ")
+            if a == 'a': ask = 0
+            if a == 'q': return
+        u2 = "https://gavilan.instructure.com:443/api/v1/courses/%s/tabs/context_external_tool_1953" % i
+        print(courses[i]['name'])
+        r3 = requests.put(u2, headers=header, params=data)
+        print("   " + r3.text)
+        time.sleep(0.300)
+    """
+    
+    
+
+def create_sandboxes():
+    names = input("what are the initials of people? Separate with spaces  ").split()
+    for N in names: 
+        print(N)
+        u2 = url + "/api/v1/accounts/1/courses"
+        data = { 
+            "course[name]": "%s Sandbox SU21 G2" % N,
+            "course[code]": "%s SU21 G2" % N,
+            "course[term_id]": "8",
+        }
+        #print(u2)
+        r3 = requests.post(u2, headers=header, params=data)
+        course_data = json.loads(r3.text)
+        id = course_data['id']
+        u3 = url + "/api/v1/courses/%i/enrollments" % id
+        usrid = input("id of %s? " % N)
+        data2 = { "enrollment[type]":"TeacherEnrollment", "enrollment[user_id]":usrid}
+        r4 = requests.post(u3, headers=header, params=data2)
+        #print(json.dumps(json.loads(r4.text),indent=2))
+        print()
+    
+
+def course_term_summary_2():
+    lines = codecs.open('cache/term_summary.txt','r','utf-8').readlines()
+    output = codecs.open('cache/term_summary.html','w','utf-8')
+    for L in lines:
+        try:
+            L = L.strip()
+            print(L)
+            ll = json.loads(L)
+            print(ll)
+            print(ll['course_code'])
+            if ll['workflow_state'] == 'unpublished':
+                ss = "<br />Course: <a href='%s' target='_blank'>%s</a><br />" % ("https://ilearn.gavilan.edu/courses/"+str(ll['id']), ll['course_code'] )
+                output.write( ss )
+                print(ss+"\n")
+        except Exception as e:
+            print(e)
+
+def get_ext_tools():
+    r = url + '/api/v1/accounts/1/external_tools'
+    s = fetch(r)
+    print(json.dumps(s,indent=2))
+
+def set_ext_tools():
+    TOOL = 733
+    r = url + '/api/v1/accounts/1/external_tools/%s' % str(TOOL)
+    data = { 'course_navigation[default]': 'disabled' }
+    s = json.loads(requests.put(r, headers=header, params=data).text)
+    print(json.dumps(s,indent=2))
+
+    
+if __name__ == "__main__":
+    options = { 1: ['Cross check schedule with ztc responses',make_ztc_list] ,  
+                30: ['List latestart classes', list_latestarts ],
+                2: ['Add announcements to homepage', change_course_ann_homepage],
+                3: ['Cross-list classes', xlist ],
+                4: ['List students who passed quiz X', get_quiz_passers],
+                5: ['List the terms', getTerms],
+                6: ['Cross list helper', eslCrosslister],
+                7: ['Show courses in a term', getCoursesInTerm],
+                8: ['Save enrollments in a course', course_enrollment],
+                9: ['Simple list of course data, search by sis_id', course_search_by_sis],
+                10: ['Overview of a term', course_term_summary],
+                11: ['Enroll ORIENTATION and STEM student shells after catching up database.', enroll_o_s_students],
+                12: ['Enroll stem students', enroll_stem_students_live],
+                13: ['Enroll orientation students (refresh local db)', enroll_orientation_students],
+                14: ['Enroll ART students', enroll_art_students_live],
+                15: ['List users who passed GOTT 1 / Bootcamp', get_gott1_passers],
+                16: ['List users who passed Plagiarism Module', get_plague_passers],
+                17: ['Remove "new analytics" from all courses navs in a semester', remove_n_analytics],
+                18: ['Create some sandbox courses', create_sandboxes],
+                19: ['Add course evals', add_evals],
+                20: ['process the semester overview output (10)', course_term_summary_2],
+                21: ['Add announcements to homepage', change_course_ann_homepage],
+                22: ['Get a course info by id',getCourses],
+                23: ['Reset course conclude date',update_course_conclude],   
+                #24: ['Add course evals to whole semester',instructor_list_to_activate_evals],   
+                25: ['ext tools',get_ext_tools],   
+                26: ['set ext tools',set_ext_tools],   
+                27: ['Fine tune term dates and winter session', course_dates_terms],
+                28: ['Cross list a semester from file', semester_cross_lister],
+                29: ['Check all courses & their sections in semester', all_semester_course_sanity_check],
+                # TODO wanted: group shell for each GP (guided pathway) as a basic student services gateway....
+                #
+              }
+    print ('')
+
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
diff --git a/cq_demo.py b/cq_demo.py
new file mode 100644
index 0000000..4fb769f
--- /dev/null
+++ b/cq_demo.py
@@ -0,0 +1,48 @@
+import codecs, json, requests
+from secrets import cq_token, ph_token 
+token = cq_token
+url = 'https://ilearn.gavilan.edu' 
+header = {'Authorization': 'Bearer ' + token}
+
+output = codecs.open('cq_gav_test.txt','a','utf-8')
+
+def fetch(target):
+    print("Fetching %s..." % target)
+    try:
+        r2 = requests.get(target, headers = header)
+    except Exception as e:
+        print("-- Failed to get: ", e)
+    try:
+        results = json.loads(r2.text)
+        count = len(results)
+        print("Got %i results" % count)
+        print(json.dumps(results,indent=2))
+        print()
+        output.write("----\nGetting: %s\n" % target)
+        output.write(json.dumps(results,indent=2))
+        output.write("\n\n")
+    except:
+        print("-- Failed to parse: ", r2.text)
+    
+
+
+
+fetch(url + '/api/v1/outcomes/270')
+
+fetch(url + '/api/v1/outcomes/269')
+exit()
+
+
+
+
+fetch(url + '/api/v1/courses/15424/outcome_results')
+
+fetch(url + '/api/v1/courses/15424/outcome_rollups')
+exit()
+
+
+fetch(url + '/api/v1/accounts/1/courses')
+fetch(url + '/api/v1/courses/12820/sections')
+fetch(url + '/api/v1/courses/12820/enrollments')
+
+
diff --git a/credentials.json b/credentials.json
new file mode 100644
index 0000000..14c3c12
--- /dev/null
+++ b/credentials.json
@@ -0,0 +1 @@
+{"installed":{"client_id":"955378242514-m954fg4f0g1n1nb6kckp68ru001hpno0.apps.googleusercontent.com","project_id":"quickstart-1569874764316","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"kSxttNuwitwdCVcQxqNh0dif","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}}
\ No newline at end of file
diff --git a/curric2022.py b/curric2022.py
new file mode 100644
index 0000000..01918f0
--- /dev/null
+++ b/curric2022.py
@@ -0,0 +1,848 @@
+import requests,json,os,re, bisect, csv, codecs, funcy, sys, shutil, time
+from datetime import datetime
+import sortedcontainers as sc
+from collections import defaultdict
+from toolz.itertoolz import groupby,sliding_window
+from sortedcontainers import SortedList
+#from durable.lang import *
+#from durable.engine import *
+from pampy import match, _
+from bs4 import BeautifulSoup as bs
+
+
+leafcount = 0
+displaynames = []
+
+from secrets import cq_user, cq_pasw
+
+
+CQ_URL = "https://secure.curricunet.com/scripts/webservices/generic_meta/clients/versions/v4/gavilan.cfc"
+PARAM = "?returnFormat=json&method=getCourses"
+
+user = cq_user
+pasw = cq_pasw
+
+err_fail_filecount = 1
+
+
+    
+def fetch_all_programs():
+    if os.path.isdir('cache/programs'):
+        m = datetime.strptime(time.ctime(os.path.getctime('cache/programs')), "%a %b %d %H:%M:%S %Y")
+        today = 'cache/programs_%s' % m.strftime('%Y_%m_%d')
+        
+        print("+  Creating folder: %s" % today)
+        shutil.move('cache/programs', today)
+        os.makedirs('cache/programs')
+
+    size = 100
+    endn = 0
+    filen = 1
+    PARAM = "?returnFormat=json&method=getPrograms&status=Active"
+    while(size > 99):
+        size, endn, items = another_request(CQ_URL+PARAM,endn)
+        out = codecs.open('cache/programs/programs_'+str(filen)+'.txt','w', 'utf-8')
+        out.write(json.dumps(items,indent=4))
+        out.close()
+        filen += 1 
+    print("Written to 'cache/programs....")
+
+
+
+
+
+
+
+def nothing(x=0):
+    pass
+
+seen = []
+
+def clean(st):
+    #return st
+    global seen
+    ok = ['b','i','ul','li','ol','strong','br','u']
+    
+    soup = bs(st, features='lxml')
+
+    """for tag in soup.recursiveChildGenerator():
+        if isinstance(tag,bs.Tag) and tag.name not in ok:
+            tag.unwrap()
+    
+    return soup.prettify()
+    """
+    
+    for T in soup.find_all(recursive=True):
+        if not T.name in ok:
+            if not T.name in seen:
+                seen.append(T.name)
+                print("-  %s" % T.name)
+                #print(seen)
+            T.unwrap()
+        else:
+            #print("+ %s" % T.name)
+            pass
+    
+    return str(soup).strip()
+    
+
+
+num_failed_course = 1
+
+def single_course_parse(c):
+    global num_failed_course
+    this_course = []
+    if "attributes" in c and "entityId" in c["attributes"]:
+        print(c["attributes"]["entityId"])
+        return (c["attributes"]["entityId"], recur_matcher(c))
+    else:
+        print("I couldn't recognize a class in that")
+        ooops = codecs.open('cache/programs/failedcourse_%i.json' % num_failed_course, 'w', 'utf-8')
+        ooops.write(json.dumps(c,indent=2))
+        ooops.close()
+        num_failed_course = num_failed_course + 1
+        return ("-1", [])
+        
+def match_style_test():
+    classes = {}
+    oo = codecs.open("cache/courses/curric2022test.json","w","utf-8")
+    for f in os.listdir('cache/courses'):
+        if re.search('classes_',f):
+            print(f)
+            cls = json.loads(codecs.open('cache/courses/'+f,'r','utf-8').read())
+            for c in cls:
+                id,output = single_course_parse(c)
+                classes[id] = "\n".join(output)
+                oo.write( classes[id] )
+                oo.write( "\n\n\n" + "-"*30 + "\n\n" )
+                oo.flush()
+
+
+
+
+
+
+
+
+def single_program_path_parse(c):
+    this_course = []
+    global num_failed_course
+    if "attributes" in c and "entityId" in c["attributes"]:
+        print(c["attributes"]["entityId"])
+        return (c["attributes"]["entityId"], pathstyle(c))
+    else:
+        print("I couldn't recognize a program in that")
+        ooops = codecs.open('cache/programs/failedcourse_%i.json' % num_failed_course, 'w', 'utf-8')
+        ooops.write(json.dumps(c,indent=2))
+        ooops.close()
+        num_failed_course = num_failed_course + 1
+        return ("-1", [])
+        
+
+def path_style_prog():
+    classes = {}
+    oo = codecs.open("cache/programs/allprogrampaths.txt","w","utf-8")
+    for f in os.listdir('cache/programs'):
+        if re.search('^programs_',f):
+            print(f)
+            cls = json.loads(codecs.open('cache/programs/'+f,'r','utf-8').read())
+            for c in cls:
+                id,output = single_program_path_parse(c)
+                classes[id] = "\n".join(output)
+                oo.write( classes[id] )
+                oo.write( "\n\n\n" + "-"*30 + "\n\n" )
+                oo.flush()
+
+def term_txt_to_code(t):
+    term_codes = {'Spring':'30','Summer':'50','Fall':'70'}
+    parts = t.split(" ")
+    if len(parts)>1:
+        yr = parts[1]
+        sem = term_codes[parts[0]]
+        return yr+sem
+    return ''
+
+
+def all_outcomes():
+    csvfile = codecs.open('cache/courses/alloutcomes.csv','w','utf-8')
+    csvwriter = csv.writer(csvfile)
+    csvwriter.writerow('code cqcourseid coursestatus termineffect dept num cqoutcomeid outcome'.split(' '))
+
+    csvfile2 = codecs.open('cache/courses/all_active_outcomes.csv','w','utf-8')
+    csvwriter2 = csv.writer(csvfile2)
+    csvwriter2.writerow('code cqcourseid coursestatus termineffect dept num cqoutcomeid outcome'.split(' '))
+
+    rr = codecs.open("cache/courses/allclasspaths.txt","r", "utf-8").readlines()
+    ww = codecs.open("cache/courses/alloutcomes.txt","w", "utf-8")
+    course_index = []
+
+    current_course = {}
+    current_course_num = 0
+    
+    term_counts = defaultdict(int)
+    
+    count = 0
+
+    for L in rr:
+        a = re.search('Course\/(\d+)',L)
+        if a:
+            course_num = a.group(1)
+            #print(course_num, current_course_num)
+            
+            if (course_num != current_course_num):
+                if  current_course_num != 0:
+                    # log the course info so we can know cq id numbers of courses
+                    course_index.append(current_course)
+                    
+                    # status
+                    count += 1
+                    #input('ok ')
+                    if count % 100 == 0:
+                        print(count)
+                        #pass
+            
+                current_course_num = course_num
+                #print(course_num)
+                current_course = {'c':'','d':'','n':'','t':'','s':'','T':'','o':[],'i':'','a':'','m':''}
+            current_course['c'] = course_num
+        
+        
+        a = re.search('Course\/(\d+)\/1\/Course\ Description\/0\/Course\ Discipline\/(.*)$',L)
+        if a:
+            current_course['d'] = a.group(2)
+        a = re.search('Course\/(\d+)\/1\/Course\ Description\/0\/Course\ Number\/(.*)$',L)
+        if a:
+            current_course['n'] = a.group(2)
+        a = re.search('Course\/(\d+)\/1\/Course\ Description\/0\/Course\ Title\/(.*)$',L)
+        if a:
+            current_course['T'] = a.group(2)
+        a = re.search('Course\/(\d+)\/1\/Course\ Description\/0\/Short\ Title\/(.*)$',L)
+        if a:
+            current_course['t'] = a.group(2)
+        a = re.search('Course\ Description\/status\/(.*)$',L)
+        if a:
+            current_course['s'] = a.group(1)
+        a = re.search('Course\ Content\/\d+\/Lecture\ Content\/Curriculum\ Approval\ Date:\s*(.*)$',L)
+        if a:
+            current_course['a'] = a.group(1)
+        a = re.search('Course\ Description\/\d+\/Internal\ Processing\ Term\/(.*)$',L)
+        if a:
+            t_code = term_txt_to_code(a.group(1))
+            current_course['m'] = t_code
+            term_counts[t_code] += 1
+
+        # Course/10/10/Course Content/1/Lecture Content/Curriculum Approval Date: 02/24/2014
+        
+        # Course/3091/1/Course Description/0/Internal Processing Term/Spring 2018
+
+        a = re.search('Learning\ Outcomes\/\d+\/(cqid_\d+)\/Learning\ Outcomes\/Description\/(.*)$',L)
+        if a:
+            current_course['o'].append(a.group(2))
+            current_course['i'] = a.group(1)
+            csvwriter.writerow([current_course['d']+current_course['n'], current_course_num, current_course['s'], current_course['m'], current_course['d'], current_course['n'], current_course['i'], a.group(2)])
+            if current_course['s']=='Active':
+                csvwriter2.writerow([current_course['d']+current_course['n'], current_course_num, current_course['s'], current_course['m'], current_course['d'], current_course['n'], current_course['i'], a.group(2)])
+            
+            
+        if re.search('Learning\ Outcomes\/Description\/',L):
+            ww.write(L)
+        if re.search('Description\/entityTitle\/',L):
+            ww.write(L)
+        if re.search('Description\/status\/',L):
+            ww.write(L)
+            
+    xx = codecs.open("cache/courses/course_cq_index.json","w", "utf-8")
+    xx.write(json.dumps(course_index, indent=2))
+    
+    #print(json.dumps(term_counts,indent=2))
+
+def ddl():
+    return defaultdict(list)
+
+def splitclassline(cl, id=''):
+    #   "PHYS 4A - Physics for Scientists and Engineers I 4.000  *Active*"
+    dbg = 1
+    ret = {'name':'','units':'','units_hi':'','code':'','status':'', 'sequence':int(id)}
+    p1 = re.search(r'^(.*?)\s\-\s(.*)$',cl)
+    if p1:
+        code = p1.groups()[0]
+        ret['code'] = code
+        rest = p1.groups()[1]
+        
+        p3 = re.search(r'^(.*)\s(\d+\.\d+)\s\-\s(\d+\.\d+)\s+\*(\w+)\*$',rest)
+        if p3:
+            name = p3.groups()[0]
+            units = p3.groups()[1]
+            units_hi = p3.groups()[2]
+            status = p3.groups()[3]
+            ret['name'] = name
+            ret['units'] = units
+            ret['units_hi'] = units_hi
+            ret['status'] = status
+            #if dbg: print( "%s  --- code: %s  - name: %s - units: %s-%s   - status: %s" % (cl,code,name,units,units_hi,status))
+            return ret
+        p2 = re.search(r'^(.*)\s(\d+\.\d+)\s+\*(\w+)\*$',rest)
+        if p2:
+            name = p2.groups()[0]
+            units = p2.groups()[1]
+            status = p2.groups()[2]
+            ret['name'] = name
+            ret['units'] = units
+            ret['status'] = status
+            #if dbg: print( "%s  --- code: %s  - name: %s - units: %s - status: %s" % (cl,code,name,units,status))
+            return ret
+            
+            
+        else:
+            if dbg: print( "%s  --- code: %s  --------------------------------" % (cl,code))
+    else:
+        if dbg: print( "%s  --- code:----------------------------------------" % cl)
+    #return (cl,'','')
+    return ret
+    
+    
+def path_style_2_html():
+    verbose = 1
+    v = verbose
+    
+    prog_title_subs = []
+    with codecs.open('cache/program_published_names.csv', 'r','utf-8') as file:
+        reader = csv.reader(file)
+        for row in reader:
+            prog_title_subs.append(row)
+    
+    
+    oo = codecs.open("cache/programs/allprogrampaths.txt","r","utf-8").readlines()
+    award_prebuild = defaultdict( ddl )
+    last_line = ""
+    
+    for L in oo:
+        L = L.strip()
+        if not re.search(r'^Program',L):
+            last_line = last_line + " " + L
+            continue
+        else:
+            if re.search(r'\/$',last_line):
+                # ignore line with trailing slash - assume no data
+                last_line = L
+                continue
+            
+            if re.search(r'Curriculum\sDivision\s\d+', last_line):
+                #print(last_line)
+                pass
+            
+            test_1 = re.search(r'^Program\/(\d+)\/Course',last_line)
+            if test_1:
+                award_prebuild[  test_1.groups()[0] ]["Info"].append(last_line)
+            test_2 = re.search(r'^Program\/(\d+)\/(\d+)\/([\w\s]+)\/',last_line)
+            if test_2:
+                award_prebuild[  test_2.groups()[0] ][test_2.groups()[2]].append(last_line)
+            last_line = L
+    output  = codecs.open("cache/programs/programs_prebuild.json","w","utf-8")
+    output.write( json.dumps(award_prebuild, indent=2) )
+    
+    
+    award_build = defaultdict( ddl )
+    
+    for AW in sorted(list(award_prebuild.keys()),key=int):
+        v = 1
+        aw = award_prebuild[AW]
+        for line in aw["Program Description"]:
+            t1 = re.search(r'Division\/(.*)$', line)
+            if t1: 
+                award_build[AW]["division"] = t1.groups()[0]
+            t1 = re.search(r'Department\/(.*)$', line)
+            if t1: 
+                award_build[AW]["dept"] = t1.groups()[0]
+            t1 = re.search(r'Program\sTitle\/(.*)$', line)
+            if t1: 
+                award_build[AW]["program_title"] = t1.groups()[0]
+            t1 = re.search(r'Award\sType\/(.*)$', line)
+            if t1: 
+                award_build[AW]["award"] = t1.groups()[0]
+            t1 = re.search(r'\/Description\/(.*)$', line)
+            if t1: 
+                award_build[AW]["description"] = t1.groups()[0]
+            t1 = re.search(r'Transfer\/CTE\/(.*)$', line)
+            if t1: 
+                award_build[AW]["transfer_cte"] = t1.groups()[0]
+            t1 = re.search(r'CTE\sProgram\?\/\/(.*)$', line)
+            if t1: 
+                award_build[AW]["is_cte"] = t1.groups()[0]
+
+        for line in aw["Info"]:
+            t1 = re.search(r'Description\/status\/(.*)$', line)
+            if t1: 
+                award_build[AW]["status"] = t1.groups()[0]
+            t1 = re.search(r'Description\/proposalType\/(.*)$', line)
+            if t1: 
+                award_build[AW]["proposal_type"] = t1.groups()[0]
+
+        for line in aw["Codes"]:
+            t1 = re.search(r'Banner\sCode\/(.*)$', line)
+            if t1: 
+                award_build[AW]["banner_code"] = t1.groups()[0]
+
+# substitute in program names more suitable for publishing
+        subbed = 0
+        for L in prog_title_subs:
+            if award_build[AW]["dept"] == L[0] and award_build[AW]["program_title"] == L[1]:
+                award_build[AW]["publish_title"] = L[2]
+                subbed = 1
+                if v: print("SUBBED")
+                if len(L)>3:
+                    award_build[AW]["publish_title2"] = L[3]
+                else:
+                    award_build[AW]["publish_title2"] = ""
+        
+        if not subbed:
+            award_build[AW]["publish_title"] = award_build[AW]["dept"]
+            award_build[AW]["publish_title2"] = ""
+        if award_build[AW]["program_title"] == "Liberal Arts: Computer Science &amp; Information Systems Emphasis":
+            award_build[AW]["publish_title"] = "Computer Science and Information Studies"
+            award_build[AW]["publish_title2"] = "Liberal Arts"
+            if v: print("-----LIB ART CSIS")
+        
+        if v:
+            print("%s / %s - %s" % (award_build[AW]["publish_title"],award_build[AW]["program_title"], award_build[AW]["award"]))
+        
+        v = 0
+        
+        for line in aw["Program Learning Outcomes"]:
+            t1 = re.search(r'Program\sLearning\sOutcomes\/\d+\/Outcome\/(\d+)\/cqid_(\d+)\/Outcome\/Outcome\/(.*)$', line)
+            if t1: 
+                if "PLO" in award_build[AW]:
+                    award_build[AW]["PLO"].append( (t1.groups()[0], t1.groups()[2]) )
+                else:
+                    award_build[AW]["PLO"] = [ (t1.groups()[0], t1.groups()[2]), ]
+        
+        st = lambda x: x[0]
+        award_build[AW]["PLO"] = sorted( award_build[AW]["PLO"], key=st )
+        award_build[AW]["PLO"] = [ x[1] for x in award_build[AW]["PLO"] ]
+        req_prebuild = defaultdict(list)
+        
+        pbd_unit_calcs = {}
+        
+        # requirements table:
+        # - most types have a 'units' column, which might be calculated
+        # - might be overridden
+        # - might be single number or a range min/max
+        
+        
+        
+        for line in aw["Program Requirements"]:
+            t1 = re.search(r'Program\sBlock\sDefinitions\/(\d+)/cqid_\d+/Program\sBlock\sDefinitions\/(.*)$', line)
+            if t1: 
+                pbd_number = t1.groups()[0]
+                if not pbd_number in pbd_unit_calcs:
+                    pbd_unit_calcs[pbd_number] = {'unit_sum':0,'unit_sum_max':0,'override':0,'min':0,'max':0}
+                t2 = re.search(r'Requirements\/\d+\/Program\sBlock\sDefinitions\/(\d+)\/cqid_\d+\/Program\sBlock\sDefinitions\/Course\sBlock\sDefinition\/(.*)$', line)
+                if t2:
+                    req_prebuild[pbd_number].append( ('h3', '0', t2.groups()[1]) )
+                    continue
+                t3 = re.search(r'Definitions\/\d+\/Program\sCourses\/(\d+)\/cqid_\d+\/Program\sCourses\/\d+\/\[Discipline\sand\sCourse\schained\scombo\]\/Course\/(.*)$',line)
+                if t3:
+                    req_prebuild[pbd_number].append( ('course', t3.groups()[0], splitclassline( t3.groups()[1], t3.groups()[0] )) )
+                    continue
+                t3a = re.search(r'Definitions\/\d+\/Program\sCourses\/(\d+)\/cqid_\d+/Program\sCourses\/\d+\/\[Condition\sSection\]\/Condition\/or$',line)
+                if t3a:
+                    req_prebuild[pbd_number].append( ('or', t3a.groups()[0]) )
+                    continue
+                t3b = re.search(r'Definitions\/\d+\/Program\sCourses\/(\d+)\/cqid_\d+/Program\sCourses\/\d+\/\[Condition\sSection\]\/Condition\/and$',line)
+                if t3b:
+                    req_prebuild[pbd_number].append( ('and', t3b.groups()[0]) )
+                    continue
+                t4 = re.search(r'Definitions\/(\d+)\/cqid_\d+/Program\sBlock\sDefinitions\/\d+\/Program\sCourses/(\d+)/cqid_\d+/Program\sCourses\/Non\-Course\sRequirements\/(.*)$',line)
+                if t4:
+                    req_prebuild[pbd_number].append( ('noncourse', t4.groups()[1], t4.groups()[2]) )
+                    continue
+                t5 = re.search(r'Definitions\/(\d+)\/cqid_\d+\/Program\sBlock\sDefinitions\/Override\sUnit\sCalculation\/1$',line)
+                if t5:
+                    pbd_unit_calcs[pbd_number]['override'] = 1
+                    continue
+                t6 = re.search(r'Definitions\/(\d+)\/cqid_\d+\/Program\sBlock\sDefinitions\/Unit\sMin\/(.*)$',line)
+                if t6:
+                    pbd_unit_calcs[pbd_number]['min'] = t6.groups()[1]
+                    continue
+                t7 = re.search(r'Definitions\/(\d+)\/cqid_\d+\/Program\sBlock\sDefinitions\/Unit\sMax/(.*)$',line)
+                if t7:
+                    pbd_unit_calcs[pbd_number]['max'] = t7.groups()[1]
+                    continue
+                t8 = re.search(r'chained\scombo\]\/Discipline',line)
+                if t8:
+                    continue
+                t8a = re.search(r'Units\s[Low|High]',line)
+                if t8a:
+                    continue
+                t9 = re.search(r'Definitions\/Block\sHeader\/(.*)$',line)
+                if t9:
+                    req_prebuild[pbd_number].append( ('blockheader', t9.groups()[0]) )
+                    continue
+                req_prebuild[pbd_number].append( ('', t1.groups()[1]) )
+        award_build[AW]["requirements"] = req_prebuild
+        award_build[AW]["unit_calcs"] = pbd_unit_calcs
+        
+        # associate unit calculations with program blocks
+        for block_key in req_prebuild.keys():
+            if block_key in pbd_unit_calcs:
+                req_prebuild[block_key].insert(0, pbd_unit_calcs[block_key])
+            else:
+                req_prebuild[block_key].insert(0, {'unit_sum':0,'unit_sum_max':0,'override':0})
+         
+         # do the unit calc math
+        for block_key in req_prebuild.keys():
+            this_block = req_prebuild[block_key]
+            pad = this_block[0]
+            if v: print("pad: ",pad)
+            block_dict = {}
+            for item in this_block[1:]:
+                print(item)
+                try:
+                    if item[0] == "or":
+                        block_dict[ item[1]+"or" ] = 1
+                    if item[0] == "h3":
+                        if v: print("+ ", item[1])
+                    if item[0] == "blockheader":
+                        if v: print("  ", item[1])
+                    if not item[0] == "course":
+                        continue
+                    block_dict[ item[1] ] = item[2]
+                    seq = int(item[1])
+                    units = ''
+                    if item[2]['units']: units = float( item[2]['units'] )
+                except Exception as e:
+                    print("ERROR ERROR\nERROR ERROR")
+                    print(e)
+                    xyz = input('hit return to continue')
+                #print( "%i \t %f \t %s" % (seq,units, item[2]['name']))
+            if v:
+                for k in sorted( block_dict.keys() ):
+                    print(k," ", block_dict[k])
+            #for k in sliding_window(3, sorted( block_dict.keys() )):
+            #    l,m,n = k
+            #    if re.search(r'or$',m): 
+            #        print("OR")
+            #        print(block_dict[l],"\n",block_dict[m],"\n",block_dict[n],"\n\n")
+            #print()
+            
+
+    output  = codecs.open("cache/programs/programs_built.json","w","utf-8")
+    output.write( json.dumps(award_build, indent=2) )
+
+
+
+
+    
+def course_path_style_2_html():
+    verbose = 1
+    v = verbose
+    
+    
+    oo = codecs.open("cache/courses/allclasspaths.txt","r","utf-8").readlines()
+    course_prebuild = defaultdict( ddl )
+    last_line = ""
+    
+    for L in oo:
+        L = L.strip()
+        if not re.search(r'^Course',L):
+            last_line = last_line + " <br /> " + L
+            continue
+        else:
+            if re.search(r'\/$',last_line):
+                # ignore line with trailing slash - assume no data
+                last_line = L
+                continue
+            
+            
+            test_1 = re.search(r'^Course\/(\d+)\/Course',last_line)
+            if test_1:
+                course_prebuild[  test_1.groups()[0] ]["Info"].append(last_line)
+            test_2 = re.search(r'^Course\/(\d+)\/(\d+)\/(.*?)\/(.*)$',last_line)
+            if test_2:
+                course_prebuild[  test_2.groups()[0] ][test_2.groups()[2]].append(last_line)
+            last_line = L
+    output  = codecs.open("cache/courses/courses_prebuild.json","w","utf-8")
+    output.write( json.dumps(course_prebuild, indent=2) )
+    
+    all_courses = {}
+    active_courses = {}
+    
+    lookup_table = {    'entityTitle':'title', 'proposalType':'type', 
+                        '\/Course\sDescription\/status':'status', 'Course\sDiscipline':'dept', 
+                        'Course\sNumber':'number', 'Course\sTitle':'name', 
+                        'Short\sTitle':'shortname', 'Internal\sProcessing\sTerm':'term', 'This\sCourse\sIs\sDegree\sApplicable':'degree_applicable',
+                        '\/Course\sDescription\/\d+\/Course\sDescription\/':'desc', 
+                        'Minimum\sUnits':'min_units', 'Minimum\sLecture\sHour':'min_lec_hour', 'Minimum\sLab\sHour':'min_lab_hour', 'Course\shas\svariable\shours':'has_var_hours',
+                        'Number\sWeeks':'weeks', 
+                        'Maximum\sUnits':'max_units', 'Credit\sStatus':'credit_status', 
+                        'TOP\sCode':'top_code', 'Classification':'classification', 'Non\sCredit\sCategory':'noncredit_category', 'Stand-Alone\sClass?':'stand_alone', 
+                        'Grade\sOption':'grade_option', 'Is\sRepeatable':'repeatable', 'Learning\sOutcomes\/Description':'slo',
+                        'Is\sThis\sCourse\sis\sRecommended\sfor\sTransfer\sto\sState\sUniversities\sand\sColleges?':'transfer_csu',
+                        'Is\sThis\sCourse\sis\sRecommended\sfor\sTransfer\sto\sUniversity\sof\sCalifornia?':'transfer_uc',
+                        '\/Catalog\sCourse\sSummary\sView\/':'catalog',
+                        '\/Course\sContent/\d+/Lecture\sContent\/':'content',
+                        '\/ASSIST\sPreview\/\d+\/Outcomes\sand\sObjectives\/':'objectives'}
+    
+    for C in sorted(list(course_prebuild.keys()),key=int):
+        v = 0
+        crs = course_prebuild[C]
+        course_build = {'slo':{}}    # defaultdict( ddl )
+        if v: print(C)
+        
+        for K in crs.keys():
+            if v: print("\t%s" % K)
+            for line in crs[K]:
+                for (str,key) in lookup_table.items():
+                    if re.search(str,line):
+                        if key == 'slo':
+                            #   \s<br\s\/>\s
+                            content_search = re.search(r'\/Learning\sOutcomes\/\d+\/cqid_(\d+)\/Learning\sOutcomes\/Description\/(.*?)$',line)
+                            if content_search: course_build['slo'][content_search.groups()[0]] = content_search.groups()[1]
+                            else:
+                                print("NO SLO? %s" % line)
+                        elif key == 'desc':
+                            content_search = re.search(r'^Course\/\d+\/\d+\/Course\sDescription\/\d+\/Course\sDescription\/(.*)$',line)
+                            course_build['desc'] = content_search.groups()[0]
+                        elif key == 'catalog':
+                            content_search = re.search(r'^Course\/\d+\/\d+\/General\sEducation\sPattern\/\d+\/Catalog\sCourse\sSummary\sView\/(.*)$',line)
+                            course_build['catalog'] = content_search.groups()[0]
+                        elif key == 'content':
+                            content_search = re.search(r'^Course\/\d+\/\d+\/Course\sContent\/\d+\/Lecture\sContent\/(.*)$',line)
+                            course_build['content'] = content_search.groups()[0]
+                        elif key == 'objectives':
+                            content_search = re.search(r'^Course\/\d+\/\d+\/ASSIST\sPreview\/\d+\/Outcomes\sand\sObjectives\/(.*)$',line)
+                            course_build['objectives'] = content_search.groups()[0]
+                        else:
+                            content_search = re.search(r'^(.*)\/(.*?)$',line)
+                            course_build[key] = content_search.groups()[1]
+                        if v: print("\t\t%s - %s" % (key, course_build[key]))
+                        continue
+                        
+        all_courses[C] = course_build
+        if course_build['status'] == 'Active':
+            active_courses[C] = course_build
+    output  = codecs.open("cache/courses/courses_built.json","w","utf-8")
+    output.write( json.dumps(all_courses, indent=2) )
+        
+    output2  = codecs.open("cache/courses/courses_active_built.json","w","utf-8")
+    output2.write( json.dumps(active_courses, indent=2) )
+        
+
+
+#########
+#########
+#########
+#########
+
+
+def another_request(url,startat):
+    global err_fail_filecount
+    newparam = "&skip=" + str(startat)
+    print((url+newparam))
+    r = requests.get(url+newparam, auth=(user,pasw))
+    try:
+        mydata = json.loads(r.text, strict=False)
+    except Exception as e:
+        print("Couldn't read that last bit")
+        #print((r.text))
+        codecs.open('cache/curric2022failfile_%i.txt' % err_fail_filecount,'w','utf-8').write(r.text)
+        err_fail_filecount += 1
+        print(e)
+        return 0,0,[]
+        
+    size = mydata['resultSetMetadata']['ResultSetSize']
+    endn = mydata['resultSetMetadata']['EndResultNum']
+    items = mydata['entityInstances']
+    print(('  Got ' + str(size) + ' instances, ending at item number ' + str(endn)))
+    return size,endn,items
+
+
+
+
+def fetch_all_classes():
+    if os.path.isdir('cache/courses'):
+        m = datetime.strptime(time.ctime(os.path.getctime('cache/courses')), "%a %b %d %H:%M:%S %Y")
+        today = 'cache/courses_%s' % m.strftime('%Y_%m_%d')
+        
+        print("+  Creating folder: %s" % today)
+        shutil.move('cache/courses', today)
+        os.makedirs('cache/courses')
+
+    size = 100
+    endn = 0
+    filen = 1
+    while(size > 99):
+        size, endn, items = another_request(CQ_URL+PARAM,endn)
+        out = codecs.open('cache/courses/classes_'+str(filen)+'.txt','w', 'utf-8')
+        out.write(json.dumps(items,indent=2))
+        out.close()
+        filen += 1 
+    print("Written to 'cache/courses....")
+
+
+
+
+#
+#
+# Main worker
+#
+
+def recur_path_matcher(item, path=[]):
+    def x2_path_update(x,y,z):
+        path.extend([str(y),x])
+        my_result_lines.append( '/'.join(path) + '/' + 'lastEdited' + '/' + z)
+        
+    path_str = "/".join(path) + "/"
+    path_str = re.sub('\/+','/',path_str)
+    path_str = re.sub('\s+',' ',path_str)
+    my_result_lines = []
+    if type(item) == type({}):
+        original_path = path.copy()
+        match( item, 
+            {'attributes': {'displayName': _}, 'lookUpDisplay': _, }, 
+                lambda x,y: my_result_lines.append("%s%s/%s" % (path_str, clean(str(x)), clean(str(y)))) ,
+            {'attributes': {'displayName': _}, 'fieldValue': _, }, 
+                lambda x,y: my_result_lines.append("%s%s/%s" % (path_str, clean(str(x)), clean(str(y)))) ,
+            {'attributes': {'fieldName': _}, 'fieldValue': _, }, 
+                lambda x,y: my_result_lines.append("%s%s/%s" % (path_str, clean(str(x)), clean(str(y)))) ,
+            {'instanceId':_, 'sectionName': _, 'sectionSortOrder':_}, 
+                lambda id,name,order: path.extend([str(order),'cqid_'+str(id),name]),
+            {'instanceId':_, 'sectionName': _, 'instanceSortOrder':_}, 
+                lambda id,name,order: path.extend([str(order),'cqid_'+str(id),name]),
+            {'sectionName': _, 'sectionSortOrder':_, 'lastUpdated': _ }, 
+                #lambda x,y,z: path.extend([str(y),x,z]),
+                x2_path_update,
+            {'sectionName': _, 'sectionSortOrder':_}, 
+                lambda x,y: path.extend([str(y),x]),
+            {'sectionName': _}, 
+                lambda x: path.append(x),
+            _, nothing    #lambda x: path.append('')
+        )
+        path = original_path
+        for K,V in list(item.items()):
+            my_result_lines.extend(recur_path_matcher(V,path))
+        
+    elif type(item) == type([]):
+        for V in item:
+            my_result_lines.extend(recur_path_matcher(V,path))
+    return my_result_lines
+
+
+
+
+def pathstyle(theclass):
+    #theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
+    #        {'entityMetadata': {'entityTitle': _,'status': _, 'entityType':_, 'entityId':_ }}, 
+    #            lambda title,status,typ,id: 
+    #                my_result_lines.append("%s%s/%s/%s   [%s]" % (path_str, str(typ), str(id), str(title),str(status))) ,
+    if "entityMetadata" in theclass:
+        id = theclass["entityMetadata"]["entityId"]
+        title = theclass["entityMetadata"]["entityTitle"]
+        typ = theclass["entityMetadata"]["entityType"]
+        action = theclass["entityMetadata"]["proposalType"]
+        status = theclass["entityMetadata"]["status"]
+        
+        #"entityId": 4077,
+        #"entityTitle": "ENGL2B - American Ethnic Literature",
+        #"entityType": "Course",
+        #"proposalType": "Deactivate Course",
+        #"status": "Historical",
+
+        result = [ "/".join([ typ,str(id),"Course Description","entityTitle",title]) ,
+        "/".join([ typ,str(id),"Course Description","entityType",typ]) ,
+        "/".join([ typ,str(id),"Course Description","proposalType",action]) ,
+        "/".join([ typ,str(id),"Course Description","status",status]) , ]
+        
+        result.extend(recur_path_matcher(theclass["entityFormData"]["rootSections"], [typ,str(id)] ))
+        #oo = codecs.open("cache/courses/curric2022test_path.json","w","utf-8")
+        #print(result)
+        return result
+    else:
+        print("didn't seem to be a class.")
+
+
+
+def single_course_path_parse(c):
+    this_course = []
+    global num_failed_course
+    if "attributes" in c and "entityId" in c["attributes"]:
+        print(c["attributes"]["entityId"])
+        return (c["attributes"]["entityId"], pathstyle(c))
+    else:
+        print("I couldn't recognize a class in that")
+        ooops = codecs.open('cache/programs/failedcourse_%i.json' % num_failed_course, 'w', 'utf-8')
+        ooops.write(json.dumps(c,indent=2))
+        ooops.close()
+        num_failed_course = num_failed_course + 1
+        return ("-1", [])
+        
+
+def path_style_test():
+    classes = {}
+    oo = codecs.open("cache/courses/allclasspaths.txt","w","utf-8")
+    for f in os.listdir('cache/courses'):
+        if re.search('^classes_',f):
+            print(f)
+            cls = json.loads(codecs.open('cache/courses/'+f,'r','utf-8').read(),strict=False)
+            for c in cls:
+                id,output = single_course_path_parse(c)
+                classes[id] = "\n".join(output)
+                oo.write( classes[id] )
+                oo.write( "\n\n\n" + "-"*30 + "\n\n" )
+                oo.flush()
+
+def make_sl():
+    return SortedList(key=lambda x: -1 * int(x['m']))
+    
+def course_rank():
+    csvfile = codecs.open('cache/courses/all_courses_ranked.csv','w','utf-8')
+    csvwriter = csv.writer(csvfile)
+    csvwriter.writerow("code,cqcourseid,coursestatus,termineffect,dept,num,numoutcomes".split(","))
+    
+    courses = json.loads(codecs.open('cache/courses/course_cq_index.json','r','utf-8').read())
+    all = defaultdict(make_sl)
+    for c in courses:
+        code = c['d']+c['n']
+        if not c['m']:
+            c['m'] = '200030'
+        all[code].add(c)
+    
+    for k in sorted(all.keys()):
+        print("\n##",k)
+        print(json.dumps(list(all[k]),indent=2))
+        for version in all[k]:
+            csvwriter.writerow( [ version['d']+version['n'], version['c'], version['s'], version['m'], version['d'], version['n'], len(version['o']) ])
+    
+
+
+
+if __name__ == "__main__":
+    
+    print ('')
+    options = { 1: ['fetch all courses', fetch_all_classes],
+                2: ['process all classes', path_style_test],
+                3: ['courses - path style to html catalog', course_path_style_2_html],
+                4: ['courses - rank by all versions', course_rank],
+                5: ['fetch all programs', fetch_all_programs],
+                6: ['process all programs', path_style_prog],
+                9: ['show course outcomes', all_outcomes],
+               10: ['programs - path style to html catalog', path_style_2_html],
+             }
+    
+    print ('')
+
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
diff --git a/curriculum.py b/curriculum.py
new file mode 100644
index 0000000..db778bb
--- /dev/null
+++ b/curriculum.py
@@ -0,0 +1,2252 @@
+import requests,json,os,re, bisect, csv, codecs
+import sortedcontainers as sc
+from collections import defaultdict
+from toolz.itertoolz import groupby
+#from docx.shared import Inches
+#from docx import Document
+#import docx
+from durable.lang import *
+from durable.engine import *
+from pampy import match, _
+from bs4 import BeautifulSoup as bs
+import pandas as pd
+import sys, locale, re
+from pipelines import getSemesterSchedule
+
+from secrets import cq_url, cq_user, cq_pasw
+
+
+#sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+
+TRACING = codecs.open('cache/progdebug.txt','w','utf-8')
+param = "?method=getCourses"
+
+
+def dbg(x): 
+    if TRACING: TRACING.write(' + %s\n' % str(x))
+
+sems = ['sp20','fa19', 'su19','sp19']
+    
+
+
+filen = 1
+def another_request(url,startat):
+    global cq_user, cq_pasw, TRACING
+    newparam = "&skip=" + str(startat)
+    print((url+newparam))
+    r = requests.get(url+newparam, auth=(cq_user,cq_pasw))
+    try:
+        TRACING.write(r.text + "\n\n")
+        TRACING.flush()
+        mydata = json.loads(r.text)
+    except Exception as e:
+        print("Couldn't read that last bit")
+        print((r.text))
+        print(e)
+        return 0,0,[]
+        
+    size = mydata['resultSetMetadata']['ResultSetSize']
+    endn = mydata['resultSetMetadata']['EndResultNum']
+    items = mydata['entityInstances']
+    print(('  Got ' + str(size) + ' instances, ending at item number ' + str(endn)))
+    return size,endn,items
+def fetch_all_classes():
+    global cq_url,param
+    size = 100
+    endn = 0
+    filen = 1
+    while(size > 99):
+        size, endn, items = another_request(cq_url+param,endn)
+        out = open('cache/courses/classes_'+str(filen)+'.txt','w')
+        out.write(json.dumps(items,indent=2))
+        out.close()
+        filen += 1 
+    print("Written to 'cache/classes....")
+    
+def fetch_all_programs():
+    global cq_url
+    size = 100
+    endn = 0
+    filen = 1
+    param = "?returnFormat=json&method=getPrograms&status=Active"
+    while(size > 99):
+        size, endn, items = another_request(cq_url+param,endn)
+        out = open('cache/programs/programs_'+str(filen)+'.txt','w')
+        out.write(json.dumps(items,indent=4))
+        out.close()
+        filen += 1 
+    print("Written to 'cache/programs....")
+def sortable_class(li):
+    dept = li[1]
+    rest = ''
+    
+    print(li)
+    # another dumb special case / error
+    if li[2] == "ASTR 1L": li[2] = "1L"
+    # little error case here
+    n = re.match(r'([A-Za-z]+)(\d+)',li[2])
+    if n:
+        num = int(n.group(2))
+    else:
+        m = re.match(r'(\d+)([A-Za-z]+)$',li[2])
+        if m:
+            num = int(m.group(1))
+            rest = m.group(2)
+        else:
+            num = int(li[2])
+    if num < 10: num = '00'+str(num)
+    elif num < 100: num = '0'+str(num)
+    else: num = str(num)
+    return dept+num+rest
+
+def c_name(c):
+    delivery = set()
+    units = []
+    slos = []
+    hybridPct = ''
+    active = 'Active'
+    id = c['entityMetadata']['entityId']
+    if c['entityMetadata']['status'] != 'Active':
+        active = 'Inactive'
+        #return ()
+    for r in c['entityFormData']['rootSections']:
+        
+        if r['attributes']['sectionName'] == 'Course Description':
+        
+            for ss in r['subsections']:
+                for f in ss['fields']:
+                    
+                    if f['attributes']['fieldName'] == 'Course Discipline':
+                        dept = f['lookUpDisplay']
+                    if f['attributes']['fieldName'] == 'Course Number':
+                        num = f['fieldValue']
+                    if f['attributes']['fieldName'] == 'Course Title':
+                        title = f['fieldValue']
+                        #print "\n" + title
+                    if f['attributes']['fieldName'] == 'Course Description':
+                        desc = re.sub(r'\n',' ', f['fieldValue'])
+        if r['attributes']['sectionName'] == 'Units/Hours/Status':
+            for ss in r['subsections']:
+                if ss['attributes']['sectionName'] == '':
+                    for f in ss['fields']:
+                        if f['attributes']['fieldName'] == 'Minimum Units' and f['fieldValue'] not in units:
+                            units.insert(0,f['fieldValue'])
+                        if f['attributes']['fieldName'] == 'Maximum Units' and f['fieldValue'] and f['fieldValue'] not in units:
+                            units.append(f['fieldValue'])
+
+        
+        # Newer entered courses have this filled out
+        if r['attributes']['sectionName'] == 'Distance Education Delivery':
+            for ss in r['subsections']:
+                if ss['attributes']['sectionName'] == 'Distance Education Delivery':
+                    for ssa in ss['subsections']:
+                        for f in ssa['fields']:
+                            if f['attributes']['fieldName'] == 'Delivery Method':
+                                    delivery.add(f['lookUpDisplay'])
+                if ss['attributes']['sectionName'] == "":
+                    if ss['fields'][0]['attributes']['fieldName'] == "If this course is Hybrid, what percent is online?":
+                        hybridPct = str(ss['fields'][0]['fieldValue'])
+
+        # Older ones seem to have it this way
+        if r['attributes']['sectionName'] == 'Distance Education':
+            for ss in r['subsections']:
+                for f2 in ss['fields']:
+                    if 'fieldName' in f2['attributes'] and f2['attributes']['fieldName'] == 'Methods of Instruction':
+                        #print f2['fieldValue']
+                        if f2['fieldValue'] == 'Dist. Ed Internet Delayed':
+                            delivery.add('Online')
+
+        # SLO
+        if r['attributes']['sectionName'] == 'Student Learning Outcomes':
+            for ss in r['subsections']:
+                if 'subsections' in ss:
+                    if ss['attributes']['sectionName'] == 'Learning Outcomes':
+                        for s3 in ss['subsections']:
+                            for ff in s3['fields']:
+                                if ff['attributes']['fieldName'] == 'Description':
+                                    slos.append(ff['fieldValue'])
+                        
+                        #print ff
+                        #[0]['fields']:
+                        #print ff['fieldValue']
+                #for f2 in ss['fields']:
+                #    if 'fieldName' in f2['attributes'] and f2['attributes']['fieldName'] == 'Methods of Instruction':
+                #        if f2['fieldValue'] == 'Dist. Ed Internet Delayed':
+                #            delivery.append('online(x)')
+
+    if len(units)==1: units.append('')  
+    if len(delivery)==0: delivery.add('')
+    u0 = 0 
+    try: 
+        u0 = units[0]
+    except:
+        pass
+        
+    u1 = 0 
+    try: 
+        u1 = units[2]
+    except:
+        pass
+        
+    return id,dept,num,active,title,u0,u1,'/'.join(delivery),hybridPct,desc,slos    
+ 
+        
+def show_classes(createoutput=1):
+    max_active = {}   # hold the id of the class if seen. only include the highest id class in main list. 
+    used_course = {}  # hold the actual course info, the version we'll actually use.
+    slo_by_id = {} # values are a list of slos.
+    slo_by_id_included = {} # just the ids of active or most recent versions.
+    #tmp = codecs.open('cache/course_temp.txt','w','utf-8')
+    for f in os.listdir('cache/courses'):
+        if re.search('classes_',f):
+            print(f)
+            cls = json.loads(open('cache/courses/'+f,'r').read())
+            for c in cls:
+                dir_data = list(c_name(c))
+                #tmp.write(str(dir_data) + "\n\n")
+                slo_by_id[dir_data[0]] = dir_data[10] # 
+                info = list(map(str,dir_data[:10]))
+                info.append(dir_data[10])
+                #pdb.set_trace()
+                #print info
+                course_key = sortable_class(info)
+                curqnt_id = int(info[0])
+                if course_key in max_active:
+                    if curqnt_id < max_active[course_key]:
+                        continue
+                max_active[course_key] = curqnt_id
+                used_course[course_key] = info
+    
+    if not createoutput: return 1
+    
+    # now we have the ideal version of each course
+    all = sc.SortedList(key=sortable_class)
+    for key,crs in list(used_course.items()): all.add(crs)
+    
+    by_dept = groupby(1,all)
+    
+    t = open('cache/courses/index.json','w')
+    t.write(json.dumps(sorted(by_dept.keys())))
+    
+    u = open('cache/courses/slos.json','w')
+    
+    for d in list(by_dept.keys()):
+        s = open('cache/courses/' + d.lower() + '.json','w')
+        for course in by_dept[d]:
+            try:
+                course.append(slo_by_id[int(d[0])])
+            except:
+                pass
+        s.write(json.dumps(by_dept[d], indent=2))
+        s.close()
+        for c in by_dept[d]:
+            ss =  slo_by_id[int(c[0])]     #  [ x.encode('ascii ','ignore') for x in slo_by_id[int(c[0])] ]
+            slo_by_id_included[int(c[0])] = ss
+            
+    u.write( json.dumps(slo_by_id_included, indent=2))
+    
+def clean_d_name(d):
+    d = d.lower()
+    d = re.sub(r'[\&\(\)\.\/\:]','',d)
+    d = re.sub(r'[\s\-]+','_',d)
+    return d
+                
+def show_programs():
+    allprogs = defaultdict(list)
+    dept_index = set([('Liberal Arts','liberal_arts'),])
+    prog_index = defaultdict(list)
+    for f in os.listdir('cache/programs'):
+        if re.search('programs_',f):
+            print(f)
+            pro = json.loads(open('cache/programs/'+f,'r').read())
+            for c in pro:
+                this_prog = prog_take_4(c)
+                if not 'dept' in this_prog: this_prog['dept'] = 'Liberal Arts'
+                if not 'type' in this_prog: 
+                    this_prog['type'] = ''             #print "*** Why no type?"
+                this_prog['key'] = clean_d_name(this_prog['title']+'_'+this_prog['type'])
+                dept_index.add( (this_prog['dept'],clean_d_name(this_prog['dept'] )) )
+                bisect.insort(prog_index[this_prog['dept']], (this_prog['title'], this_prog['type'], clean_d_name(this_prog['dept'])+'/'+clean_d_name(this_prog['title'])+'/'+clean_d_name(this_prog['type'])))
+                allprogs[this_prog['dept']].append( this_prog )
+    for D,li in list(allprogs.items()):    
+        dept = clean_d_name(D)
+        s = open('cache/programs/'+dept+'.json','w')
+        s.write( json.dumps(sorted(li,key=lambda x: x['title']),indent=2) )
+        s.close()
+    
+    s = open('cache/programs/index.json','w')
+    s.write( json.dumps({'departments':sorted(list(dept_index)), 'programs':prog_index}, indent=2) )
+    s.close()
+    
+    #r = open('cache/deg_certs.json','w')
+    #r.write( json.dumps(sorted(list(allprogs.items())), indent=2) )
+    #r.close()
+    
+    organize_programs_stage2(  )
+    
+def dd(): return defaultdict(dd)
+    
+def organize_courses():
+    keys = "id,dept,num,active,title,low_unit,hi_unit,is_online,hybrid_pct,desc,slos".split(",")
+    
+    depts = defaultdict(dd)
+    
+    for f in os.listdir('cache/courses'):
+        if f == 'index.json': continue
+        if f == 'slos.json': continue
+        #print f
+        u = open('cache/courses/' + f,'r')
+        w = json.loads(u.read())
+        for A in w:
+            course = {}
+            i = 0
+            for k in keys:
+                course[k] = A[i]
+                i += 1
+            depts[ course['dept'] ][ course['num'] ] = course
+            
+            print((A[7], "\t", A[8], "\t",A[4]))
+    o = open('cache/courses_org.json','w')
+    o.write(json.dumps(depts,indent=2))
+    
+def check_de():
+    for f in os.listdir('cache/courses'):
+        if f == 'index.json': continue
+        if f == 'slos.json': continue
+        #print f
+        u = open('cache/courses/' + f,'r')
+        w = json.loads(u.read())
+        for A in w:
+            print((A[7], "\t", A[8], "\t",A[4]))
+    
+def clean_programs():
+    #rewrite_list = 0   ########################################### Careful you don't overwrite the existing file!
+    
+    re_list = open('req_phrases.txt','r').readlines()
+    req_what_do = {}
+    last_times_seen = {}
+    req_times_seen = defaultdict(int)
+    for L in re_list:
+        L = L.strip()
+        parts = L.split('|')
+        req_what_do[parts[0]] = parts[1]
+        req_times_seen[parts[0]] = 0
+        last_times_seen[parts[0]] = parts[2]           
+            
+    attained = csv.DictReader(open("cache/degrees_attained.csv"))
+    att_keys = []
+    for row in attained:
+        att_keys.append(row['Program'])
+    
+    #temp = open('cache/temp.txt','w')
+    #temp.write(json.dumps(sorted(att_keys), indent=2))
+    
+    progs = json.loads(open('cache/programs.json','r').read())
+    
+    # list of phrases that describe requirements
+    reqs = Set()
+    
+    prog_keys = []
+    for k in progs:
+        if not 'title' in k or not 'type' in k or not 'dept' in k:
+            pass
+            #print "Funny prog: " 
+            #print k
+        else:
+            ty = re.sub('Degree','',k['type'])
+            ty = re.sub('\.','',ty)
+            prog_title =  k['dept'] + ": " + k['title'] + " " + ty
+            prog_keys.append(prog_title)
+            for b in k['blocks']:
+                rule = ''
+                if 'courses' in b and len(b['courses']):
+                    if 'rule' in b and not b['rule']==' ': 
+                        #print b['rule']
+                        reqs.add(b['rule'])
+                        rule = b['rule']
+                        req_times_seen[rule] += 1
+                        if req_what_do[rule] == 'q':
+                            print(("\nIn Program: " + prog_title))
+                            print(("What does this rule mean?   " + rule))
+                            print(("(I see it " + last_times_seen[rule] + " times.)"))
+                            for C in b['courses']: print(("   " + C))
+                            z = eval(input())
+                    for c in b['courses']:
+                        if re.search('header2',c):
+                            parts = c.split('|')
+                            #print "" + parts[1]
+                            reqs.add(parts[1])
+                            rule = parts[1]
+                            req_times_seen[rule] += 1
+                
+                            if req_what_do[rule] == 'q':
+                                print(("\nIn Program: " + prog_title))
+                                print(("What does this rule mean?   " + rule))
+                                print(("(I see it " + last_times_seen[rule] + " times.)"))
+                                for C in b['courses']: print(("   " + C))
+                                z = eval(input())
+                            
+    # Key for the list
+    # q - ask whats up with this rule
+    # n1 - single class required
+    # n2 - two classes required
+    # n3
+    # n4
+    # u1 - u99 - that many units required (minimum. ignore max)
+    # a - all of them
+    # x - ignore
+    # s - special or more logic needed
+    # e - recommended electives
+    #reqs_list = open('req_phrases.txt','w')
+    #for a in sorted( list(reqs) ):
+    #    if a == ' ': continue
+    #    reqs_list.write(a+"|" + req_what_do[a] + "|" + str(req_times_seen[a]) + "\n")
+                        
+                        
+        #mat = process.extractOne(prog_title, att_keys)
+        #print "Title: " + prog_title + "  Closest match: " + mat[0] + "  at " + str(mat[1]) + "% conf"
+    #temp.write(json.dumps(sorted(prog_keys),indent=2))  
+def course_lil_format(s):
+    # "02-125706|THEA12B - Acting II 3.000  *Historical*"
+    
+    parts = s.split('|')
+    parts2 = parts[1].split(' - ')
+    parts3 = parts2[1].split(' ')[0:-3]
+    return parts2[0], parts3     ###   code,   name
+    
+def header_lil_format(s):
+    # "04-125802header2|Choose 2 courses from following list:"
+    
+    parts = s.split('|')
+    return parts[1]
+    
+    
+def organize_programs():
+    re_list = open('req_phrases.txt','r').readlines()
+    req_what_do = {'':'x'}
+    last_times_seen = {}
+    req_times_seen = defaultdict(int)
+    
+    num_programs = 0
+    num_w_special_logic = 0
+    num_okay = 0
+    
+    fout = open('program_worksheet.txt','w')
+    
+    
+    for L in re_list:
+        L = L.strip()
+        parts = L.split('|')
+        req_what_do[parts[0]] = parts[1]
+        req_times_seen[parts[0]] = 0
+        last_times_seen[parts[0]] = parts[2]           
+            
+    progs = json.loads(open('cache/programs.json','r').read())
+    prog_keys = []
+    output = ''
+    for k in progs:
+        rule_sequence = []
+        if not 'title' in k or not 'type' in k or not 'dept' in k:
+            pass
+            #print "Funny prog: " 
+            #print k
+        else:
+            num_programs += 1
+            ty = re.sub('Degree','',k['type'])
+            ty = re.sub('\.','',ty)
+            prog_title =  k['dept'] + ": " + k['title'] + " " + ty
+            output += "\n" + prog_title + "\n"
+            prog_keys.append(prog_title)
+            for b in k['blocks']:
+                rule = ''
+                if 'courses' in b and len(b['courses']):
+                    if 'rule' in b and not b['rule']==' ': 
+                        rule = b['rule']
+                        output += "  Rule: ("+ req_what_do[rule]+ ") " +  b['rule'] + "\n"
+                        rule_sequence.append(req_what_do[rule])
+                        #reqs.add(b['rule'])
+                        req_times_seen[rule] += 1
+                        if req_what_do[rule] == 'q':
+                            print(("\nIn Program: " + prog_title))
+                            print(("What does this rule mean?   " + rule))
+                            print(("(I see it " + last_times_seen[rule] + " times.)"))
+                            for C in b['courses']: print(("   " + C))
+                            z = eval(input())
+                    
+                    miniblocks = []
+                    this_miniblock = {'courses':[], 'header':''}
+                    for c in sorted(b['courses'])[::-1]:
+                        if re.search('header2',c):
+                            parts = c.split('|')
+                            if this_miniblock['courses'] or req_what_do[this_miniblock['header']]!='x':
+                                miniblocks.append(this_miniblock)
+                                rule_sequence.append(req_what_do[this_miniblock['header']])
+                            rule = parts[1]
+                            this_miniblock = {'header':rule,'courses':[] }
+                            req_times_seen[rule] += 1
+                
+                            if req_what_do[rule] == 'q':
+                                print(("\nIn Program: " + prog_title))
+                                print(("What does this rule mean?   " + rule))
+                                print(("(I see it " + last_times_seen[rule] + " times.)"))
+                                for C in b['courses']: print(("   " + C))
+                                z = eval(input())
+                        else:
+                            code,name = course_lil_format(c)
+                            
+                            this_miniblock['courses'].append(code)
+                    if not this_miniblock['header']:
+                        output += "    "
+                        for ccc in this_miniblock['courses']:
+                            output += ccc + " "
+                        output += "\n"
+                    # final course, final mb append
+                    if this_miniblock['courses']:
+                        miniblocks.append(this_miniblock)
+                        rule_sequence.append(req_what_do[this_miniblock['header']])
+                        
+                    if miniblocks:
+                        for m in miniblocks:
+                            if m['header']:
+                                output += "    Miniblock rule: ("+ req_what_do[rule] + ") " + m['header'] + "\n"
+                                output += "      "
+                                for c in m['courses']:
+                                    output += c + " "
+                                output += "\n"
+            if 's' in rule_sequence:
+                num_w_special_logic += 1
+            else:
+                num_okay += 1
+            output += "  Summary: [" + " ".join(rule_sequence) + " ]" + "\n"
+    
+    fout.write(output)
+    print(("Number of programs: " + str(num_programs)))
+    print(("Number without special logic: " + str(num_okay)))
+    print(("Number with special logic: " + str(num_w_special_logic)))
+    # Key for the list
+    # q - ask whats up with this rule
+    # n1 - single class required
+    # n2 - two classes required
+    # n3
+    # n4
+    # u1 - u99 - that many units required (minimum. ignore max)
+    # a - all of them
+    # x - ignore
+    # s - special or more logic needed
+    # e - recommended electives
+    
+def divide_courses_list(li,rwd,online):
+    # return a list of lists. 
+    lol = []
+    cur_list = []
+    
+    for L in sorted(li):
+        if re.search('header2',L):
+            if cur_list: lol.append(cur_list)
+            cur_list = []
+            L = header_lil_format(L)
+            L = rwd[L] + ": " + L
+        else:
+            L,x = course_lil_format(L)
+            if online[L]: L = L + "    " + online[L]
+        if L[0]!='x': cur_list.append(L)    
+    lol.append(cur_list)
+    return lol
+    
+def organize_programs2():
+    re_list = open('req_phrases.txt','r').readlines()
+    classes = json.loads(open('cache/courses_org.json','r').read())
+    classes_bycode = {}
+    
+    for d in list(classes.keys()):
+        for c in list(classes[d].keys()):
+            classes_bycode[d+" "+c] = classes[d][c]['is_online']
+            classes_bycode[d+c] = classes[d][c]['is_online']
+            #print d+c+":\t"+classes_bycode[d+c]
+        
+    req_what_do = {'':'x', ' ':'x'}
+    last_times_seen = {}
+    req_times_seen = defaultdict(int)
+    
+    num_programs = 0
+    num_w_special_logic = 0
+    num_okay = 0
+    
+    fout = open('cache/program_worksheet.txt','w')
+    
+    cout = open('cache/classes_online.json','w')
+    cout.write(json.dumps(classes_bycode))
+    cout.close()
+    
+    
+    for L in re_list:
+        L = L.strip()
+        parts = L.split('|')
+        req_what_do[parts[0]] = parts[1]
+        req_times_seen[parts[0]] = 0
+        last_times_seen[parts[0]] = parts[2]           
+            
+    progs = json.loads(open('cache/programs.json','r').read())
+    prog_keys = []
+    output = ''
+    for k in progs:
+        rule_sequence = []
+        if not 'title' in k or not 'type' in k or not 'dept' in k:
+            pass
+            #print "Funny prog: " 
+            #print k
+        else:
+            num_programs += 1
+            ty = re.sub('Degree','',k['type'])
+            ty = re.sub('\.','',ty)
+            prog_title =  k['dept'] + ": " + k['title'] + " " + ty
+            output += "\n" + prog_title + "\n"
+            for b in sorted(k['blocks'], key=lambda x: x['order'] ):
+                rule = ''
+                if 'courses' in b and len(b['courses']) and 'rule' in b and req_what_do[b['rule']]!='x':
+                    #req_what_do[b['rule']]
+                    output += "  "+req_what_do[b['rule']]+": " + b['rule'] + "\n"
+                    output += json.dumps(divide_courses_list(b['courses'],req_what_do,classes_bycode),indent=2) + "\n"
+            """
+            prog_keys.append(prog_title)
+            for b in k['blocks']:
+                rule = ''
+                if 'courses' in b and len(b['courses']):
+                    if 'rule' in b and not b['rule']==' ': 
+                        rule = b['rule']
+                        output += "  Rule: ("+ req_what_do[rule]+ ") " +  b['rule'] + "\n"
+                        rule_sequence.append(req_what_do[rule])
+                        #reqs.add(b['rule'])
+                        req_times_seen[rule] += 1
+                        if req_what_do[rule] == 'q':
+                            print "\nIn Program: " + prog_title
+                            print "What does this rule mean?   " + rule
+                            print "(I see it " + last_times_seen[rule] + " times.)"
+                            for C in b['courses']: print "   " + C
+                            z = raw_input()
+                    
+                    miniblocks = []
+                    this_miniblock = {'courses':[], 'header':''}
+                    for c in sorted(b['courses'])[::-1]:
+                        if re.search('header2',c):
+                            parts = c.split('|')
+                            if this_miniblock['courses'] or req_what_do[this_miniblock['header']]!='x':
+                                miniblocks.append(this_miniblock)
+                                rule_sequence.append(req_what_do[this_miniblock['header']])
+                            rule = parts[1]
+                            this_miniblock = {'header':rule,'courses':[] }
+                            req_times_seen[rule] += 1
+                
+                            if req_what_do[rule] == 'q':
+                                print "\nIn Program: " + prog_title
+                                print "What does this rule mean?   " + rule
+                                print "(I see it " + last_times_seen[rule] + " times.)"
+                                for C in b['courses']: print "   " + C
+                                z = raw_input()
+                        else:
+                            code,name = course_lil_format(c)
+                            
+                            this_miniblock['courses'].append(code)
+                    if not this_miniblock['header']:
+                        output += "    "
+                        for ccc in this_miniblock['courses']:
+                            output += ccc + " "
+                        output += "\n"
+                    # final course, final mb append
+                    if this_miniblock['courses']:
+                        miniblocks.append(this_miniblock)
+                        rule_sequence.append(req_what_do[this_miniblock['header']])
+                        
+                    if miniblocks:
+                        for m in miniblocks:
+                            if m['header']:
+                                output += "    Miniblock rule: ("+ req_what_do[rule] + ") " + m['header'] + "\n"
+                                output += "      "
+                                for c in m['courses']:
+                                    output += c + " "
+                                output += "\n"
+            if 's' in rule_sequence:
+                num_w_special_logic += 1
+            else:
+                num_okay += 1
+            output += "  Summary: [" + " ".join(rule_sequence) + " ]" + "\n"
+    """
+    fout.write(output)
+    print(("Number of programs: " + str(num_programs)))
+    print(("Number without special logic: " + str(num_okay)))
+    print(("Number with special logic: " + str(num_w_special_logic)))
+    # Key for the list
+    # q - ask whats up with this rule
+    # n1 - single class required
+    # n2 - two classes required
+    # n3
+    # n4
+    # u1 - u99 - that many units required (minimum. ignore max)
+    # a - all of them
+    # x - ignore
+    # s - special or more logic needed
+    # e - recommended electives
+# sorting by order key of dict
+def cmp_2(a):
+    return a['order']
+def cmp_order(a,b):
+    if a['order'] > b['order']: return 1
+    if a['order'] < b['order']: return -1
+    if a['order'] == b['order']: return 0
+    
+# decipher the grouped up courses line
+def split_course(st):
+    # "01-127153|SOC1A - Introduction to Sociology 3.000  *Active*"
+    if 'header2' in st:
+        return st.split("|")[1] #"Header - " + st
+        
+    parts = re.search( r'^(.*)\|(.+?)\s-\s(.+?)\s([\d|\.|\s|\-]+)\s+(\*.+\*)([\s\|\sOR]*)$', st)
+    if parts:
+        #print "Matched: ", parts
+        name = parts.group(3)
+        units = parts.group(4)
+        units = re.sub( r'(\d)\.000', r'\1', units)
+        units = re.sub( r'\.500', r'.5', units)
+        
+        if units=='1500 3 ': 
+            units = 3
+            name += " 1500"    # hack for HIST 4
+        return {'cn_code':parts.group(1), 'code':parts.group(2), 'name':name,
+                'units':units, 'status':parts.group(5), 'or':parts.group(6) }
+    print("*** Didn't match that class")
+    return 0
+
+# Any number gets an X (checked). Blank or zero gets no check.
+def units_to_x(u):
+    if u: return 'X'
+    return ' '
+
+def p_block_rule(r,printme,doc,out=0):
+    if printme:
+        if out: out.write("\t".join([r,'Units','Spring 19','Summer 19','Fall 19']) + "\n")
+        if not len(doc.tables):
+            t = doc.add_table(1, 5, style='Table Grid')
+        else:
+            t = doc.tables[-1]
+        t.rows[0].cells[0].text = r
+        t.rows[0].cells[1].text = 'Units'
+        t.rows[0].cells[2].text = 'Spring 19'
+        t.rows[0].cells[3].text = 'Summer 19'
+        t.rows[0].cells[4].text = 'Fall 19'
+    else:
+        if out: out.write("\t" + r + "\n")
+        t = doc.tables[-1].add_row()
+        t = doc.tables[-1].add_row()
+        t.cells[0].text = r
+    
+def p_cert_header(type,doc,r='',out=0):
+    if out: out.write("DEGREE: " + type + " (" + r + ")" + "\n")
+    if r: doc.add_heading(type + " (" + r + ")", 2)
+    else: doc.add_heading(type , 2)
+    t = doc.add_table(1, 5, style='Table Grid')
+    t.rows[0].cells[0].width = Inches(3.0)
+    #print(type)
+
+    
+def p_block_header(r,doc,out=0):
+    t = doc.tables[-1].add_row()
+    t = doc.tables[-1].add_row()
+    t.cells[0].text = r
+    if out: out.write("\t"+r+"\n" )
+
+def p_cert_course_missing(cd,doc,out=0):
+    if out: out.write(cd['code'] + " - " + cd['name'] + "\t" + cd['units'] + "\n")
+    t = doc.tables[-1].add_row()
+    t.cells[0].text = cd['code'] + " - " + cd['name']
+    t.cells[1].text = cd['units']
+    
+    
+def p_cert_course(cd,history,doc,out=0):
+    if out: 
+        line = "\t" + units_to_x(history['sp19']) + "\t" \
+               + units_to_x(history['su19']) + "\t" + units_to_x(history['fa19'])
+        out.write(cd['code'] + " - " + cd['name'] + "\t" + cd['units'] + line + "\n")
+        
+    t = doc.tables[-1].add_row()
+    t.cells[0].text = cd['code'] + " - " + cd['name']
+    if cd['or']: t.cells[0].text += " OR "
+    
+    t.cells[1].text = str(cd['units'])
+    t.cells[2].text = units_to_x(history['sp19'])
+    t.cells[3].text = units_to_x(history['su19'])
+    t.cells[4].text = units_to_x(history['fa19'])
+    #print("\t" + cd['code'] + "\t" + cd['name'] + "\t" + cd['units']+line)
+    
+    
+def p_end_block(out=0):
+    if out: out.write("\n")
+    
+def p_end_cert(bigdoc, out=0):
+    if out: out.write("\n\n\n")
+    bigdoc.add_page_break()
+
+
+def ask_for_rule(r):
+    print(("Can't find this rule: " + r)) 
+    print("""Possible answers:
+        # q - ask whats up with this rule           # u1 - u99 - that many units required (minimum. ignore max)
+        # n1 - single class required                a - all of them
+        # n2 - two classes required                 x - ignore
+        # n3                                        s - special or more logic needed
+        # n4                                        e - recommended electives""")
+    answer = input("What should it be? ").strip()
+    f= open("cache/req_phrases.txt","a+",encoding="utf-8")
+    f.write("\n" + r + "|" + answer + "|1")
+    f.close()
+    return answer
+
+def action_to_english(a):
+    if a == 'x': return 0
+    if a == 'e': return 'Electives'
+    if a == 's': return 'More logic needed / Special rule'
+    if a == 'a': return "Required - Complete ALL of the following courses:"
+    m = re.search(r'^([a-z])([\d\.]+)$',a)
+    if m:
+        if m.group(1) == 'u':
+            return "Choose %s units from the following courses: " % m.group(2)
+        if m.group(1) == 'n':
+            return "Choose %s courses from the following: " % m.group(2)
+    return 0
+
+
+# block = { rule, num } and  courses is a DataFrame
+# Return True if the courses satisfy the rule
+def check_a_block(b, courses, verbose=False):
+    indent = "    "
+    if verbose: 
+        print((indent+"Trying the rule: " + b['englrule']))
+    
+    if b['rule'] == 'all':
+        for C in b['courses']:
+            if verbose: print(C)
+            if not C[3]: 
+                if verbose: print((indent+"Failed."))
+                return False
+        return True
+    elif b['rule'] == 'min_units':
+        num = float(b['num'])
+        count = 0.0
+        for C in b['courses']:
+            if C[3]: count += C[2]
+        return count >= num
+    elif b['rule'] == 'min_courses':
+        num = float(b['num'])
+        count = 0
+        for C in b['courses']:
+            if C[3]: count += 1
+        if not count >= num:
+            if verbose: print((indent+"Failed."))
+        return count >= num
+    if b['rule'] in [ 'elective', 'special' ]: return 1
+    print("I didn't understand the rule")
+    
+    return True
+
+def read_block_english_to_code():
+    blockrules =  {}
+    for L in open('cache/req_phrases.txt','r',encoding='utf-8').readlines():
+        parts = L.strip().split('|')
+        blockrules[ parts[0] ] = [ parts[1], parts[2] ]
+    return blockrules
+    
+    
+def read_section_online_history():
+    sections = pd.read_csv('cache/one_year_course_modes.csv')    # todo: this file depends on other fxns. which?
+    sections.set_index('Unnamed: 0',inplace=True)
+    sections.sort_values('Unnamed: 0', inplace=True)
+    for i, course in sections.iterrows():
+        if course['sp19'] or course['su19'] or course['fa19']:
+            sections.loc[i,'was_online'] = 1
+        else:
+            sections.loc[i,'was_online'] = 0
+    return sections
+
+
+# Use an easy data structure (dataframes and dicts) and functions that operate on them. 
+# This is the 3rd attempt.
+def simple_find_online_programs():
+    
+    ## Step 1: Gather the relevant details. 
+    ## Read in all data, and perform whatever analysis that can be
+    ## done individually. * list of depts                       * list of which classes offered online
+    ##                    * the rules in english vs. code       * the programs themselves
+    ##
+    ##
+    ## Step 2: Do the big pass through the programs (degrees and certs). Focus on the leaves and 
+    ## branches first. Those are the individual courses and the blocks. 
+    ## Process each block on this first pass. 
+    ##
+    ## Result of each block is a dataframe (holding course info) and a dict (holding details, labels,
+    ## conclusions and analysis).
+    ## 
+    ## After the blocks, There's enough info to process the cert. Do that, and conclude if it
+    ## is online, close, or far. (Later this same pipeline will work for whether it is evening, etc...)
+    ##
+    ## Step 3: Do the second pass, and output to documents, web, or whatever other format. 
+    ##
+    ##
+    
+    # 1. Gathering data
+    section_history = read_section_online_history()   # a dataframe indexed by course codename. 
+    blockrules = read_block_english_to_code()
+    alldepts = [x[1] for x in json.loads( open('cache/programs/index.json','r').read() )['departments']]
+    
+    
+    # todo: courses with a HYPHEN in NAME get parsed wrong.
+    
+    # 2. First pass: Process blocks, certs.
+    for prog in alldepts:
+        fname = 'cache/programs/'+prog+'.json'
+        print(("Reading %s" % fname))
+        inp = open(fname,'r')
+        filedata = inp.read()
+        p_info = json.loads(filedata)
+        for p in p_info:
+            print(("  "+p['dept'] + "\t" + p['type'] + "\t" + p['title']))
+            b = p['blocks']
+            b.sort(key=cmp_2)
+            for block in b:
+                if 'rule' in block:
+                    ### RIGHT HERE - fxn to extract block to DF
+                    print(("    " + block['rule']))
+                    for_df = []
+                    for crs in block['courses']:
+                        c_data = split_course(crs)
+                        if type(c_data) is dict:
+                            c_data['code'] = re.sub(r'\s','',c_data['code'])
+                            try:
+                                c_data['was_online'] = section_history.loc[ c_data['code'] , 'was_online' ]
+                            except KeyError:
+                                c_data['was_online'] = 0
+                            for_df.append(c_data)
+                        else:
+                            print(("       ", c_data))
+                    if len(for_df):
+                        this_df = pd.DataFrame(for_df)
+                        print(this_df)
+            #input("\n\nPress enter to continue...\n\n")
+
+
+
+def check_a_block_a(b,verbose=False):
+    indent = "    "
+    if verbose: print((indent+"Trying the rule: " + b['englrule']))
+    
+    if b['rule'] == 'all':
+        for C in b['courses']:
+            if verbose: print(C)
+            if not C[3]: 
+                if verbose: print((indent+"Failed."))
+                return False
+        return True
+    elif b['rule'] == 'min_units':
+        num = float(b['num'])
+        count = 0.0
+        for C in b['courses']:
+            if C[3]: count += C[2]
+        return count >= num
+    elif b['rule'] == 'min_courses':
+        num = float(b['num'])
+        count = 0
+        for C in b['courses']:
+            if C[3]: count += 1
+        if not count >= num:
+            if verbose: print((indent+"Failed."))
+        return count >= num
+    if b['rule'] in [ 'elective', 'special' ]: return 1
+    print("I didn't understand the rule")
+    
+    return True
+    
+def smart_find_online_programs():
+
+    big_block_list = []
+
+    with ruleset('curriculum'):
+    
+        # COURSES in BLOCKS
+        @when_all(  (m.relationship == 'contains') & (+m.course)  )  
+        def show_contained_class(c):
+            #print( str(c.m.block) + " is a block that contains " \
+            #       + str(c.m.course) + " with " + str(c.m.units) + " units" ) 
+            pass
+        
+        # BLOCK Rule/Condition with and without numbers
+        @when_all( (+m.blockrule) & (+m.number) )    
+        def show_block(c):
+            #print( str(c.m.block) + " is a block that needs " + str(c.m.blockrule) + " of " + c.m.number ) 
+            big_block_list.append( [ c.m.block, "rule", c.m.blockrule, c.m.number, c.m.englrule ] )
+        
+        @when_all( (+m.blockrule) & (-m.number) )    
+        def show_block(c):
+            #print( str(c.m.block) + " is a block that needs " + str(c.m.blockrule) ) 
+            print(("++RULE: " + str(c.m)))
+            big_block_list.append( [ c.m.block, "rule", c.m.blockrule, 0, c.m.englrule ] )
+        
+        # Has course historically been OFFERED ONLINE
+        @when_all(m.sem1>0 or m.sem2>0 or m.sem3>0)
+        def is_online(c):
+            #print("Class counts as online: " + str(c.m.course))
+            c.assert_fact('curriculum', { 'course': c.m.course, 'status': 'was_offered_online', 'value': True })
+        
+        # Or NEVER ONLINE
+        @when_all(m.sem1==0 and m.sem2==0 and m.sem3==0)
+        def is_online(c):
+            #print("Class was never online: " + str(c.m.course))
+            c.assert_fact('curriculum', { 'course': c.m.course, 'status': 'was_offered_online', 'value': False })
+    
+        # Has course in the block OFFERED ONLINE?
+        @when_all( c.zero << +m.blockrule,
+                   c.first << (m.relationship == 'contains') & (m.block==c.zero.block),
+                   c.second  << (m.course == c.first.course ) & (m.status == 'was_offered_online') & (m.value==True)  )
+        def is_online_inblock(c):
+            #print("  and it was online! " + c.first.block + " / " + c.second.course)
+            #print(c.first.block + "\t" + c.first.course['code'] + "\t Yes online")
+            print("    Yes online")
+            big_block_list.append( [ c.first.block, c.first.course, c.first.units, True, c.first ] )
+        
+        # Has course in the block *NOT OFFERED ONLINE?
+        @when_all( c.three << +m.blockrule,
+                   c.four << (m.relationship == 'contains') & (m.block==c.three.block),
+                   c.five  << (m.course == c.four.course ) & (m.status == 'was_offered_online') & (m.value==False)  )
+        def is_online_inblock(c):
+            #print("  and it was online! " + c.four.block + " / " + c.five.course)
+            #print(c.first.block + "\t" + c.first.course['code'] + "\t NOT online")
+            print("    NOT online")
+            big_block_list.append( [ c.four.block, c.four.course, c.four.units, False, c.four ] )
+       
+        
+    
+    sections = pd.read_csv('cache/one_year_course_modes.csv')
+    sections.set_index('Unnamed: 0',inplace=True)
+    sections.sort_values('Unnamed: 0', inplace=True)
+    alldepts = [x[1] for x in json.loads( open('cache/programs/index.json','r').read() )['departments']]
+    
+    #history = sections.df.to_dict('index')
+    
+    print('starting...')
+    for i, course in sections.iterrows():
+        try:
+            assert_fact('curriculum', { 'course': str(i), 'sem1': int(course['sp19']), 'sem2': int(course['su19']), 'sem3':int(course['fa19']) })
+        except Exception as e:
+            pass
+    
+    blockrules =  {}
+    for L in open('cache/req_phrases.txt','r',encoding='utf-8').readlines():
+        parts = L.strip().split('|')
+        blockrules[ parts[0] ] = [ parts[1], parts[2] ]
+
+    blockindex = 0
+    
+    for prog in alldepts:
+        p_info = json.loads(open('cache/programs/'+prog+'.json','r').read())
+        for p in p_info:
+            deg_longname = p['dept'] + ' - ' + p['type'] + ' - ' + p['title']
+            print(deg_longname)
+            big_block_list.append( [ deg_longname ] )
+            
+            for block in sorted(p['blocks'],key=cmp_2):
+                if not 'rule' in block: continue
+                
+                # Look up code for what is needed with this block of classes.
+                the_rule = block['rule'].strip()
+                if not the_rule in blockrules:
+                    blockrules[ the_rule ] = [ ask_for_rule( the_rule ), 1 ]
+                
+                action = blockrules[ the_rule][0]
+                engl = action_to_english(action)
+                if not engl: continue
+                print(("  + " + engl))
+                
+                blockindex += 1
+                blocklabel = 'block_' + str(blockindex)
+                
+                
+                
+                # Assert if the courses make the block qualify
+                #print(action)
+                # needs to be a rule too....... # Required - Complete ALL of the following courses:
+                
+                #print("\n\n")
+                try:
+                    match = re.search(r'^([a-z])([\d\.]+)$',action)
+                    if action == 'a':
+                        assert_fact('curriculum', { 'block':blocklabel, 'degree': deg_longname, 'blockrule': 'all', 'englrule':engl})
+                    elif action == 'x':
+                        pass
+                    elif action == 'e':
+                        assert_fact('curriculum', { 'block':blocklabel, 'degree': deg_longname, 'blockrule': 'elective', 'englrule':engl})
+                    elif action == 's':
+                        assert_fact('curriculum', { 'block':blocklabel, 'degree': deg_longname, 'blockrule': 'special', 'englrule':engl})
+                    elif match and match.group(1) == 'u':
+                        assert_fact('curriculum', { 'block':blocklabel, 'degree': deg_longname, 'blockrule': 'min_units', 'number': match.group(2), 'englrule':engl })
+                    elif match and match.group(1) == 'n':
+                        assert_fact('curriculum', { 'block':blocklabel, 'degree': deg_longname, 'blockrule': 'min_courses', 'number': match.group(2), 'englrule':engl })
+                except MessageNotHandledException as e:
+                    pass
+                    #print(e)
+
+                    
+                for crs in block['courses']:
+                    if re.search(r'header2',crs):
+                        descr = crs.split("|")[1]
+                        big_block_list.append( [ 'header', descr ] )
+                        continue
+                    c_data = split_course(crs)
+                    #c_data['code'] = re.sub(r'\s','',c_data['code'])
+                            
+                    try:
+                        if 'code' in c_data and c_data['code']:
+                            fixed_code = re.sub(r'\s','',c_data['code'])
+                            history = sections.loc[fixed_code]
+                        else:
+                            msg = "BAD COURSE DATA: " + str(crs)
+                            data = {'code':'?','name':'?','units':'?'}
+                            continue
+                    except Exception as e:
+                        msg = "COULDNT FIND ONLINE DATA for " + c_data['code'] + " - " + c_data['name']
+                        continue
+                    #p_cert_course(c_data,history,output,doc)
+                    
+                    # Handle the class
+                    #print("\t" + str(c_data))
+                    try:
+                        print(("  Asserting " + blocklabel + "\t" + json.dumps({ 'block':blocklabel, 'course': fixed_code,
+                        'relationship': 'contains', 'units':float(c_data['units']),
+                        'code': fixed_code, 'name': c_data['name'], 
+                        'status': c_data['status'], 'or': c_data['or'] })))
+                        assert_fact('curriculum', { 'block':blocklabel, 'course': fixed_code,
+                        'relationship': 'contains', 'units':float(c_data['units']),
+                        'code': fixed_code, 'name': c_data['name'], 
+                        'status': c_data['status'], 'or': c_data['or'] })
+                    except Exception as e:
+                        pass
+                        #print(e)
+                # END block of courses 
+                
+
+        #print("Finished reading "+deg_longname)    
+        # END cert or degree    
+    eval(input('hit return...'))
+    # Big Structure of all degrees
+    degs_main = {}
+    this_deg = ''
+    for R in big_block_list: 
+        if R[0] == 'header':   # its a funny header, not quite a rule....
+            #print(R)
+            degs_main[this_deg]['blocks'].append( {'rule':'', 'englrule':'', 'courses':[], 'header':R[1] } )
+        elif not R[0].startswith('block'):                 # everything starts with block except new degrees
+            degs_main[R[0]] = { 'deg':R[0], 'blocks':[] }
+            this_deg = R[0]
+            #print(this_deg)
+        elif R[1] == 'rule':
+            degs_main[this_deg]['blocks'].append( {'rule':R[2], 'englrule':R[4], 'courses':[], 'header':'' } )
+            #print("  "+R[4])
+            if len(R) > 3:
+                degs_main[this_deg]['blocks'][-1]['num'] = R[3]
+        else:
+            degs_main[this_deg]['blocks'][-1]['courses'].append(R)
+            #print("    "+str(R))
+    
+    # Print them
+    bigdoc = Document()
+    for k,v in list(degs_main.items()):
+    
+        print((v['deg']))        
+        qualifies = True
+        if not re.search(r'chievement',v['deg']):
+            qualifies = False                               ## JUST DOING CAs
+            print("  Skipping because not a CA")
+        if not qualifies: continue
+        for vv in v['blocks']:
+            for CC in vv['courses']:
+                print(("    " + "\t".join([ CC[0], CC[1], str(CC[3]), CC[4]['name']])))
+            if not check_a_block_a(vv,1): 
+                qualifies = False
+                break
+        if not qualifies: continue
+        
+        print("  + OK, including this one.")
+        
+        bigdoc.add_heading('Gavilan College', 2)
+        #bigdoc.add_heading(v['deg'], 2)
+        p_cert_header(v['deg'],bigdoc)
+        print_headers = 1
+        
+        for vv in v['blocks']:
+            p_block_rule(vv['englrule'],print_headers,bigdoc)
+            print_headers = 0
+            
+            more = ''
+            if 'num' in vv: more = ' / ' + str( vv['num'] )
+            
+            #print( "    " + vv['rule'] + more )
+            if vv['header']: 
+                p_block_header(vv['header'],bigdoc)
+                #print("        ("+vv['header']+")")
+            for vvv in vv['courses']:
+                #print(vvv[4])
+                #print(vvv)
+                #print("        " + json.dumps(vvv))
+                p_cert_course(vvv[4], sections.loc[ vvv[1] ],bigdoc)
+        p_end_cert(bigdoc)
+    bigdoc.save('output/onlinecerts/all_cert_achievement.docx')
+ 
+
+
+
+# 9/2021 clean programs to good json
+def organize_programs_stage2():
+    alldepts = [x[1] for x in json.loads( open('cache/programs/index.json','r').read() )['departments']]
+    output = codecs.open('cache/deg_certs.json','w','utf-8')
+    
+    all_progs = []
+    
+    for prog in alldepts:
+        fname = 'cache/programs/'+prog+'.json'
+        print(("Reading %s" % fname))
+        filedata = open(fname,'r').read()
+        p_info = json.loads(filedata)
+        
+        
+        for p in p_info:
+            pretty_p = {}
+            print(p['dept'] + "\t" + p['type'] + "\t" + p['title'])
+            pretty_p['title'] = p['title']
+            pretty_p['dept'] = p['dept']
+            if 'desc' in p: pretty_p['desc'] = p['desc']
+            if 'type' in p: pretty_p['type'] = p['type']
+            print(" - %s\n  - %s\n" % (p['dept'],p['title']))
+            pretty_p['groups'] = []
+            
+            b = p['blocks']
+            b.sort(key=cmp_2)
+            for block in b:
+                this_block = {'courses':[],'header':""}
+                if 'rule' in block:
+                    #print("\t"+block['order'] + "\t" + block['rule'])
+                    #p_block_rule(block['rule'],output,print_headers,doc)
+                    this_block['header'] = block['rule']
+                    
+                    for crs in sorted(block['courses']):
+                        if re.search(r'header2',crs):
+                            if len(this_block['courses']):
+                                pretty_p['groups'].append(this_block)
+                                this_block = {'courses':[],'header':""}
+                            parts = crs.split("|")
+                            #print(parts)
+                            this_block['header'] = parts[1]
+                            continue
+                        c_data = split_course(crs)  
+                        if type({})==type(c_data) and 'code' in c_data:
+                            code = c_data['code']
+                        if type({})==type(c_data) and 'or' in c_data and c_data['or']: code += " or"
+                        
+                        if c_data:
+                            this_block['courses'].append( [ code,c_data['name'],c_data['units'] ])
+                        # a string or a dict
+                        # {'cn_code':parts.group(1), 'code':parts.group(2), 'name':parts.group(3),
+                        # 'units':parts.group(4), 'status':parts.group(5), 'or':parts.group(6) }
+                pretty_p['groups'].append(this_block)
+            all_progs.append(pretty_p)
+    output.write(json.dumps( all_progs,indent=2))
+
+    
+# of all the programs, what can be accomplished online?    
+def find_online_programs():
+    #sections = summarize_online_sections()
+    sections = pd.read_csv('cache/one_year_course_modes.csv')
+    sections.set_index('Unnamed: 0',inplace=True)
+    
+    bigdoc = Document()
+    #bigdoc.styles.add_style('Table Grid', docx.styles.style._TableStyle, builtin=True)
+    
+    alldepts = [x[1] for x in json.loads( open('cache/programs/index.json','r').read() )['departments']]
+    
+    for prog in alldepts:
+    
+        #prog = 'administration_of_justice'
+        fname = 'cache/programs/'+prog+'.json'
+        print(("Reading %s" % fname))
+        input = open(fname,'r')
+        filedata = input.read()
+        p_info = json.loads(filedata)
+        #print p_info
+        
+        output = open('output/onlinecerts/'+prog+'.txt','w')
+        for p in p_info:
+            #print(p['dept'] + "\t" + p['type'] + "\t" + p['title'])
+            
+            if re.search(r'chievement',p['type']):
+                use_bigdoc = 1
+                bigdoc.add_heading('Gavilan College', 2)
+                bigdoc.add_heading(p['dept'], 2)
+                p_cert_header(p['type'],bigdoc,p['title'],output)
+
+            else:
+                use_bigdoc = 0
+            
+            #doc = Document()
+            #doc.add_heading('Gavilan College', 2)
+            #p_cert_header(p['type'],p['title'],output,doc)
+            b = p['blocks']
+            b.sort(key=cmp_2)
+            
+            print_headers = 1
+            
+            for block in b:
+                if 'rule' in block:
+                    #print("\t"+block['order'] + "\t" + block['rule'])
+                    #p_block_rule(block['rule'],output,print_headers,doc)
+                    if use_bigdoc: p_block_rule(block['rule'],output,print_headers,bigdoc)
+
+                    print_headers = 0
+                    for crs in block['courses']:
+                        if re.search(r'header2',crs):
+                            parts = crs.split("|")
+                            #p_block_header(parts[1],output,doc)
+                            if use_bigdoc: p_block_header(parts[1],output,bigdoc)
+                            continue
+                            
+                        c_data = split_course(crs)
+                        try:
+                            if 'code' in c_data and c_data['code']:
+                                fixed_code = re.sub(r'\s','',c_data['code'])
+                                history = sections.loc[fixed_code]
+                            else:
+                                print(("BAD COURSE DATA: " + str(crs)))
+                                #p_cert_course_missing({'code':'?','name':'?','units':'?'},output,doc)
+                                if use_bigdoc: p_cert_course_missing({'code':'?','name':'?','units':'?'},output,bigdoc)
+                                continue
+                        except Exception as e:
+                            #print("COULDNT FIND ONLINE DATA for " + c_data['code'] + " - " + c_data['name'])
+                            #p_cert_course_missing(c_data,output,doc)
+                            if use_bigdoc: p_cert_course_missing(c_data,output,bigdoc)
+                            #print(e)
+                            continue
+                        #print("\t\t[", crs, "]")
+                        #print("\t\t", c_data)
+                        #p_cert_course(c_data,history,output,doc)
+                        if use_bigdoc: p_cert_course(c_data,history,output,bigdoc)
+                #p_end_block(output)
+            if use_bigdoc: p_end_cert(output,bigdoc)
+            #doc_title = re.sub(r'\/','_',p['title'])
+            #doc.save('output/onlinecerts/'+prog+'_' + doc_title + '.docx')
+    bigdoc.save('output/onlinecerts/all_ca.docx')    
+        
+        
+# take a string of all the types of classes offered, return a vector of [tot,lec,hyb,onl]
+def string_to_types(st):
+    l,h,o,s = (0,0,0,0)
+    for p in st.split(','):
+        s += 1
+        if p == 'online': o+=1
+        elif p == 'face to face': l += 1
+        elif p == 'hybrid': h += 1
+        #else: print "Didn't catch this: ", p
+    return [s,l,h,o]
+    
+def my_default_counter():
+    temp = {}
+    for S in sems:
+        temp[S] = 0
+    return temp
+    #return {'units':'','Spring 19':0,'Summer 19':0,'Fall 19',0}
+    
+# Of the recent schedules, what was actually offered online?
+def summarize_online_sections():
+    scheds = list(map(getSemesterSchedule,sems))
+    all = pd.concat(scheds,sort=True)
+    selected = all[['code','type','sem']]
+    selected.to_csv('cache/one_year_course_sections.csv')
+    
+    # Count the online sections offered by semester
+    counter = defaultdict(my_default_counter)
+    for index,row in selected.iterrows():
+       # print(row)
+        code = row['code']
+        code = re.sub('\s','',code)
+        entry = counter[code]
+        if row['type'] == 'online':
+            entry[ row['sem'] ] += 1
+    df_counter = pd.DataFrame.from_dict(counter,orient='index')
+    #print(df_counter)
+    df_counter.to_csv('cache/one_year_course_modes.csv')
+    #return df_counter
+    
+    bycode = selected.groupby('code')
+    try:
+        ff = bycode.agg( lambda x: string_to_types(','.join(x)) )
+    except Exception as e:
+        print("There was a problem with the schedules. One may not have the 'type' column.")
+        print("Check 'cache/one_year_course_modes.csv' for details")
+        return
+        
+    types_by_course = {}
+    for row_index, row in ff.iterrows():
+        types_by_course[row_index.replace(" ","")] = row['type']
+    
+    df = pd.DataFrame.from_dict(types_by_course,orient='index',columns=['sections','lec','hyb','online'])
+    #print(df)
+    df.to_csv('cache/one_year_online_courses.csv')
+    print("Saved to cache/one_year_online_courses.csv")
+    return df
+
+def fibonacci(n):
+    return match(n,
+        1, 1,
+        2, 1,
+        _, lambda x: fibonacci(x-1) + fibonacci(x-2)
+    )
+
+
+def test_pampy():
+    for i in [1,2,3,4,5,7,9,15]:
+        print(("fib(%i) is: %i" % (i,fibonacci(i))))
+    
+
+def cq_parse_experiment(root=0, indent=''):
+    # call this on anything that's a list. It'll recurse on each element of it. 
+    
+    # if the value was false, roll it back up and dont
+    # display
+    
+    ret = ''
+    if type(root) == type({}):
+        ret += indent + "{"
+        for K,V in list(root.items()):
+            ret +=  K + ": " + \
+                   cq_parse_experiment(V,indent+" ")+ ", " +indent
+        ret += "}"
+        
+    elif type(root) == type([]):
+        for K in root:
+            ret += "[" + cq_parse_experiment(K, indent+" ") + "]"
+    
+    elif type(root) == type("abc"): ret += root
+    elif type(root) == type(55): ret += str(root)
+    elif type(root) == type(5.5): ret += str(root)
+    else: ret += str(root)
+    
+    return ret
+            
+def cq_start():
+    root = json.loads( open('cache/programs/programs_1.txt','r').read())
+    outt = open('cache/test_prog.txt','w')
+    outt.write(cq_parse_experiment(root,'\n'))
+
+"""my first pattern
+
+"dataTypeDetails": {
+        "scale": 2, 
+        "type": "numeric", 
+        "precision": 6
+    }, 
+                                                                    
+
+
+
+
+
+
+def cq_pattern_backup1(root=0, indent=''):
+    # call this on anything that's a list. It'll recurse on each element of it. 
+    
+    # if the value was false, roll it back up and dont
+    # display
+    
+    ret = ''
+
+    #  xxxx    Rules here catches them top-down
+    
+    if type(root) == type({}):
+        ret += indent + "{"
+        for K,V in list(root.items()):
+            ret +=  '"'+K+'"' + ": " + \
+                   str(cq_pattern(V,indent+" "))+ ", " +indent
+        ret += "}"
+        
+    elif type(root) == type([]):
+        for K in root:
+            ret += "[" + str(cq_pattern(K, indent+" ")) + "]"
+    
+    elif type(root) == type("abc"): ret += '"'+root+'"'
+    elif type(root) == type(55): ret += str(root)
+    elif type(root) == type(5.5): ret += str(root)
+    elif type(root) == type(False):
+        if root == False: return "False"
+        elif root == True: return "True"
+    else: 
+        result = lookForMatch(pat,rule)
+        if result: ret = str(result)
+        else: ret += '"'+str(root)+'"'
+    
+    return ret
+
+
+
+"""
+
+
+
+def found(*x): 
+    #print(len(x))
+    print(x)
+    
+    return str(x)
+
+def lookForMatch(rules,item):
+    var1 = ''
+    for i,x in enumerate(rules):
+        if i % 2 == 1:
+            a = match(item, var1, x, default='')
+            if a:
+                labels[i-1 / 2] += 1
+                break
+        else:
+            var1 = x
+    
+    
+    #a = match(root,*curic_patterns,default='')
+    if a: 
+        #print("Matched: " + str(a))
+        return a
+    #print("Didn't match: " + str(item) + "\n")
+    return False
+    
+    
+    
+#from curriculum_patterns import pat
+from patterns_topdown import pat
+
+labels = defaultdict(int)
+
+def cq_pattern(root=0, indent=''):
+    # call this on anything that's a list. It'll recurse on each element of it. 
+    
+    # if the value was false, roll it back up and dont
+    # display
+    
+    ret = ''
+
+    #  xxxx    Rules here catches them top-down
+    # instead we'll do each element of data structure, and then try to match the whole thing.
+    
+    if type(root) == type({}):
+        ret = {}
+        for K,V in list(root.items()):
+            ret[K]  = cq_pattern(V,indent+" ")
+        
+    elif type(root) == type([]):
+        ret = []
+        for K in root:
+            ret.append(cq_pattern(K, indent+" "))
+    
+    elif type(root) == type("abc"): ret = root          # ret += '"'+root+'"'
+    elif type(root) == type(55): ret = root          # ret += str(root)
+    elif type(root) == type(5.5): ret = root          # ret += str(root)
+    elif type(root) == type(False):
+        if root == False: ret = root          # return "False"
+        elif root == True: ret = root          # return "True"
+
+    result = lookForMatch(pat,root)
+    if result: ret = result
+    
+    if ret: return ret
+    return root
+    
+    
+
+def myprinter(item, indent=''):
+    if type(item) == type( ('a','b') ) and len(item)==2 and type(item[0])==type('a') and type(item[1])==type( {"a":2} ):
+        return "[[" + item[0] + ": " + ('\n'+indent+'  ').join( [ K+":> "+ myprinter(V,indent+"    ") for K,V in item[1].items() ] ) + "]]"
+    if type(item) == type( {} ):
+        return "{" + ('\n'+indent+'  ').join( [ K+": "+ myprinter(V,indent+"    ") for K,V in item.items() ] )+"}"
+    if type(item) == type( [] ):
+        return "[" + ('\n'+indent+'  ').join( [ myprinter(I,indent+"    ") for I in item ] )+"]"
+    return '"|'+str(item)+'|"'
+
+
+def cq_pattern_start():
+    root = json.loads( open('cache/programs/programs_2.txt','r').read())
+    outt = open('cache/test_prog.txt','w')
+    
+    result = cq_pattern(root,'\n')
+    for R in result:
+        outt.write(myprinter(R)+"\n")
+    
+    k_srt = sorted(labels.keys())
+    
+    for k in k_srt:
+        v = labels[k]
+        print("  Slot %i:\t%i hits" % (k/2,v))
+
+def baby_int(j):
+    if j=='': return 0
+    return int(j)
+
+def find_deg_in_cluster( clusters, deg ):
+    for k,v in clusters.items():
+        if deg in v: return k
+    return "pathway_not_found"
+
+def try_match_deg_programs():
+    # my index, from curricunet, is the "longname". The 'attained' file has medium. kind of. 
+    
+    type_lookup = { "Certificate of Proficiency":"CP", "A.A. Degree":"AA", "A.S. Degree":"AS", "Certificate of Achievement":"CA", "A.S.-T Degree":"AS_T", "A.A.-T Degree":"AA_T", "NC-Cmptncy: NC Certificate of Competency":"COMP", "NC-Complet: NC Certificate of Completion":"COMP"  }
+    
+    # Curricunet
+    curicunet_version = {}
+    for f in os.listdir('cache/programs'):
+        if not re.search('index|programs',f):
+            #print(f)
+            pro = json.loads(open('cache/programs/'+f,'r').read())      #  blocks title dept key type desc
+            for c in pro:
+                longname = c['dept'] + " | " + c['title'] + " | " + c['type']
+                curicunet_version[longname] = 0
+                abbrev = "??"
+                if c['type'] in type_lookup:
+                    abbrev = type_lookup[ c['type'] ]
+                #print("  " + abbrev + ":   " + longname)
+
+    # for each in 'attained' list, try to match to correspondent in variants/long list.
+    #
+    #
+    
+    gp_clusters = {}
+    current_cluster = "X"
+    gp_file = open('cache/g_path_cluster2020.txt','r')
+    for L in gp_file:
+        L = L.strip()
+        if L:
+            if L.startswith('#'):
+                mch = re.search(r'^\#\s(.*)$',L)
+                if mch:
+                    current_cluster = mch.group(1)
+                    gp_clusters[ current_cluster ] = []
+            else:
+                gp_clusters[ current_cluster ].append( L )
+    
+    #print( gp_clusters )
+    #x = input('paused')
+    
+    matchers = csv.reader(open('cache/deg_name_variants.csv','r'),delimiter=",")
+    by_long = {}
+    by_medium = {}
+    by_med_unmatched = {}
+    by_gp_name = {}
+    line = 0
+    for row in matchers:   # variants
+        if line==0: 
+            pass
+        else:
+            by_long[ row[3] ] = row
+            by_gp_name[ row[2] ] = row
+            by_medium[ row[1] ] = row   # # # **  
+            by_med_unmatched[ row[1] ] = row   # # # **  
+            #if row[1]: print(row[1])
+            
+            # remove from curricunet list so i can see whats left
+            if row[3] in curicunet_version:
+                curicunet_version[ row[3] ] = 1
+        line += 1
+    
+    by_medium[''] = [0,0,0,0,0,0,0,0,0,0]
+    #print(by_medium)
+    # Attained List
+    attained = csv.reader(open('cache\degrees_attained.csv','r'),delimiter=",")    #  1  6  22  17
+    
+    line = 0
+    matched = {}
+    unmatched = {}
+    #print("These ones I can't match.")
+    
+    for row in attained:
+        if line==0:
+            attained_columns = row
+            attained_columns.append("total")
+            attained_columns.insert(0,"shortname")
+            attained_columns.insert(0,"pathway")
+            attained_columns.insert(0,"dept")
+            attained_columns.insert(0,"type")
+            
+            attained_columns.insert(5,"longname")
+        else:
+            row.insert(0,'sn')
+            row.insert(0,'p')
+            row.insert(0,'d')
+            row.insert(0,'t')
+            row.insert(5,'')
+            #print("Matching by medium name: %s" % str(row))
+            #print("Matched to: %s" % str(by_medium[ row[4] ]) )
+            
+            matching_longname = by_medium[row[4]][3]
+            
+            if len(matching_longname):
+                #print("matching longname: %s" % matching_longname)
+                row[5] = matching_longname    ### THE matching longname
+                m_parts = matching_longname.split(" | ")
+                dept = m_parts[0]
+                ttype = m_parts[2]
+                row[1] = dept
+                row[0] = ttype
+                matched[row[4]] = row
+                
+                row[3] = by_medium[row[4]][0]    # shortname
+                
+                row[2] = find_deg_in_cluster(gp_clusters, by_medium[row[4]][2])
+                
+                print("OK: " + str(row))
+            else:
+                row[0] = ''
+                row[1] = ''
+                row[2] = ''
+                row[3] = ''
+                row[5] = ''
+                print("XX: " + str(row))
+                unmatched[row[4]] = row
+        line += 1
+    print("matched %i and missed %i." % (len(matched),len(unmatched)))
+    #print("\nactually missed %i." % len(by_med_unmatched))
+    
+    print("\nLeftover degrees:")
+    for k,v in curicunet_version.items():
+        if not v: print(k)
+    
+    
+    mash_cols = "type dept pathway shortname mediumname longname grad09 10 11 12 13 14 15 16 17 18 total".split(" ")
+    mash_rows = []
+    
+    # attained / matched
+    for xrow in matched.values():
+        mash_rows.append(xrow)
+        
+    # attained / unmatched
+    for xrow in unmatched.values():
+        mash_rows.append(xrow)
+    
+    # curricunet leftovers
+    
+    
+    mydf = pd.DataFrame(mash_rows, columns=mash_cols)
+    mydf.to_csv('cache/attainment_masterlist.csv',index=False)
+    return
+
+
+
+# open('cache/programs/programs_1.txt','r').read()
+
+"""   SEE serve.py   .... i mean ...  interactive.py
+def dict_generator(indict, pre=None):
+    pre = pre[:] if pre else []
+    if isinstance(indict, dict):
+        for key, value in indict.items():
+            if isinstance(value, dict):
+                for d in dict_generator(value, pre + [key]):
+                    yield d
+            elif isinstance(value, list) or isinstance(value, tuple):
+                for v in value:
+                    for d in dict_generator(v, pre + [key]):
+                        yield d
+            else:
+                yield str(pre) + "   " + str([key, value]) + "\n"
+    else:
+        yield pre + [indict]
+        yield str(pre) + "   " + str([indict]) + "\n"
+        
+ 
+ 
+def print_dict(v, prefix='',indent=''):
+    if isinstance(v, dict):
+        return [   print_dict(v2, "{}['{}']".format(prefix, k) + "<br />", indent+" " )  for k, v2 in v.items()  ]
+    elif isinstance(v, list):
+        return [ print_dict(  v2,  "{}[{}]".format(prefix , i) + "<br />", indent+" ")  for i, v2 in enumerate(v) ]
+    else:
+        return '{} = {}'.format(prefix, repr(v)) + "\n"
+        
+
+def walk_file():
+    j = json.loads(open('cache/programs/programs_2.txt','r').read())
+    
+    return print_dict(j)
+
+from flask import Flask
+from flask import request
+
+def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+def server_save(key,value):
+    codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
+    
+def flask_thread(q):
+    app = Flask(__name__)
+
+    @app.route("/")
+    def home():
+        return tag('h1','This is my server.') + "<br />" + a('want to shut down?','/sd')
+
+    @app.route("/save/<key>/<val>")
+    def s(key,val):
+        server_save(key,val)
+        return tag('h1','Saved.') + "<br />" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
+
+    @app.route("/crazy")
+    def hello():
+        r = '<link rel="stylesheet" href="static/bootstrap.min.css">'
+        r += tag('style', 'textarea { white-space:nowrap; }')
+        r += tag('body', \
+            tagc('div','container-fluid', \
+                tagc('div','row', \
+                    tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
+                    tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
+        
+         
+        
+        return r
+    
+    @app.route("/sd")
+    def sd():
+        print('SIGINT or CTRL-C detected. Exiting gracefully')
+        func = request.environ.get('werkzeug.server.shutdown')
+        if func is None:
+            raise RuntimeError('Not running with the Werkzeug Server')
+        func()
+        return "Server has shut down."
+    app.run()
+    
+    
+from queue import Queue
+
+q = Queue()
+
+def serve():
+    import webbrowser
+    import threading
+    x = threading.Thread(target=flask_thread, args=(q,))
+    x.start()
+    webbrowser.open_new_tab("http://localhost:5000")
+    
+                
+    
+    
+    #s = open('cache/programs/index.json','w')
+    #s.write( json.dumps({'departments':sorted(list(dept_index)), 'programs':prog_index}, indent=2) )
+    #s.close()
+"""
+
+
+
+
+
+# feb 2020 goal: 
+# - treemap of graduates in each division, dept, pathway, degree type, major
+# - rollover or other interactive explorer of pathways
+
+
+# sept short term goals:
+# 1. viable presentation on web pages w/ good overview
+# 1a.  - will necessarily include courses, learning outcomes cause it depends on them. 
+# 2. show progs close to 50% limit
+# 3. foundation for visualization, model degree attainment, and simulation
+# 4. prep for work on iLearn -> SLO -> any useful contributions I can make
+#
+
+
+# sept 8, 2020 approach:
+# 1 hr: pull labels, types, interesting notes, discern structures that are most
+#       important. The 20% that gets me 80%. 
+#
+# 1 hr: 2-3 short experiments for different ways of pattern matching them.
+#
+# 1 hr: best (rushed) effort to condense it all into accurate (if incomplete) 
+#       compact data structure. 
+#
+# 1 hr: php to fetch and display for a given prog, deg, dept, cert, or overview. 
+#
+# show draft live on wed sept 10. 
+# 
+
+
+
+
+"""
+def attempt_match8020(rules,item):
+    var1 = ''
+    for i,x in enumerate(rules):
+        if i % 2 == 1:
+            a = match(item, var1, x, default='')
+            if a:
+                labels8020[i-1 / 2] += 1
+                
+        else:
+            var1 = x
+    
+    
+    #a = match(root,*curic_patterns,default='')
+    if a: 
+        print("Matched: " + str(a))
+        return a
+    print("Didn't match: " + str(item) + "\n")
+    return False
+"""    
+
+
+def clever_printer(item, indent=''):
+    if type(item) == type( ('a','b') ) and len(item)==2 and type(item[0])==type('a') and type(item[1])==type( {"a":2} ):
+        return "[[" + item[0] + ": " + ('\n'+indent+'  ').join( [ K+":> "+ myprinter(V,indent+"    ") for K,V in item[1].items() ] ) + "]]"
+    if type(item) == type( {} ):
+        return "{" + ('\n'+indent+'  ').join( [ K+": "+ myprinter(V,indent+"    ") for K,V in item.items() ] )+"}"
+    if type(item) == type( [] ):
+        return "[" + ('\n'+indent+'  ').join( [ myprinter(I,indent+"    ") for I in item ] )+"]"
+    return '"|'+str(item)+'|"'
+
+
+def print_return(x):
+    print('got a hit')
+    print()
+    return x
+    
+    
+from patterns_8020 import pat8020
+
+labels8020 = defaultdict(int)
+
+def cq_8020(root=0, indent=''):
+    # Try to match the root, and if no match, try to break it up  ( dicts, lists ) and
+    # recurse on those parts.
+    
+    # if no matches below this point in the tree, return false
+    ret = []
+    
+    for pattern in pat8020:
+        m = match( root, pattern, print_return )
+        if m: 
+            print('case 1')
+            print('this:   ' + str(m))
+            print('matched this pattern: ' + str(pattern))
+            print(print_return)
+            
+            xyz = input('enter to continue')
+            ret.append(m)
+
+    """
+    if type(root) == type({}):
+        for K,V in list(root.items()):
+            m = cq_8020(V)
+            if m: 
+                print('case 2')
+                ret.append(m)
+        
+    elif type(root) == type([]):
+        for V in root:
+            m = cq_8020(V)
+            if m: 
+                print('case 3')
+                ret.append(m)"""
+    
+    return ret 
+
+
+
+
+def cq_8020_start():
+    """ (programs) entityType  entityTitle   status   proposalType    sectionName      lastUpdated      lastUpdatedBy    
+        fieldName     displayName     lookUpDisplay    fieldValue     instanceSortOrder
+        lookUpDataset   (array of dicts, each has keys: name, value, and corresponding values.)
+        
+        subsections or fields (arrays) - ignore for now just takem in order
+        
+        (courses) same as above?  
+        
+        html values: markdown convert?
+        
+    """
+    root = json.loads( open('cache/programs/programs_2.txt','r').read())
+    outt = open('cache/test_prog8020.txt','w')
+    
+    result = cq_8020(root,'\n')
+    
+    outt.write( json.dumps( result, indent=2 ) )
+    
+
+
+
+
+##### Restored from an earlier version
+
+def recurse3(sec,path=''):
+    output = ''
+    if 'subsections' in sec and len(sec['subsections']):
+        for subsec in sec['subsections']:
+            #pdb.set_trace()
+            id = get_id_sortorder(subsec)
+            output += recurse3(subsec, path + subsec['attributes']['sectionName'] + " ("+id+") | ")
+    if 'fields' in sec and len(sec['fields']):
+        for subfld in sec['fields']:
+            try:
+                fld = handleField(subfld)
+                if fld:
+                    dbg('Field: %s' % str(fld))
+                    output += path + subfld['attributes']['fieldName'] + " | " + fld + "\n"
+            except Exception as e:
+                print("Problem in field: %s"% str(e))
+                print(subfld)
+                x = input('enter to continue')
+    return output
+
+
+def get_id_sortorder(sec):
+    ord = ''
+    if 'instanceSortOrder' in sec['attributes']:
+        ord = str(sec['attributes']['instanceSortOrder'])
+    if 'sectionSortOrder' in sec['attributes']:
+        ord = str(sec['attributes']['sectionSortOrder'])
+    if ord and int(ord)<10: ord = '0'+ord
+    if 'instanceId' in sec['attributes']:
+        return ord + '-' + str(sec['attributes']['instanceId'])
+    elif 'sectionSortOrder' in sec['attributes']:
+        return ord + '-' + str(sec['attributes']['sectionSortOrder'])
+    else: return ord
+
+
+def include_exclude(str,inc,exc=[]):
+    # True if str contains anything in inc, and does not contain anything in exc
+    good = False
+    for i in inc:
+        if i in str: good = True
+    if not good: return False
+    for e in exc:
+        if e in str: return False
+    return True
+
+def pbd3(str):
+    # get the id from the 'Program Block Definitions' in the 3rd position
+    p = str.split("|")
+    if len(p)>3: str = p[2]
+    m = re.search(r'Program\sBlock\sDefinitions\s\(([\-\d]+)\)',str)
+    if m:
+        if m.group(1) != '0': 
+            return m.group(1)
+    return 0
+
+def handleField(f):
+    lud = ''
+    if 'lookUpDisplay' in f: lud = boolToStr(f['lookUpDisplay'])
+    #fv = unicode(f['fieldValue']).replace('\n', ' ').replace('\r', '')
+    fv = str(f['fieldValue']).replace('\n', ' ').replace('\r', '')
+    if not lud and not fv: return False
+    
+    return f['attributes']['fieldName'] + ': ' + lud + " / " + fv
+
+def boolToStr(b):
+    if isinstance(b,bool): 
+        if b: return "True"
+        return "False"
+    return b
+
+# Almost final formatting
+def prog_info_to_entry(c):
+    out = {}
+    p1 = c.split(" | ")
+    if p1[2]=="Program Title": 
+        print(p1[3][18:])
+        return {'title':p1[3][18:]}
+    if p1[2]=="Department": 
+        d =  p1[3][12:]
+        prt = d.split(" / ")
+        return {'dept':prt[0]}
+    if p1[2]=="Award Type": 
+        return {'type':p1[3][12:].split(' /')[0]}
+    if p1[2]=="Description":
+        desc = p1[3][16:]
+        soup = bs(desc, 'html.parser')
+        for s in soup.find_all('span'): s.unwrap()
+        for e in soup.find_all(True):
+            e.attrs = {}
+        dd = str(soup)
+        dd = re.sub('\u00a0',' ',dd)
+        return {'desc':dd}
+    return {}
+
+def cbd_to_entry(c):
+    parts = c.split(" | ")
+    if parts[3]=='Course Block Definition':
+        p2 = parts[4].split(" / ")
+        return { 'rule':p2[1] }
+    return {}
+
+def pc5(str):
+    # get the id from the 'Program Courses' in the 5th position
+    p = str.split("|")
+    if len(p)>5: str = p[4]
+    m = re.search(r'Program\sCourses\s\(([\-\d]+)\)',str)
+    if m:
+        if m.group(1) != '0': 
+            return m.group(1)
+    return 0
+
+def remove_prefix(str,i):
+    p = str.split(" | ")
+    if len(p) > i:
+        return " | ".join(p[i:])
+    return str
+
+def course_to_entry(c,order="0"):
+    p1 = c.split(" | ")
+    dbg("    c2e: %s" % str(c))
+    if p1[1] == "Course":
+        p2 = p1[2].split(" / ")
+        origname = order+"|"+p2[0][8:]
+        id = p2[1]
+        #return {'id':id,'origname':origname}
+        dbg("    c2e is course: %s" % str(origname))
+        return origname
+    if p1[1] == "Condition":
+        #print p1[2][11:13]
+        if p1[2][11:13] == 'or':
+            #return {'ornext':1}
+            dbg("    c2e is OR")
+            return " | OR "
+    if p1[0] == "Non-Course Requirements":
+        #pdb.set_trace()
+        dbg("    c2e is header: %s" % str(p1[1][28:]))
+        return order + "header2" + "|" + p1[1][28:]
+    return ''      
+
+def courseline_to_pretty(line):
+    # from this: 01-125780|THEA1 - Theatre History: Greece to Restoration 3.000 *Active*
+    #            09-125764|THEA19 - Acting and Voice for TV/Film/Media 3.000 *Historical* | OR 
+    #            11-129282header2|Choose additional units from the courses below to complete the unit minimum requirement:
+    # to decent looking
+    return line
+    out = ''
+    oor = 0
+    #pdb.set_trace()
+    parts = line.split("|")
+    if re.search('header2',parts[0]):
+        out = "<div class='secondary'>" + parts[1] + "</div>"
+    elif len(parts) > 2 and parts[2]==" OR":
+        oor = 1
+    m = re.search(r'(.*)\s\-\s(.*)\s([0-9{1,3}\.\s\-]+)\s\*(\w*)\*',parts[1])
+    if m:
+        code = m.group(1)
+        name = m.group(2)
+        units = m.group(3)
+        active = m.group(4)
+        if oor: name += "<strong>OR</strong>"
+        out = "<div class='aclass'><span class='cl_code'>"+code+"</span><span class='cl_name'>"+name+"<span class='cl_unit'>"+units+"</span></div>"
+    return out
+
+
+
+# restarted oct 2019 and try to simplify
+def prog_take_4(program):
+    fullyProcessed = ''
+    for r in program['entityFormData']['rootSections']:
+        dbg('a recurse3 call...')
+        fullyProcessed += recurse3(r,program['entityMetadata']['entityTitle']+" | ")
+    taken = []
+    
+    for L in (program['entityMetadata']['entityTitle'] + fullyProcessed).split('\n'):
+        if include_exclude(L,['Description','Department','Award Type','Program Title','Course Block Definition','Program Courses','Outcome | Outcome | Outcome | Outcome'], ['Map SLO to']):
+            taken.append(L)
+    program_struct = { 'blocks':[]}
+    # start dividing up course blocks
+    blocks = groupby(pbd3,taken)
+    for k,v in blocks.items():      # each of the PDBs
+        block = { 'order':str(k)  }
+        for a in v:
+            dbg('block:  ' + str(k))
+            course_list = []
+            if k == 0:
+                program_struct.update(prog_info_to_entry(a))            
+            else:
+                #pdb.set_trace()
+                block.update(cbd_to_entry(a))
+                courses = groupby(pc5,blocks[k])
+                for C,cval in courses.items():   # each of the courses
+                    df = [remove_prefix(x,5) for x in cval]
+                    #my_c = { 'order':str(C) }
+                    courseline = ''
+                    for K in df: 
+                        c2e = course_to_entry(K,C)
+                        dbg("    c2e: %s" % str(c2e))
+                        if re.search('header2',c2e):
+                            course_list.append( courseline_to_pretty(courseline))
+                            courseline = c2e
+                            continue
+                        if re.search('3\sUnit\sMin',c2e): 
+                            dbg("    --courseline: %s" % str(courseline))
+                            courseline = re.sub('1\.000\s+\-\s+2\.000','3.000',courseline)
+                            dbg("    ---courseline changed: %s" % str(courseline))
+                            continue    # hack for span non native opt 2
+                            # TODO
+                        courseline += c2e
+                        dbg("    courseline: %s" % str(courseline))
+                        #if courseline: 
+                        #    my_c.update(courseline)
+                        #    #if 'id' in my_c and my_c['id'] in ids:
+                        #    #    my_c['reference'] = ids[my_c['id']]
+                    dbg('--new courseline--')
+                    if courseline: course_list.append( courseline_to_pretty(courseline))
+        block['courses'] = sorted(course_list)
+        if block: program_struct['blocks'].append(block)    
+    #jsonout.write( json.dumps(program_struct,indent=2) )
+    #return '\n'.join(taken)
+    return program_struct
+
+
+
+
+
+if __name__ == "__main__":
+
+    #cq_8020_start()
+    #exit()
+    
+    print ('')
+    options = { 1: ['Fetch all class data from curricunet',fetch_all_classes] ,  
+                2: ['Fetch all program data from curricunet', fetch_all_programs] ,
+                3: ['Translate class data to condensed json files', show_classes] ,
+                4: ['Translate program data to condensed json files', show_programs] ,
+                5: ['Try to consolidate lists of programs and degrees and # attained', try_match_deg_programs] ,
+                #5: ['Check DE', check_de] ,
+                #6: ['Sort courses', organize_courses] ,
+                #7: ['Clean up degree/program entries', clean_programs] ,
+                #8: ['Reorganize degree/program entries', organize_programs] ,
+                #9: ['Reorganize degree/program entries, take 2', organize_programs2] ,
+                10:['Find online programs', find_online_programs],
+                11:['Which courses were scheduled as online?', summarize_online_sections],
+                12:['First try with logic rules', smart_find_online_programs],
+                13:['Another try, simplified', simple_find_online_programs],
+                14:['Parse programs with pattern matching', cq_start],
+                15:['Parse programs with pattern matching, take 2', cq_pattern_start],
+                #16:['Baby web server', serve],
+                16:['80 20 effort. Sept 2020', cq_8020_start],
+                17:['Organize programs stage 2 (2021)', organize_programs_stage2],
+              }
+    
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
diff --git a/curriculum2020.py b/curriculum2020.py
new file mode 100644
index 0000000..f47b222
--- /dev/null
+++ b/curriculum2020.py
@@ -0,0 +1,661 @@
+from pampy import match, _
+import json, pypandoc, requests,json,os,re, bisect, csv, codecs
+import sortedcontainers as sc
+from collections import defaultdict
+from toolz.itertoolz import groupby
+
+import pdb
+
+
+pat8020 = []
+
+
+""" (programs) entityType  entityTitle   status   proposalType    sectionName      lastUpdated      lastUpdatedBy    
+    fieldName     displayName     lookUpDisplay    fieldValue     instanceSortOrder
+    lookUpDataset   (array of dicts, each has keys: name, value, and corresponding values.)
+    
+    subsections or fields (arrays) - ignore for now just takem in order
+    
+    (courses) same as above?  
+    
+    html values: markdown convert?
+    
+"""
+
+
+"""pat8020.append(   {"displayName": _} )
+pat8020.append(   {"entityType": _} )
+pat8020.append(   {"entityTitle": _} )
+pat8020.append(   {"lookUpDisplay": _} )
+pat8020.append(   {"fieldValue": _} )
+"""
+
+err = "no error\n"
+
+
+def to_md(s):
+    output = pypandoc.convert_text(s,'md',format='html')
+    return output
+
+
+def print_return(x):
+    print('got a hit')
+    print()
+    return x
+    
+
+def cq_8020(root,indent=0):
+
+    ret = []
+    idt = " " * indent
+    
+
+    try:
+        m = match( root, 
+            {"attributes": { "fieldName": "Department" }, 
+             "lookUpDisplay": _ }, lambda x: idt + "Department: " + x.strip() if x.strip() else "",
+            {"attributes": { "fieldName": "Division" }, 
+             "lookUpDisplay": _ }, lambda x: idt + "Division: " + x.strip() if x.strip() else "",
+            {"attributes": { "fieldName": "Discipline" }, 
+             "lookUpDisplay": _ }, lambda x: idt + "Discipline: " + x.strip() if x.strip() else "",
+            {"attributes": { "fieldName": "Program Title" }, 
+             "lookUpDisplay": _ }, lambda x: idt + "Program Title: " + x.strip() if x.strip() else "",
+            {"attributes": { "fieldName": "Outcome" }, 
+             "fieldValue": _ }, lambda x: idt + "Outcome: " + x.strip() if x.strip() else "",
+             
+             
+            {"attributes": { "fieldName": "Award Type" }, 
+             "lookUpDisplay": _ }, lambda x: idt + "Award Type: " + x,
+            {"attributes": { "fieldName": "Course" }, 
+             "lookUpDisplay": _ }, lambda x: idt + "Course: " + x.strip() if x.strip() else "",
+            {"attributes": { "fieldName": "Description" }, 
+             "fieldValue": _ }, lambda x: idt + "Description: " + to_md(x),
+            {"attributes": { "fieldName": "Justification" }, 
+             "fieldValue": _ }, lambda x: idt + "Justification: " + x.strip() if x.strip() else "",
+            {"fieldName": _}, lambda x: idt + "field name: " + x.strip() if x.strip() else "",
+            {"fieldValue": _}, lambda x: idt + "field value: " + x.strip() if x.strip() else "",
+            #{"entityType": _}, lambda x: idt + "entityType: " + x,
+            {"entityTitle": _}, lambda x: idt + "entityTitle: " + x.strip() if x.strip() else "",
+            {"lookUpDisplay": _}, lambda x: idt + "lookUpDisplay: " + to_md(x.strip()) if x.strip() else "",
+            
+            # Units
+            { "name": "Max", "value": _ }, lambda x: "%sMax: %s" % (idt,x),
+            { "name": "Min", "value": _ }, lambda x: "%sMin: %s" % (idt,x),
+            { "name": "Text", "value": _ }, lambda x: "%sText: %s" % (idt,x),
+
+            default=False )
+        if m: 
+            print('case 1: ' + str(m) )
+            ret.append(m)
+    except Exception as e:
+        m = 0
+        pass
+        #print("GOT EXCEPTION.")
+        #err += str(e)
+
+    if (not m) and type(root) == type(  {}  ):
+        """
+        for K,V in list(root.items()):
+            print( [K,V])
+            m = match( [K,V],
+                ["lookUpDisplay", _ ],  lambda x: idt + "lookup display: " + to_md(str(x).strip()) if str(x).strip() else "",
+                ["fieldName", _ ],  lambda x: idt + "field name: " + x,
+                ["fieldValue", _ ],  lambda x: idt + "field value: " + to_md(str(x).strip()) if str(x).strip() else "",
+                ["entityType", _ ],  lambda x: idt + "entity type: " + x,
+                ["entityTitle", _ ],  lambda x: idt + "entity title: " + x,
+                ["displayName", _ ],  lambda x: idt + "display name: " + x,
+                ["sectionSortOrder", _ ],  lambda x: idt + "section sort order: " + str(x),
+                default=False)
+            if m: 
+                print('case 2 ' + str(m))
+                ret.append(m)
+        
+        #else:
+        """
+        for V in root.values():
+            m = cq_8020(V,indent+2)
+            if m:
+                print('case 4 ' + str(m))
+                ret.extend(m)
+        
+    elif (not m) and type(root) == type([]):
+        for V in root:
+            m = cq_8020(V,indent+2)
+            if m: 
+                print('case 3')
+                ret.extend(m)
+    
+    return ret 
+
+
+
+def cq_8021(root,indent=0):
+
+    ret = []
+    idt = " " * indent
+    m = 0
+
+    try:
+        m = match( root, 
+            {"attributes": { "fieldName": "Department" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Department", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Division" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Division", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Discipline" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Discipline", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Program Title" }, 
+             "lookUpDisplay": _ }, lambda x:  {"key":"Program Title", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Outcome" }, 
+             "fieldValue": _ }, lambda x:  {"key":"Outcome", "value": x.strip() } if x.strip() else 0,
+             
+             
+            {"attributes": { "fieldName": "Award Type" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Award Type", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Course" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Course", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Description" }, 
+             "fieldValue": _ }, lambda x: {"key":"Description", "value": to_md(x.strip()) } if x.strip() else 0,
+            {"attributes": { "fieldName": "Justification" }, 
+             "fieldValue": _ }, lambda x: {"key":"Justification", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Assessment" }, 
+             "fieldValue": _ }, lambda x: {"key":"Assessment", "value": x.strip() } if x.strip() else 0,
+
+            {"attributes": { "fieldName": "Disk Name" }, 
+             "fieldValue": _ }, lambda x: {"key":"Disk Name", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Attached File Name" }, 
+             "fieldValue": _ }, lambda x: {"key":"Attached File Name", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Title" }, 
+             "fieldValue": _ }, lambda x: {"key":"Title", "value": x.strip() } if x.strip() else 0,
+             
+            {"fieldName": _}, lambda x: {"key": x.strip()} if x.strip() else 0,
+            {"fieldValue": _}, lambda x:  {"value": x.strip()} if x.strip() else 0,
+            {"entityType": _}, lambda x: {"key": "Type", "value": x.strip()} if x.strip() else 0,
+            {"entityTitle": _}, lambda x:  {"key": "Title", "value": x.strip()} if x.strip() else 0,
+            {"lookUpDisplay": _}, lambda x:  {"value": x.strip()} if x.strip() else 0,
+            
+            # Units
+            { "name": "Max", "value": _ }, lambda x:  {"key": "max", "value": x.strip()} if x.strip() else 0,
+            { "name": "Min", "value": _ }, lambda x: {"key": "min", "value": x.strip()} if x.strip() else 0,
+            { "name": "Text", "value": _ }, lambda x: {"value": x.strip()} if x.strip() else 0,
+
+            default=False )
+        if m: 
+            print('case 1: ' + str(m) )
+            ret.append(m)
+    except Exception as e:
+        m = 0
+        pass
+        #print("GOT EXCEPTION.")
+        #err += str(e)
+
+    if (not m) and type(root) == type(  {}  ):
+        """
+        for K,V in list(root.items()):
+            print( [K,V])
+            m = match( [K,V],
+                ["lookUpDisplay", _ ],  lambda x: idt + "lookup display: " + to_md(str(x).strip()) if str(x).strip() else "",
+                ["fieldName", _ ],  lambda x: idt + "field name: " + x,
+                ["fieldValue", _ ],  lambda x: idt + "field value: " + to_md(str(x).strip()) if str(x).strip() else "",
+                ["entityType", _ ],  lambda x: idt + "entity type: " + x,
+                ["entityTitle", _ ],  lambda x: idt + "entity title: " + x,
+                ["displayName", _ ],  lambda x: idt + "display name: " + x,
+                ["sectionSortOrder", _ ],  lambda x: idt + "section sort order: " + str(x),
+                default=False)
+            if m: 
+                print('case 2 ' + str(m))
+                ret.append(m)
+        
+        #else:
+        """
+        for V in root.values():
+            m = cq_8021(V,indent+2)
+            if m:
+                print('case 4 ' + str(m))
+                ret.extend(m)
+                #for mm in m:
+                #    if 'key' in mm and 'value' in mm:
+                #        ret.extend(mm)
+        
+    elif (not m) and type(root) == type([]):
+        for V in root:
+            m = cq_8021(V,indent+2)
+            if m: 
+                print('case 3')
+                ret.extend(m)
+    
+    return ret 
+
+def cq_8021_start():
+    root = json.loads( open('cache/programs/programs_1.txt','r').read())
+    outt = open('cache/test_prog8020.txt','w')
+    outt_err = open('cache/test_prog8020err.txt','w')
+
+    result = cq_8021(root)
+
+    outt.write( json.dumps(result, indent=2))
+    #outt_err.write( err )
+
+
+
+
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # 
+#  # # # # # # # # # # # # # # # # # # # # # # # 
+# # # # # # # # # # # # # # # # # # # # # # # # 
+#  # # # # # # # # # # # # # # # # # # # # # # # 
+# # # # # # # # # # # # # # # # # # # # # # # # 
+#  # # # # # # # # # # # # # # # # # # # # # # # 
+
+##
+## In this attempt I try to keep the data structure intact, but swapping in parts I recognize for a 
+## more compact version.
+#
+## Recursively do this....
+##
+## As I elaborate on it, the non-swapped parts will hopefully stand out more and more, and I can
+## track down all the problems.
+##
+
+def cq_8022(root,indent=0):
+
+    ret = []
+    idt = " " * indent
+    m = 0
+
+    try:
+        m = match( root, 
+        
+            # Clear empties
+            { "attributes": { "fieldName": _ },  "fieldValue": "" }, "NULL",
+            { "attributes": { "fieldName": _ }, "lookUpDisplay": "", "fieldValue": _ }, lambda x,y: {"key":x,"value":y},
+            { "attributes": { "fieldName": _ }, "lookUpDisplay": _, "fieldValue": "" }, lambda x,y: {"key":x,"value":y},
+
+
+        
+            {"attributes": { "fieldName": "Exception Identifier" },  "fieldValue": _ }, lambda x: {"key":"Exception ID", "value": x},
+            {"attributes": { "fieldName": "Department" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Department", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Division" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Division", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Discipline" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Discipline", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Program Title" }, 
+             "fieldValue": _ }, lambda x:  {"key":"Program Title", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Outcome" }, 
+             "fieldValue": _ }, lambda x:  {"key":"Outcome", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Award Type" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Award Type", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Course" }, 
+             "lookUpDisplay": _ }, lambda x: {"key":"Course", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Description" }, 
+             "fieldValue": _ }, lambda x: {"key":"Description", "value": to_md(x.strip()) } if x.strip() else 0,
+            {"attributes": { "fieldName": "Justification" }, 
+             "fieldValue": _ }, lambda x: {"key":"Justification", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Assessment" }, 
+             "fieldValue": "-" }, lambda x: "NULL",
+            {"attributes": { "fieldName": "Assessment" }, 
+             "fieldValue": _ }, lambda x: {"key":"Assessment", "value": x.strip() } if x.strip() else 0,
+
+            {"attributes": { "fieldName": "Disk Name" }, 
+             "fieldValue": _ }, lambda x: {"key":"Disk Name", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Attached File Name" }, 
+             "fieldValue": _ }, lambda x: {"key":"Attached File Name", "value": x.strip() } if x.strip() else 0,
+            {"attributes": { "fieldName": "Title" }, 
+             "fieldValue": _ }, lambda x: {"key":"Title", "value": x.strip() } if x.strip() else 0,
+             
+            {"entityType": _}, lambda x: {"key": "Type", "value": x.strip()} if x.strip() else 0,
+            {"entityTitle": _}, lambda x:  {"key": "Title", "value": x.strip()} if x.strip() else 0,
+            {"lookUpDisplay": _}, lambda x:  {"value": x.strip()} if x.strip() else 0,
+            
+            {"attributes": { "fieldName": "Course" }, "lookUpDisplay": _ }, lambda x:  {"key": "Course", "value": x.strip()} if x.strip() else 0,
+            
+            # Units
+            { "name": "Max", "value": _ }, lambda x:  {"key": "max", "value": x.strip()} if x.strip() else 0,
+            { "name": "Min", "value": _ }, lambda x: {"key": "min", "value": x.strip()} if x.strip() else 0,
+            { "name": "Text", "value": _ }, lambda x: {"value": x.strip()} if x.strip() else 0,
+            
+            
+            # Programs
+            { "attributes": { "fieldName": "Course Block Definition" }, 
+              "fieldValue": _ }, lambda x: { "key":"Course block d.", "value": x.strip() },
+            { "attributes": { "fieldName": "Unit Min" },
+              "fieldValue": _ }, lambda x: { "key":"Unit min", "value": x },
+            { "attributes": { "fieldName": "Unit Max" },
+              "fieldValue": _ }, lambda x: { "key":"Unit max", "value": x },
+            { "attributes": { "fieldName": "Units Low" },
+              "fieldValue": _ }, lambda x: { "key":"Units low", "value": x },
+            { "attributes": { "fieldName": "Units High" },
+              "fieldValue": _ }, lambda x: { "key":"Units high", "value": x },
+            { "attributes": { "fieldName": "Override Unit Calculation" },
+              "fieldValue": _ }, lambda x: { "key":"override unit calc", "value": x },
+            { "attributes": { "fieldName": "Override Defalut Unit Calculations" },
+              "fieldValue": _ }, lambda x: { "key":"override default unit calc", "value": x },
+            { "attributes_unchanged": { "sectionOrInstance": "section" },
+              "subsections_unchanged": [], "fields": [] }, lambda x:  "NULL" ,
+            { "attributes": { "sectionOrInstance": "section" },
+              "subsections": [], "fields": [] }, lambda x:  "NULL" ,
+            { "attributes_unchanged": { "sectionOrInstance": "section" },
+              "subsections": [], "fields": [] }, lambda x:  "NULL" ,
+            { "attributes": { "sectionName": "[Discipline and Course chained combo]", "sectionSortOrder": _ },
+              "fields": _ }, lambda w,x: { "sortOrder":w, "key":"course", "value": x },
+            #
+            #{ "key": "_", "value": "_" }
+            
+            default=False ) 
+        if m:
+            print(' '*indent + 'case 1: ' + str(m) )
+            return 1,m
+            
+    except Exception as e:
+        m = 0
+        
+    
+    
+    if (not m) and type(root) == type( [] ):
+    
+        # an array that only has DICTS, which only have 2 (or 3) keys, key,value,(sortOrder)
+        # we want to collapse it into a dict.
+        this_change = 0
+        maybe_new_dict = {}
+        is_collapsable = 1
+        for z in root:
+            if type(z)==type({}):
+                for ea in list(z.keys()):
+                    if not ea in ['sortOrder','key','value']:
+                        is_collapsable = 0
+            else:
+                is_collapsable = 0
+            if not is_collapsable:
+                break
+            if is_collapsable:
+                kk = list(z.keys())
+                if 'sortOrder' in kk and 'key' in kk and 'value' in kk:
+                    maybe_new_dict[str(z['sortOrder'])+'_'+z['key']] = z['value']
+                elif 'key' in kk and 'value' in kk:
+                    maybe_new_dict[z['key']] = z['value']
+                else:
+                    maybe_new_dict['value'] = z['value']
+        
+        if is_collapsable:
+            return 1,maybe_new_dict
+            
+            
+        my_list = []
+        for x in root:
+            changed, m = cq_8022(x, indent+1)
+            this_change += changed
+            if changed:
+                if m != "NULL":
+                    my_list.append(m)
+                    print(' '*indent + 'case 5: ' +str(m))
+            else:
+                my_list.append(x)
+        if this_change:
+            changed2,m2 = cq_8022(my_list,indent+1)
+            return changed2+this_change , m2
+        
+    if (not m) and type(root) == type(  {}  ):
+        my_d_clone = {}
+        this_change = 0
+        for k,V in root.items():
+            
+            changed,m = cq_8022(V,indent+1)
+            this_change += changed
+            
+            if this_change:
+                print(' '*indent + 'case 4: ' +str(m))
+                my_d_clone[k] = m
+            else:
+                #my_d_clone[k+'_unchanged'] = V
+                my_d_clone[k] = V
+        if this_change:
+            changed2,m2 = cq_8022(my_d_clone,indent+1)
+            return changed2+this_change , m2
+            
+
+    return 0,root
+            
+    """if not changed and k == "fields" and type(V) == list:
+        #new_dict = {"err":[] }
+        new_list = []
+        for item in V:
+            if item == "NULL": continue
+            if type(item) == dict:
+                if len(item.keys())==2 and ("key" in item.keys()) and ("value" in item.keys()):
+                    #print("\n" + str(item.keys()))
+                    #pdb.set_trace()
+                    new_list.append( {"key": item["key"], "value": item["value"] } )
+                else:
+                    changed,m = cq_8022(item, indent+1)
+                    this_change += changed
+                    if changed:
+                        new_list.append(m)
+                    else:
+                        new_list.append(item)
+        m = new_list
+        this_change += 1
+            
+            
+            
+    
+    
+    elif (not m) and type(root) == type([]):
+    myclone = []
+    this_change = 0
+    for V in root:
+        changed,m = cq_8022(V,indent+1)
+        this_change += changed
+        if m: 
+            print('case 3 (' + str(indent) + ') ' + str(m))
+            myclone.append(m)
+        else:
+            myclone.append(V)
+    if this_change:
+        return cq_8022(myclone,indent+1)
+    
+    return this_change,myclone"""
+    
+
+def cq_8022_start():
+    root = json.loads( open('cache/programs/programs_demo.txt','r').read())
+    outt = open('cache/test_prog8020.txt','w')
+    outt_err = open('cache/test_prog8020err.txt','w')
+
+    #changed = 1
+    #while changed:
+    changed,result = cq_8022(root)
+
+    outt.write( json.dumps(result, indent=2))
+    #outt_err.write( err )
+
+# # # # # # # # # #
+# # # #
+# #
+#
+# May 2021
+
+
+def sortable_class(li):
+    dept = li[1]
+    rest = ''
+    
+    # little error case here
+    n = re.match(r'([A-Za-z]+)(\d+)',li[2])
+    if n:
+        num = int(n.group(2))
+    else:
+        m = re.match(r'(\d+)([A-Za-z]+)$',li[2])
+        if m:
+            num = int(m.group(1))
+            rest = m.group(2)
+        else:
+            num = int(li[2])
+    if num < 10: num = '00'+str(num)
+    elif num < 100: num = '0'+str(num)
+    else: num = str(num)
+    return dept+num+rest
+
+
+def c_name(c):
+    delivery = set()
+    units = []
+    slos = []
+    hybridPct = ''
+    active = 'Active'
+    id = c['entityMetadata']['entityId']
+    if c['entityMetadata']['status'] != 'Active':
+        active = 'Inactive'
+        #return ()
+    for r in c['entityFormData']['rootSections']:
+        
+        if r['attributes']['sectionName'] == 'Course Description':
+        
+            for ss in r['subsections']:
+                for f in ss['fields']:
+                    
+                    if f['attributes']['fieldName'] == 'Course Discipline':
+                        dept = f['lookUpDisplay']
+                    if f['attributes']['fieldName'] == 'Course Number':
+                        num = f['fieldValue']
+                    if f['attributes']['fieldName'] == 'Course Title':
+                        title = f['fieldValue']
+                        #print "\n" + title
+                    if f['attributes']['fieldName'] == 'Course Description':
+                        desc = re.sub(r'\n',' ', f['fieldValue'])
+        if r['attributes']['sectionName'] == 'Units/Hours/Status':
+            for ss in r['subsections']:
+                if ss['attributes']['sectionName'] == '':
+                    for f in ss['fields']:
+                        if f['attributes']['fieldName'] == 'Minimum Units' and f['fieldValue'] not in units:
+                            units.insert(0,f['fieldValue'])
+                        if f['attributes']['fieldName'] == 'Maximum Units' and f['fieldValue'] and f['fieldValue'] not in units:
+                            units.append(f['fieldValue'])
+
+        
+        # Newer entered courses have this filled out
+        if r['attributes']['sectionName'] == 'Distance Education Delivery':
+            for ss in r['subsections']:
+                if ss['attributes']['sectionName'] == 'Distance Education Delivery':
+                    for ssa in ss['subsections']:
+                        for f in ssa['fields']:
+                            if f['attributes']['fieldName'] == 'Delivery Method':
+                                    delivery.add(f['lookUpDisplay'])
+                if ss['attributes']['sectionName'] == "":
+                    if ss['fields'][0]['attributes']['fieldName'] == "If this course is Hybrid, what percent is online?":
+                        hybridPct = str(ss['fields'][0]['fieldValue'])
+
+        # Older ones seem to have it this way
+        if r['attributes']['sectionName'] == 'Distance Education':
+            for ss in r['subsections']:
+                for f2 in ss['fields']:
+                    if 'fieldName' in f2['attributes'] and f2['attributes']['fieldName'] == 'Methods of Instruction':
+                        #print f2['fieldValue']
+                        if f2['fieldValue'] == 'Dist. Ed Internet Delayed':
+                            delivery.add('Online')
+
+        # SLO
+        if r['attributes']['sectionName'] == 'Student Learning Outcomes':
+            for ss in r['subsections']:
+                if 'subsections' in ss:
+                    if ss['attributes']['sectionName'] == 'Learning Outcomes':
+                        for s3 in ss['subsections']:
+                            for ff in s3['fields']:
+                                if ff['attributes']['fieldName'] == 'Description':
+                                    slos.append(ff['fieldValue'])
+                        
+                        #print ff
+                        #[0]['fields']:
+                        #print ff['fieldValue']
+                #for f2 in ss['fields']:
+                #    if 'fieldName' in f2['attributes'] and f2['attributes']['fieldName'] == 'Methods of Instruction':
+                #        if f2['fieldValue'] == 'Dist. Ed Internet Delayed':
+                #            delivery.append('online(x)')
+
+    if len(units)==1: units.append('')  
+    if len(delivery)==0: delivery.add('')
+    u0 = 0 
+    try: 
+        u0 = units[0]
+    except:
+        pass
+        
+    u1 = 0 
+    try: 
+        u1 = units[2]
+    except:
+        pass
+        
+    return id,dept,num,active,title,u0,u1,'/'.join(delivery),hybridPct,desc,slos    
+
+
+
+
+
+
+def show_classes2020():
+    pass
+
+def show_classes2020_start():
+    outt = open('cache/test_class2021_all.txt','w')
+    max_active = {}   # hold the id of the class if seen. only include the highest id class in main list. 
+    used_course = {}  # hold the actual course info, the version we'll actually use.
+    slo_by_id = {} # values are a list of slos.
+    slo_by_id_included = {} # just the ids of active or most recent versions.
+    #tmp = codecs.open('cache/course_temp.txt','w','utf-8')
+    for f in os.listdir('cache/courses'):
+        if re.search('classes_',f):
+            print(f)
+            cls = json.loads(open('cache/courses/'+f,'r').read())
+            for c in cls:
+                dir_data = list(c_name(c))
+                #tmp.write(str(dir_data) + "\n\n")
+                slo_by_id[dir_data[0]] = dir_data[10] # 
+                info = list(map(str,dir_data[:10]))
+                info.append(dir_data[10])
+                #pdb.set_trace()
+                #print info
+                course_key = sortable_class(info)
+                curqnt_id = int(info[0])
+                if course_key in max_active:
+                    if curqnt_id < max_active[course_key]:
+                        continue
+                max_active[course_key] = curqnt_id
+                if course_key in used_course:
+                    while course_key in used_course:
+                        course_key += '_'
+                used_course[course_key] = info
+                print("\t%s" % course_key)
+                outt.write( json.dumps(info, indent=2))
+
+    out2 = open('cache/test_class2021.txt','w')
+
+
+    
+    out2.write( json.dumps(used_course, indent=2) )
+
+if __name__ == "__main__":  
+    print ('')
+    options = { 1: ['take 1 - programs',     cq_8021_start],
+                2: ['take 2 - programs',     cq_8022_start],
+                3: ['take 1 - classes',      show_classes2020_start],
+                }
+    
+    for key in options:
+        print((str(key) + '.\t' + options[key][0]))
+    
+    print('')
+    #resp = eval(input('Choose: '))
+    resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
+
+
+
+
+
+
+
+
diff --git a/curriculum_patterns.py b/curriculum_patterns.py
new file mode 100644
index 0000000..7ff9ebd
--- /dev/null
+++ b/curriculum_patterns.py
@@ -0,0 +1,481 @@
+from pampy import _
+
+curic_patterns = []
+
+
+
+curic_patterns.append( {
+            "attributes": {
+                "fieldName": "Division", 
+                "fieldId": 65000, 
+                "isLookUpField": True, 
+                "displayName": "Division"
+            }, 
+            "lookUpDisplay": _, 
+            "dataTypeDetails": {
+                "type": "lookup"
+            }, 
+            "fieldValue": _
+        } )
+
+def div1(a,b): 
+    r = "Division: %s, id: %s" % (a,b)
+    print(r)
+    return(r)
+
+curic_patterns.append(div1)
+
+curic_patterns.append( {
+            "attributes": {
+                "fieldName": "Department", 
+                "fieldId": 65001, 
+                "isLookUpField": True, 
+                "displayName": "Department"
+            }, 
+            "lookUpDisplay": _, 
+            "dataTypeDetails": {
+                "type": "lookup"
+            }, 
+            "fieldValue": _
+        })
+def d2(a,b): 
+    r = "Department: %s, id: %s" % (a,b)
+    print(r)
+    return r
+
+curic_patterns.append(d2)
+
+
+
+curic_patterns.append({
+    "attributes": {
+        "fieldName": "Award Type", 
+        "fieldId": 60221, 
+        "isLookUpField": True, 
+        "displayName": "Award Type"
+    }, 
+    "lookUpDisplay": _, 
+    "dataTypeDetails": {
+        "type": "lookup"
+    }, 
+    "fieldValue": _
+})
+def d3(a,b): 
+    r = "Award: %s, id: %s" % (a,b)
+    print(r)
+    return r
+
+curic_patterns.append(d3)
+
+
+p1 = {
+    "attributes": {
+        "fieldName": "Description", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Description"
+    }, 
+    "dataTypeDetails": {
+        "type": "string"
+    }, 
+    "fieldValue": _
+}
+
+def pp1(a,b): 
+    r = "Description (id:%s) %s" % (a,b)
+    #print(r[:40])
+    return r
+
+curic_patterns.append(p1)
+curic_patterns.append(pp1)
+
+
+p2 = {"attributes": {
+        "fieldName": "Program Title", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Program Title"
+    }, 
+    "dataTypeDetails": {
+        "type": "string", 
+        "maxLength": 250
+    }, 
+    "fieldValue":_
+}
+
+def pp2(a,b):
+    r = "Program (id:%s) %s" % (a,b)
+    #print(r)
+    return r
+
+curic_patterns.append(p2)
+curic_patterns.append(pp2)
+    
+
+
+
+p3 = { "attributes": {
+        "fieldName": "Course", 
+        "fieldId": _, 
+        "isLookUpField": True, 
+        "displayName": "Course"
+    }, 
+    "lookUpDataset": [
+        [
+            {
+                "name": "Max", 
+                "value": _
+            }, 
+            {
+                "name": "IsVariable", 
+                "value": _
+            }, 
+            {
+                "name": "Min", 
+                "value": _
+            }, 
+            {
+                "name": "Text", 
+                "value": _
+            }
+        ]
+    ], 
+    "dataTypeDetails": {
+        "type": "lookup"
+    }, 
+    "lookUpDisplay": _, 
+    "fieldValue": _
+}
+
+def pp3(a,b,c,d,e,f,g):
+    r = "Course (%s / %s) %s (%s), var? %s  %s - %s" % (a,g, f, e, c, b, d)
+    #print(r)
+    return r
+
+curic_patterns.append(p3)
+curic_patterns.append(pp3)
+    
+
+p4 = {
+    "attributes": {
+        "sectionOrInstance": "section", 
+        "sectionName": "Unit Range", 
+        "sectionSortOrder": 2, 
+        "oneToManySection": False
+    }, 
+    "subsections": [], 
+    "fields": [
+        {
+            "attributes": {
+                "fieldName": "Units Low", 
+                "fieldId": 59608, 
+                "isLookUpField": False, 
+                "displayName": "Units Low"
+            }, 
+            "dataTypeDetails": {
+                "scale": 2, 
+                "type": "numeric", 
+                "precision": 6
+            }, 
+            "fieldValue": _
+        }, 
+        {
+            "attributes": {
+                "fieldName": "Units High", 
+                "fieldId": 59609, 
+                "isLookUpField": False, 
+                "displayName": "Units High"
+            }, 
+            "dataTypeDetails": {
+                "scale": 2, 
+                "type": "numeric", 
+                "precision": 6
+            }, 
+            "fieldValue": _
+        }
+    ]
+}
+
+def pp4(a,b):
+    r = "Unit Range: %s - %s" % (a,b)
+    return r
+
+curic_patterns.append(p4)
+curic_patterns.append(pp4)
+    
+p5 =  {
+            "attributes": {
+                "fieldName": "Discipline", 
+                "fieldId": _, 
+                "isLookUpField": True, 
+                "displayName": "Discipline"
+            }, 
+            "lookUpDisplay": _, 
+            "dataTypeDetails": {
+                "type": "lookup"
+            }, 
+            "fieldValue": _
+        }
+def pp5(a,b,c):
+    r = "Discipline (%s) %s / %s" % (a,b,c)
+    #print(r)
+    return r
+
+curic_patterns.append(p5)
+curic_patterns.append(pp5)
+
+
+
+p6 = { "attributes": {
+        "fieldName": "Course Block Definition", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Course Block Definition"
+    }, 
+    "dataTypeDetails": {
+        "type": "string"
+    }, 
+    "fieldValue": _
+}
+
+def pp6(a,b):
+    r = "Block (%s) %s" % (a,b)
+    #print(r)
+    return r
+    
+    
+p7 = {
+    "attributes": {
+        "fieldName": "Block Header", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Block Header"
+    }, 
+    "dataTypeDetails": {
+        "type": "string", 
+        "maxLength": 4000
+    }, 
+    "fieldValue": _
+}
+
+def pp7(a,b):
+    r = "Block Header (%s) %s" % (b,a)
+    #print(r)
+    return r
+    
+    
+p8 = {
+    "attributes": {
+        "fieldName": "Block Footer", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Block Footer"
+    }, 
+    "dataTypeDetails": {
+        "type": "string", 
+        "maxLength": 4000
+    }, 
+    "fieldValue": _
+}
+
+def pp8(a,b):
+    r = "Block Footer (%s) %s" % (b,a)
+    #print(r)
+    return r
+    
+curic_patterns.append(p6)
+curic_patterns.append(pp6)
+curic_patterns.append(p7)
+curic_patterns.append(pp7)
+curic_patterns.append(p8)
+curic_patterns.append(pp8)
+
+
+
+                                        
+                                        
+                            
+                            
+                            
+######################      
+######################   Trying to remove more junk      
+######################                  
+
+j1 = {"attributes": 
+         {"fieldName": _, 
+         "fieldId": _, 
+         "isLookUpField": False, 
+         "displayName": _, 
+         }, 
+        "dataTypeDetails": 
+         {"type": "string", 
+         "maxLength": _, 
+         }, 
+        "fieldValue": _, 
+        }
+                                        
+                                        
+def jj1(a,b,c,d,e):
+    r = "String Label: %s (id %s) Value: %s" % (a,b,e)
+    #print(r)
+    return r
+
+curic_patterns.append(j1)
+curic_patterns.append(jj1)
+
+
+j2 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": False, 
+                 "displayName": _, 
+                 }, 
+                "dataTypeDetails": 
+                 {"scale": _, 
+                 "type": "numeric", 
+                 "precision": _, 
+                 }, 
+                "fieldValue": _, 
+                }
+
+def jj2(a,b,c,d,e,f):
+    r = "Generic Num Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,f)
+    #print(r)
+    return r
+    
+j3 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": True, 
+               "displayName": _, 
+               }, 
+              "lookUpDisplay": _, 
+              "dataTypeDetails": 
+               {"type": "lookup", 
+               }, 
+              "fieldValue": _, 
+                }
+
+def jj3(a,b,c,d,e):
+    r = "Generic lookup Field: Name: %s / %s, ID: %i, Displayname: %s, Value: %s " % (a,c,b,d,e)
+    #print(r)
+    return r
+
+
+curic_patterns.append(j2)
+curic_patterns.append(jj2)
+
+curic_patterns.append(j3)
+curic_patterns.append(jj3)
+
+
+j4 = {"attributes": 
+           {"fieldName": _, 
+           "fieldId": _, 
+           "isLookUpField": False, 
+           "displayName": _, 
+           }, 
+          "dataTypeDetails": 
+           {"type": "flag", 
+           }, 
+          "fieldValue": _, 
+          }
+
+def jj4(a,b,c,d):
+    r = "Generic Flag Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,d)
+    #print(r)
+    return r
+    
+curic_patterns.append(j4)
+curic_patterns.append(jj4)
+
+
+
+j5 = {"attributes": 
+         {"fieldName": _, 
+         "fieldId": _, 
+         "isLookUpField": False, 
+         "displayName": _, 
+         }, 
+        "dataTypeDetails": 
+         {"scale": _, 
+         "type": "numeric", 
+         "precision": _, 
+         }, 
+        "fieldValue": _, 
+        }
+
+def jj5(a,b,c,d,e,f):
+    r = "Numeric Field, Name: %s / %s  Id: %s, Value: %s" % (a,c,b,f)
+
+
+j6 = {"attributes": 
+           {"fieldName": _, 
+           "fieldId": _, 
+           "isLookUpField": False, 
+           "displayName": _, 
+           }, 
+          "dataTypeDetails": 
+           {"type": "string",  }, 
+          "fieldValue": _,  }
+
+def jj6(a,b,c,d):
+    r = "String+Label field. Label: %s / %s  Value: %s  Id: %s" % (a,c,d,b)
+    #print(r)
+    return r
+    
+curic_patterns.append(j5)
+curic_patterns.append(jj5)
+curic_patterns.append(j6)
+curic_patterns.append(jj6)
+    
+    
+    
+"""j2 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": False, 
+                 "displayName": _, 
+                 }, 
+                "dataTypeDetails": 
+                 {"scale": 2, 
+                 "type": "numeric", 
+                 "precision": 6, 
+                 }, 
+                "fieldValue": _, 
+                }
+
+def jj2(a,b,c,d):
+    r = "Generic Num Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,d)
+    print(r)
+    return r
+    
+j2 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": False, 
+                 "displayName": _, 
+                 }, 
+                "dataTypeDetails": 
+                 {"scale": 2, 
+                 "type": "numeric", 
+                 "precision": 6, 
+                 }, 
+                "fieldValue": _, 
+                }
+
+def jj2(a,b,c,d):
+    r = "Generic Num Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,d)
+    print(r)
+    return r
+"""    
+    
+    
+
+
+
+
+
+
+
+
diff --git a/depricated.py b/depricated.py
new file mode 100644
index 0000000..8590302
--- /dev/null
+++ b/depricated.py
@@ -0,0 +1,1289 @@
+
+#get_schedule('201770') 
+
+
+# from pipelines - canvas data
+
+"""
+timestamp = nowAsStr()
+
+requestParts = [ method,
+    host,
+    '', #content Type Header
+    '', #content MD5 Header
+    path,
+    '', #alpha-sorted Query Params
+    timestamp,
+    apiSecret ]
+
+#Build the request
+requestMessage = '\n'.join( requestParts )
+requestMessage = requestMessage.encode('ASCII')
+print((requestMessage.__repr__()))
+hmacObject = hmac.new(bytearray(apiSecret,'ASCII'), bytearray('','ASCII'), hashlib.sha256)      # 
+hmacObject.update(requestMessage)
+hmac_digest = hmacObject.digest()
+sig = base64.b64encode(hmac_digest)
+headerDict = {
+    'Authorization' : 'HMACAuth ' + apiKey + ':' + str(sig),
+    'Date' : timestamp
+}
+
+
+"""
+
+# Don't know
+def demo():
+    resp = do_request('/api/account/self/file/sync')
+    mylog.write(json.dumps(resp, indent=4))
+    sample_table = resp['files'][10]
+    filename = sample_table['filename']
+    print(sample_table['table'])
+
+    response = requests.request(method='GET', url=sample_table['url'], stream=True)
+    if(response.status_code != 200):
+        print(('Request response went bad. Got back a ', response.status_code, ' code, meaning the request was ', response.reason))
+    else:
+        #Use the downloaded data
+        with open(local_data_folder + filename, 'wb') as fd:
+            for chunk in response.iter_content(chunk_size=128):
+                fd.write(chunk)
+        print("Success")
+    if filename.split('.')[-1] == 'gz':
+        plain_filename = 'canvas_data/' + ".".join(filename.split('.')[:-1])
+        pf = open(plain_filename,'w')
+        with gzip.open('canvas_data/' + filename , 'rb') as f:
+            pf.write(f.read())
+            
+     
+    
+       
+            
+            
+# How to drop columns
+#columns = ['Col1', 'Col2', ...]
+#df.drop(columns, inplace=True, axis=1)
+
+# left join, one on column, one on index
+#merged = pd.merge(result,users,left_index=True,right_on='id', how='left')
+
+
+"""
+You can call set_index on the result of the dataframe:
+
+In [2]:
+data=[['Australia',100],['France',200],['Germany',300],['America',400]]
+pd.DataFrame(data,columns=['Country','Volume']).set_index('Country') 
+
+Out[2]:
+           Volume
+Country          
+Australia     100
+France        200
+Germany       300
+America       400
+"""
+
+
+
+def stats():
+    # nothing seems to happen here?
+    
+    #input = csv.DictReader(codecs.open(schedfile,'r','utf-8'))
+    input = csv.DictReader(open(schedfile,'r'))
+    out2 = open('temp2.csv','w')
+    clean = {}
+    for r in input:
+        if r['crn']: clean[ r['crn'] ] = r
+    
+    for c,r in list(clean.items()):    
+        try:
+            if int(r['cap'])==0: continue
+            else: prct = (1.0 * int( r['act'] )) / int(r['cap'])
+            if prct < 0.01: continue
+            o_str = ''
+            if r['location'].strip()=='ONLINE': o_str = 'online'
+            #print r['location']
+            date_parts = r['date'].split('-')
+            start = strptime(date_parts[0], '%m/%d')
+            if start > semester_begin: o_str += "\tlatestart " + date_parts[0]
+            out2.write( "".join([c, "\t", r['sub'], "\t", r['crs'], "\t", str(round(prct,2)), "% full\t", o_str, "\n"]) )
+        except:
+            pass
+
+
+
+
+
+#########   from curriculum. py
+
+
+# open('cache/programs/programs_1.txt','r').read()
+
+"""   
+SEE serve.py   .... i mean ...  interactive.py
+def dict_generator(indict, pre=None):
+    pre = pre[:] if pre else []
+    if isinstance(indict, dict):
+        for key, value in indict.items():
+            if isinstance(value, dict):
+                for d in dict_generator(value, pre + [key]):
+                    yield d
+            elif isinstance(value, list) or isinstance(value, tuple):
+                for v in value:
+                    for d in dict_generator(v, pre + [key]):
+                        yield d
+            else:
+                yield str(pre) + "   " + str([key, value]) + "\n"
+    else:
+        yield pre + [indict]
+        yield str(pre) + "   " + str([indict]) + "\n"
+        
+ 
+ 
+def print_dict(v, prefix='',indent=''):
+    if isinstance(v, dict):
+        return [   print_dict(v2, "{}['{}']".format(prefix, k) + "<br />", indent+" " )  for k, v2 in v.items()  ]
+    elif isinstance(v, list):
+        return [ print_dict(  v2,  "{}[{}]".format(prefix , i) + "<br />", indent+" ")  for i, v2 in enumerate(v) ]
+    else:
+        return '{} = {}'.format(prefix, repr(v)) + "\n"
+        
+
+def walk_file():
+    j = json.loads(open('cache/programs/programs_2.txt','r').read())
+    
+    return print_dict(j)
+
+from flask import Flask
+from flask import request
+
+def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+def server_save(key,value):
+    codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
+    
+def flask_thread(q):
+    app = Flask(__name__)
+
+    @app.route("/")
+    def home():
+        return tag('h1','This is my server.') + "<br />" + a('want to shut down?','/sd')
+
+    @app.route("/save/<key>/<val>")
+    def s(key,val):
+        server_save(key,val)
+        return tag('h1','Saved.') + "<br />" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
+
+    @app.route("/crazy")
+    def hello():
+        r = '<link rel="stylesheet" href="static/bootstrap.min.css">'
+        r += tag('style', 'textarea { white-space:nowrap; }')
+        r += tag('body', \
+            tagc('div','container-fluid', \
+                tagc('div','row', \
+                    tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
+                    tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
+        
+         
+        
+        return r
+    
+    @app.route("/sd")
+    def sd():
+        print('SIGINT or CTRL-C detected. Exiting gracefully')
+        func = request.environ.get('werkzeug.server.shutdown')
+        if func is None:
+            raise RuntimeError('Not running with the Werkzeug Server')
+        func()
+        return "Server has shut down."
+    app.run()
+    
+    
+from queue import Queue
+
+q = Queue()
+
+def serve():
+    import webbrowser
+    import threading
+    x = threading.Thread(target=flask_thread, args=(q,))
+    x.start()
+    webbrowser.open_new_tab("http://localhost:5000")
+    
+                
+    
+    
+    #s = open('cache/programs/index.json','w')
+    #s.write( json.dumps({'departments':sorted(list(dept_index)), 'programs':prog_index}, indent=2) )
+    #s.close()
+"""
+
+
+
+
+    
+# Prompt for course id, return list of user dicts.   TODO this duplicates courses.py  ??
+def getUsersInCourse(id=0):   # returns list
+    if not id:
+        id = str(input("The Course ID? ")) 
+    id = str(id)
+    return fetch('/api/v1/courses/%s/users' % id, 0)
+    
+
+
+
+#### curriculum.py
+
+
+def recur_look_for_leafs(item,indent=0,show=1):
+    global leafcount, displaynames
+    ii = indent * " "
+    is_leaf = am_i_a_leaf(item)
+    if type(item) == type({}):
+        status = ""
+        if show:
+            status = "Dict"
+        if is_leaf:
+            leafcount += 1
+            status = "Leaf Dict"
+        if status:
+            print("\n%s%s" % (ii,status))
+        indent += 1
+        ii = indent * " "
+        for K,V in list(item.items()):
+            if show or is_leaf:
+                print("%s%s:" % (ii, K), end="")
+                if K =='displayName': displaynames.append(V)
+            recur_look_for_leafs(V,indent+1,show or is_leaf)
+        
+    elif type(item) == type([]):
+        status = ""
+        if show: status = "List (" + str( len(item) ) + ")"
+        if is_leaf: status = "Leaf List (" + str( len(item) ) + ")"
+        if status:
+            print("\n%s%s" % (ii,status))
+        indent += 1
+        ii = indent * " "
+        for V in item:
+            recur_look_for_leafs(V,indent+1, show or is_leaf)
+    
+    elif type(item) == type("abc"): 
+        if show: print("%s%s" % (' ', item))
+    elif type(item) == type(55): 
+        if show: print("%s%i" % (' ', item))
+    elif type(item) == type(5.5):  
+        if show: print("%s%f" % (' ', item))
+    elif type(item) == type(False): 
+        if show: print("%s%s" % (' ', str(item)))
+        
+
+def am_i_a_leaf(item):
+    if type(item) == type({}):
+        for K,V in list(item.items()):
+            if type(V) == type({}) or  type(V) == type([]):
+                return False
+        
+    elif type(item) == type([]):
+        for V in item:
+             if type(V) == type({}) or  type(V) == type([]):
+                return False
+    
+    elif type(item) == type("abc"): return True
+    elif type(item) == type(55): return True
+    elif type(item) == type(5.5): return True
+    elif type(item) == type(False):
+        if item == False: return True
+        elif item == True: return True
+    return True
+
+def sampleclass():
+    theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
+    #print(json.dumps(theclass,indent=2))
+    recur_look_for_leafs(theclass)
+    print(leafcount)
+    print(sorted(displaynames))
+    
+    
+    
+def recur_matcher(item, depth=0):
+    indent = depth * " "
+    my_result_lines = []
+    if type(item) == type({}):
+        if not match( item, 
+            {'entityMetadata': {'entityTitle': _,'status': _, 'entityType':_, 'entityId':_ }}, 
+                lambda title,status,typ,id: 
+                    my_result_lines.append("%s%s: %s (id %s)  status: %s" % (indent, str(typ), str(title), str(id), str(status))) ,
+            {'attributes': {'displayName': _}, 'lookUpDisplay': _, }, 
+                lambda x,y: my_result_lines.append("%s%s: %s" % (indent, clean(str(x)), clean(str(y)))) ,
+            {'attributes': {'displayName': _}, 'fieldValue': _, }, 
+                lambda x,y: my_result_lines.append("%s%s: %s" % (indent, clean(str(x)), clean(str(y)))) ,
+            {'sectionName': _}, 
+                lambda x: my_result_lines.append("%sSection: %s" % (indent, str(x))) ,
+            _, nothing
+        ):
+            for K,V in list(item.items()):
+                my_result_lines.extend(recur_matcher(V,depth+1))
+    elif type(item) == type([]):
+        for V in item:
+            my_result_lines.extend(recur_matcher(V,depth+1))
+    return my_result_lines
+
+
+
+    
+    
+def matchstyle():
+    theclass = json.loads( codecs.open('cache/courses/samplecourse.json','r','utf-8').read() )
+    print("\n".join(recur_matcher(theclass)))
+
+
+#                7: ['pattern matcher style', matchstyle],
+#                8: ['pattern matcher - test on all classes', match_style_test],
+
+
+
+#####  from localcache
+
+stem_course_id = '11015'   # TODO
+
+# NO LONGER USED - SEE COURSES
+def enroll_stem_students():
+    depts = "MATH BIO CHEM PHYS ASTR GEOG".split(" ")
+    students = set()
+    for d in depts:
+        students.update(dept_classes(d))
+    print(students)
+
+    to_enroll = [ x for x in students if x not in already_enrolled ]
+
+    print(to_enroll)
+    print("prev line is people to enroll\nnext line is students already enrolled in stem")
+    print(already_enrolled)
+
+    for s in to_enroll:
+        t = url + '/api/v1/courses/%s/enrollments' % stem_course_id
+        data = { 'enrollment[user_id]': s[1], 'enrollment[type]':'StudentEnrollment',
+                 'enrollment[enrollment_state]': 'active' }
+        print(data)
+        print(t)
+        if input('enter to enroll %s or q to quit: ' % s[0]) == 'q':
+            break
+        r3 = requests.post(t, headers=header, params=data)
+        print(r3.text)
+
+
+#####
+#####   from users.py     pretty much just use sql now
+
+
+# unused?    
+def getAllTeachersInTerm():        # a list
+    # classes taught in last 3 semesters
+    # How many of them were published and used
+    # hits in last week/month/year
+    # most common department
+    # email addr
+    all_courses = {}
+    teachers = {}  # keyed by goo   
+    # { 'name':'', 'id':'', 'email':'', 'goo':'', 'classes':[    (#name,#id,#pubd,#hitsbyteacher)  ...  ] }
+    
+    # This is a bit different from the 1 year schedule above, because it looks at
+    # people who were active in their shells in iLearn.
+    
+    outfile = codecs.open('teacherdata/historical_shells_used.json','w', encoding='utf-8')
+    for term in last_4_semesters_ids:   #  [60,]:
+        print(("Fetching term: " + str(term)))
+        all_courses[term] = \
+            fetch('/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100')
+    i = 0
+    j = 0
+    for k,v in list(all_courses.items()):       ##### term   k, list v
+        for a_class in v:
+            print((a_class['name']))
+            published = 0
+            if a_class['workflow_state'] in ['available','completed']: 
+                j += 1
+                published = 1
+            i += 1
+            #if i > 20: break
+            tch = fetch('/api/v1/courses/' + str(a_class['id']) + '/search_users?enrollment_type=teacher')
+            for r in tch:   ##### TEACHER r  of COURSE a_class
+                name = str(r['sortable_name'])
+                if not 'sis_import_id' in r:    
+                    print("This user wasn't available: " + name)
+                    continue
+                goo = str(r['sis_import_id'])
+                print((r['sortable_name']))
+                if not name in teachers:
+                    email = getEmail(r['id'])
+                    teachers[name] = { 'name':r['sortable_name'], 'id':r['id'], 'email':email, 'goo':goo, 'classes':[] }
+                info = (a_class['name'],a_class['id'],published)
+                teachers[name]['classes'].append( info )
+        
+    ## TODO: hits in courses by teachers    https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
+    
+    for t,v in list(teachers.items()):
+        teachers[t]['num_courses'] = len(v['classes'])
+        teachers[t]['num_active_courses'] = sum(  [x[2] for x in v['classes']]  )
+        depts = [ dept_from_name(x[0]) for x in v['classes'] ]
+        teachers[t]['dept'] = most_common_item(depts)
+    
+    #print(str(j), "/", str(i), " sections are published")
+    outfile.write(json.dumps(teachers))
+
+
+"""
+def teacherActivityLog(uid=1):   ### Next: save results in a hash and return that....
+    global results, users, users_by_id
+    #get_users()   # do this if you think  'teachers/users.json' is outdated.
+    
+    load_users()
+    
+    #for x in users_by_id.keys():
+    #    if x < 20: 
+    #        print x
+    #        print users_by_id[x]
+    
+    
+    teachers = csv.reader(open('teachers/current_semester.txt','r'), delimiter="\t")
+    for row in teachers:
+        print(row[0] + " is id: " + row[1])
+        uid = row[1]
+        print("Comes up as: " + str(users_by_id[int(uid)]))
+        info = users_by_id[int(uid)]
+        goo = info['login_id']
+        
+        output_file = open('logs/users/byweek/'+ goo.lower() + '.csv', 'w')
+        
+        
+        # okay, actually, the first week here is the week before school IRL
+        start = isoweek.Week.withdate( datetime.date(2017,8,21))
+        end = isoweek.Week.thisweek()
+        byweek = []
+        
+        i = 0
+        while(1):
+            results = []
+            start = start + 1
+            if start > end: break
+            
+            myStart = start.day(0).isoformat() + 'T00:00-0700'
+            myEnd = start.day(6).isoformat() + 'T11:59:59-0700'
+            t = url + "/api/v1/users/" + str(uid) + "/page_views?start_time=" + myStart + '&end_time=' + myEnd + "&perpage=500"
+            print(t)
+            while(t): 
+                print(".", end=' ')
+                t = fetch(t)
+            print("")
+            thisWeek = len(results)
+            print("Week # " + str(i) + "\t" + str(thisWeek))
+            byweek.append( "Week # " + str(i) + "\t" + str(thisWeek) )
+            output_file.write( start.isoformat() + "," + str(thisWeek) + "\n")
+            i += 1
+        for j in byweek:
+            print(j)
+"""
+
+"""
+def summarize_student_teacher_role(u):
+    # u is a "group" from the groupby fxn
+    # term is sp18 now
+    t = 0
+    s = 0
+    for a in u:
+        if a=='TeacherEnrollment': t += 1
+        else: s += 1
+    if NUM_ONLY:
+        if t > s: return 'teacher'
+        return 'student'
+    else:
+        if t > s: return '1'
+        return '0'
+"""        
+"""    
+def user_roles2():
+    # cross list users, classes enrolled, and their roles
+    global role_table, term_courses
+    
+    role_table = enrollment_file()
+    user_table = users_file()
+    course_table = courses_file()   # from canvas
+    term_table = term_file()
+    schedule = current_schedule()   # from banner
+    
+    # current semester
+    current = term_table[lambda d: d.course_section=='2018 Spring']
+    term_id = current['id'].values[0]
+    term_courses = course_table[lambda d: d.termid==term_id]  # courses this semester
+    
+    # add is_online flag (for courses listed in schedule as online-only)
+    term_courses['is_online'] = term_courses['code'].map( lambda x: course_is_online( get_crn_from_name(x) ) )
+    
+    new_df = pd.DataFrame(columns=['type','oo','num'])
+    
+    m = 0
+    data = []
+    for u in user_table.iterrows():
+        if m % 1000 == 0: print("on row " + str(m))
+        m += 1
+        data.append(categorize_user(u))
+        #if m > 1500: break
+    new_df = pd.DataFrame(data,columns=['i','type','onlineonly','numcls']).set_index('i')
+    print(new_df)
+    
+    user_table = user_table.merge(new_df,left_index=True,right_index=True)
+    user_table.to_csv('canvas_data/users_online.csv')
+"""
+
+### IS THIS IN CANVAS_DATA.py?
+
+
+
+
+""" Collate the raw logs into something more compact and useful. Version 1:
+ - # of accesses, user/day
+ - # of participations, user/day
+ - 
+ 
+ - where day is the number of days into the semester. Classes shorter than 16 weeks should get a multiplier
+ - 
+ 
+ - 2 initial goals: 
+   a. data for statistics / clustering / regression / learning
+   b. data for visualization
+"""
+def req_to_db(fname_list):
+    fields = ','.join("id timestamp timestamp_year timestamp_month timestamp_day user_id course_id root_account_id course_account_id quiz_id discussion_id conversation_id assignment_id url user_agent http_method remote_ip interaction_micros web_application_controller web_applicaiton_action web_application_context_type web_application_context_id real_user_id session_id user_agent_id http_status http_version".split(" "))
+    sqlite_file = 'canvas_data/data.db'
+    conn = sqlite3.connect(sqlite_file) 
+    c = conn.cursor()
+    # merge all requests into db
+    by_date_course = defaultdict( lambda: defaultdict(int) )
+    by_date_user   = defaultdict( lambda: defaultdict(int) )
+    df_list = []
+    df_list_crs = []
+    users = defaultdict( lambda: defaultdict(int) )
+    i = 0
+    limit = 300
+    for fname in fname_list:
+        print((fname+"\n"))
+        for line in gzip.open('canvas_data/'+fname,'r'):
+            r = line.split('\t')
+            #tot = len(fields.split(','))
+            #i = 0
+            #for x in fields.split(','):
+            #    print x + "\t" + r[i]
+            #    i+= 1
+                
+            qry = "insert into requests("+fields+") values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+            conn.execute(qry, r)
+            
+            
+            # New method for below: 
+            # read collated data from sqlite
+            # collate from more logs
+            # write back....?
+            
+            """
+            date = datetime.datetime.strptime( r['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
+            if r['userid'] in users:
+                users[r['userid']]['freq'] += 1
+                if users[r['userid']]['lastseen'] < date:
+                    users[r['userid']]['lastseen'] = date
+            else:
+                users[r['userid']] = {"id":r['userid'], "lastseen":date, "freq":1}
+            by_date_course[ r['day'] ][ r['courseid'] ] += 1
+            by_date_user[ r['day'] ][ r['userid'] ] += 1
+            #if r['userid'] in by_user: by_user[r['userid']] += 1
+            #else: by_user[r['userid']] = 1
+            #if r['courseid'] in by_course: by_course[r['courseid']] += 1
+            #else: by_course[r['courseid']] = 1
+        #mylog.write("by_user = " + str(by_user))
+        df_list.append(pd.DataFrame(data=by_date_user))
+        df_list_crs.append(pd.DataFrame(data=by_date_course))
+        """
+        i += 1
+        if i > limit: break
+        conn.commit()
+    conn.close()
+    
+    
+    
+    
+"""
+Making columns:
+table_data = [['a', 'b', 'c'], ['aaaaaaaaaa', 'b', 'c'], ['a', 'bbbbbbbbbb', 'c']]
+for row in table_data:
+    print("{: >20} {: >20} {: >20}".format(*row))
+
+Transpose a matrix:
+rez = [[m[j][i] for j in range(len(m))] for i in range(len(m[0]))]
+
+"""
+    
+    
+
+
+    """
+    ilearn_by_id = {}
+    ilearn_by_name = {}
+    for x in ilearn_list:
+        ilearn_by_id[x[3]] = x
+        ilearn_by_name[x[0]] = x
+    
+    for ml in open('cache/teacher_manual_name_lookup.csv','r').readlines():
+        parts = ml.strip().split(',')
+        try:
+            manual_list[parts[0]] = ilearn_by_id[parts[1]]
+        except Exception as e:
+            print "Teacher missing: " + parts[0]
+            
+    il_names = [ x[0] for x in ilearn_list ]
+    il_byname = {}
+    for x in ilearn_list: il_byname[x[0]] = x
+    sched_list_missed = [x for x in sched_list]
+    
+    #
+    # key is long name (with middle name) from schedule, value is tuple with everything 
+    name_lookup = manual_list
+    matches = []
+    
+    #print ilearn_list
+    
+    num_in_sched = len(sched_list)
+    num_in_ilearn = len(ilearn_list)
+    
+    #for i in range(min(num_in_sched,num_in_ilearn)):
+    #    print "|"+sched_list[i] + "|\t\t|" + ilearn_list[i][0] + "|"
+    
+    print("Sched names: %i, iLearn names: %i" % (num_in_sched,num_in_ilearn))
+    
+    for s in sched_list:
+        for t in il_names:
+            if first_last(s) == t:
+                #print '  MATCHED ' + s + ' to ' + t
+                sched_list_missed.remove(s)
+                try:
+                    name_lookup[s] = ilearn_by_name[ first_last(s) ]
+                except Exception as e:
+                    print "Teacher missing (2): " + s
+                il_names.remove(first_last(s))
+                matches.append(s)
+                
+    
+    print "Matched: " + str(matches)
+    
+    print "\nDidn't match: " + str(len(sched_list_missed)) + " schedule names."
+    
+    print "\nFinal results: "
+    print name_lookup
+    
+    nlf = codecs.open('cache/sched_to_ilearn_names.json','w','utf-8')
+    nlf.write(json.dumps(name_lookup,indent=2))
+    # STRING DISTANCE
+            #sim = find_most_similar(s,i_names)
+            #print '  CLOSEST MATCHES to ' + s + ' are: ' + str(sim)
+            #mm.write(s+',\n')
+    """
+    
+    
+        #ilearn_list = sorted(list(set(map(
+    #                                lambda x: #(tfi[x]['name'],tfi[x]['email'],tfi[x]['dept'],str(tfi[x]['id']),tfi[x]['goo']), 
+    #                                tfi.keys()))))
+    #i_names = [ x[0] for x in ilearn_list ]
+    
+    #print json.dumps(i_names,indent=2)
+    #return
+    
+    
+    
+    # how to filter a dict based on values
+    # filtered = {k: v for k, v in course_combos.items() if v['dept'] == 'LIB' or v['dept'] == 'CSIS' }
+    
+    # more pandas
+    # gapminder['continent'].unique()
+
+
+
+
+
+    #for name,group in bycode:
+    #    #print name
+    #    print name, " ", group['type']
+    
+    #onl = gg.agg( lambda x: has_online(x) )
+    #ttl = gg.agg( lambda x: len(x) )
+    #ttl = ttl.rename(columns={'type':'total_sections'})
+    
+    #onl.join(gg.agg( lambda x: has_hybrid(x) ),how='outer')
+    #onl.join(gg.agg( lambda x: has_lecture(x) ), how='outer')
+    
+    #onl['num_sections'] = 0
+    #onl['num_lec'] = 0
+    #onl['num_online'] = 0
+    
+    #all = pd.merge([onl,hyb,lec])
+    #print onl
+    #total=len, f2f=lambda x:  )   set(x)
+    #{   'num_sections': "count",
+    #    'num_lec': lambda x:  5,
+    #    'num_online': lambda x:  5 } )
+    #print gg
+"""    
+def has_online(series):
+    # if any items of the series have the string 'online', return 1
+    for i in series: 
+        if i == 'online': return 1
+    return 0
+def has_lecture(series):
+    # if any items of the series have the string 'online', return 1
+    for i in series: 
+        if i == 'online': return 1
+    return 0
+def has_hybrid(series):
+    # if any items of the series have the string 'online', return 1
+    for i in series: 
+        if i == 'hybrid': return 1
+    return 0
+"""
+#### RIGHT HERE IS WHERE I THINK... MAYBE THIS ISN'T THE RIGHT APPROACH. I DON'T SEEM
+#### TO BE ABLE TO QUERY THE FACT BASE. IS THAT TRUE? SHOULD I JUST BE USING TABLES?
+
+#### CHANGING COURSE... USE THE RULES TO UPDATE A DATABASE/TABLE/DATAFRAME
+#### OR SET OF DICTS.
+
+# ultimately i want this to be more flexible, so i can categorize degrees as 'available evening' etc
+#
+
+
+# Simple data structure. In this function, a degree is
+""" degree = { 'name': 'History AA',
+               'blocks': [ { 'original_title':'xxx', 'rulecode':'u3', 
+                             'courses': [ {'code':'math1a', 'units': '3.0', 'wasonline':False },
+                                          {'code':'math2a', 'units': '3.0', 'wasonline':False },
+                                          {'code':'math3a', 'units': '3.0', 'wasonline':False } ] },
+                           { 'original_title':'xyz', 'rulecode':'a', 
+                             'courses': [ {'code':'math5a', 'units': '3.0', 'wasonline':False },
+                                          {'code':'math6a', 'units': '3.0', 'wasonline':False },
+                                          {'code':'math7a', 'units': '3.0', 'wasonline':False } ] } ] } 
+                                          
+"""
+
+
+
+
+
+
+
+# Wrapper to get 2 schedules at once
+def dl_sched():
+    global SEMESTER, semester_begin, filename, short_sem
+    SEMESTER = 'Fall 2019'
+    short_sem = 'fa19'
+    semester_begin = strptime('08/26', '%m/%d')
+    filename = 'fa19_sched.json'
+    
+    txt = login()
+    codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
+    #stats()
+    #reg_nums()
+    
+    #todo: these semesters
+    SEMESTER = 'Summer 2019'
+    short_sem = 'su19'
+    semester_begin = strptime('06/17', '%m/%d')
+    filename = 'su19_sched.json'
+    
+    txt = login()
+    codecs.open('output/'+filename,'w').write( json.dumps( to_section_list(txt) ) )
+    #stats()
+    #reg_nums()
+
+
+
+
+
+
+
+
+    
+    
+# Send a personalized email regarding ZTC    
+def send_z_email(fullname, firstname, addr, courses_list):
+    FULLNAME = fullname    #"Sabrina Lawrence"
+    FNAME = firstname      # "Sabrina"
+    to_email = addr        # "slawrence@gavilan.edu"
+    courses = courses_list # ["CSIS45", "CSIS85"]
+    
+    course_template = "<a href='%s'>%s</a> &nbsp; &nbsp;"
+    url_template = "https://docs.google.com/forms/d/e/1FAIpQLSfZLQp6wHFEdqsmpZ7jz2Y8HtKLo8XTAhrE2fyvTDOEgquBDQ/viewform?usp=pp_url&entry.783353363=%s&entry.1130271051=%s"    #   % (FULLNAME, COURSE1)
+    
+    bare_link = "https://forms.gle/pwZJHdWSkyvmH4L19"
+    
+    COURSELINKS = ''
+    PLAINCOURSES = ''
+    for C in courses:
+        ut = url_template % (FULLNAME, C)
+        COURSELINKS += course_template % (ut, C)
+        PLAINCOURSES += C + "   "
+    
+    text_version = open('cache/ztc_mail1.txt','r').read()
+    html_version = open('cache/ztc_mail1_h.txt','r').read()
+    
+    # replace these:  $FNAME   $COURSELINKS    $LINK
+    
+    email = re.sub( r'\$FNAME', FNAME, text_version )
+    email = re.sub( r'\$COURSELINKS', PLAINCOURSES, email )
+    email = re.sub( r'\$LINK', bare_link, email )
+    
+    email_h = re.sub( r'\$FNAME', FNAME, html_version )
+    email_h = re.sub( r'\$COURSELINKS', COURSELINKS, email_h )
+    
+    print(email_h+"\n\n"+email)
+
+    from O365 import Account
+
+    credentials = ('phowell@gavilan.edu', 'xxx')
+    client_secret = 'xxx'  # expires 10/28/2020
+    tenant_id = "4ad609c3-9156-4b89-9496-0c0600aeb0bb"
+    # application client id: 29859402-fa55-4646-b717-752d90c61cde
+
+    account = Account(credentials, auth_flow_type='credentials', tenant_id=tenant_id)
+    if account.authenticate():
+        print('Authenticated!')
+   
+    #account = Account(credentials)
+    #if account.authenticate(scopes=['message_all']):
+    #    print('Authenticated!')
+    m = account.new_message()
+    m.to.add(addr)
+    m.subject = 'Quick question about your course textbook'
+    m.body = "email_h"
+    m.send()
+
+    """
+    import smtplib 
+    from email.mime.multipart import MIMEMultipart
+    from email.mime.text import MIMEText
+    
+    msg = MIMEMultipart('alternative')
+    msg['Subject'] = "Quick question about your course textbook"
+    msg['From'] = "gavdisted@gmail.com"
+    msg['To'] = to_email
+
+    msg.attach(MIMEText(email, 'plain'))
+    msg.attach(MIMEText(email_h, 'html'))
+
+
+    #s = smtplib.SMTP('smtp.gmail.com', 587) 
+    #s.starttls() 
+    #s.login("gavdisted", "xxx") 
+    
+    
+    s = smtplib.SMTP_SSL('smtp.office365.com',587)
+    s.ehlo()
+    s.starttls()
+    s.login('phowell@gavilan.edu', 'xxx')
+    
+    #s.sendmail(msg['From'], msg['To'], msg.as_string()) 
+    s.sendmail(msg['From'], msg['To'], "Testing") 
+    s.quit()"""
+
+
+
+def getInactiveTeachersInTerm(t=23):        # a list
+    global results
+    teachers = {}
+    emails = {}
+    outfile = codecs.open('canvas/inactive_teachers.txt','w', encoding='utf-8')
+    efile = codecs.open('canvas/inactive_teachers_emails.txt','w', encoding='utf-8')
+    
+    #yn = raw_input('All courses? y=all n=only active   ')
+    #all = 0
+    #if yn=='y': all = 1
+    
+    if not t:
+        t = askForTerms()
+    else: t = [ t, ]
+    for term in t:
+        r = url + '/api/v1/accounts/1/courses?enrollment_term_id=' + str(term) + '&perpage=100'
+        while(r): r = fetch(r)
+    all_courses = results #json.loads(results)
+    #print "All unpublished courses: " 
+    i = 0
+    j = 0
+    for k in all_courses:
+        j += 1
+        if k['workflow_state'] != 'available':
+            i += 1
+            print(str(i), "\t", k['name'], "\t", k['workflow_state'])
+            results = []
+            t2 = url + '/api/v1/courses/' + str(k['id']) + '/search_users?enrollment_type=teacher'
+            
+            
+            while(t2): t2 = fetch(t2)
+            #print results
+            for r in results:
+                key = r['sortable_name'] + "\t" + str(r['id'])
+                #if not 'email' in r: pdb.set_trace()
+                emails[key] = str(r['sis_user_id'])
+                #print r
+                if key in teachers:
+                    teachers[key].append(k['name'])
+                else:
+                    teachers[key] = [ k['name'], ]
+            #print json.dumps(results, indent=4, sort_keys=True)
+            #a = raw_input()
+            
+    print(str(i), "/", str(j), " sections are unpublished")
+    for t in list(emails.keys()):
+        efile.write(emails[t] + ", ")
+    for t in list(teachers.keys()):
+        outfile.write(t + "\t")
+        for c in teachers[t]:
+            outfile.write(c + ",")
+        outfile.write("\n")
+    #f.write(json.dumps(teachers, indent=4, sort_keys=True))
+    print("Output file is in ./teachers/current_semester.txt")
+    #print json.dumps(all_courses, indent=4, sort_keys=True)
+    """for x in all_courses:
+        qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
+        t = url + qry
+        while(t): t = fetch(t)
+    """
+
+
+
+
+
+    #for t,v in teachers.items():
+    #    outfile.write( "|".join(   [   v['goo'], v['name'], v['email'], v['dept'], str(v['num_courses']), str(v['num_active_courses']) ] ) + "\n" )
+        
+        #{"goo": "G00275722", "name": "Agaliotis, Paul", "num_courses": 1, "num_active_courses": 1, "id": 5092, "dept": "AMT", "classes": [["AMT120 POWERPLANT TECH FA18 10958", 5322, 1]], "email": "PAgaliotis@gavilan.edu"},
+        
+    #for t in teachers.keys():
+    #    outfile.write(t + "\t")
+    #    for c in teachers[t]:
+    #        outfile.write(c + ",")
+    #    outfile.write("\n")
+    #f.write(json.dumps(teachers, indent=4, sort_keys=True))
+    #print "Output file is in ./teachers/current_semester.txt"
+    #print json.dumps(all_courses, indent=4, sort_keys=True)
+    """for x in all_courses:
+        qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
+        t = url + qry
+        while(t): t = fetch(t)
+    """
+
+
+
+
+
+def course_location(course):
+    if len(course[0]) > 13:
+        period = Set( [course_location_raw(course[0][13])], )
+    else:
+        period = Set()
+        
+    if len(course) > 1:
+        period.add(course_location_raw(course[1][13]))
+    
+    if len(course) > 2:
+        period.add(course_location_raw(course[2][13]))
+    
+    if len(course) > 3:
+        period.add(course_location_raw(course[3][13]))
+    
+    if len(course) > 4:
+        period.add(course_location_raw(course[4][13]))
+    
+    if len(course) > 5:
+        period.add(course_location_raw(course[5][13]))
+
+    
+    if 'TBA' in period:
+        period.remove('TBA')
+    
+    period = list(period)
+    
+    if len(period)==0:
+        return ''
+        
+    if len(period)==1:
+        return period[0]
+    
+    if len(period)==2 and 'Online' in period:
+        period.remove('Online')
+        return 'Hybrid at ' + period[0]
+    return '/'.join(period)
+    
+    
+def course_time(course):
+    # is it morning, mid, or evening?
+    
+    period = Set( [raw_course_time(course[0][7])], )
+    
+    if len(course) > 1:
+        #time += ", " + course[1][7]
+        period.add(raw_course_time(course[1][7]))
+    
+    if len(course) > 2:
+        #time += ", " + course[2][7]
+        period.add(raw_course_time(course[2][7]))
+    
+    if len(course) > 3:
+        #time += ", " + course[3][7]
+        period.add(raw_course_time(course[3][7]))
+    
+    if len(course) > 4:
+        #time += ", " + course[4][7]
+        period.add(raw_course_time(course[4][7]))
+    
+    if len(course) > 5:
+        #time += ", " + course[5][7]
+        period.add(raw_course_time(course[5][7]))
+
+    #print raw_course_time(course[0][7]), 
+    
+    if 'TBA' in period:
+        period.remove('TBA')
+    
+    period = list(period)
+    
+    if len(period)==0:
+        return ''
+        
+    if len(period)==1:
+        return period[0]
+        
+    return '/'.join(period)
+
+
+        
+def course_teacher(course):
+    t = Set()
+    for c in course:
+        t.add(c[11])
+    return " / ".join(list(t))
+
+
+    
+
+
+
+def reg_nums():
+    courses = []
+    dates = []
+    sections = categorize()
+    
+    today = todays_date_filename()
+    
+    out = open(today+'.csv','w')
+    dates = {'loc':{}, 'time':{}, 'start':{}, 'teacher':{}}
+    i = 1
+    for f in os.listdir('.'):
+        m = re.search('reg_'+short_sem+'_(\d+)\.csv',f)
+        if m:
+            filein = open(f,'r').readlines()[1:] 
+            d = m.group(1)
+            dates[d] = {}
+            for L in filein:
+                parts = L.split(',')           # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
+                if not re.search('(\d+)',parts[0]): continue   
+                if len(parts)<8: continue
+                if not parts[8]: continue
+                if float(parts[8])==0: continue
+                
+                dates[d][parts[0] + " " + parts[1]] =   (1.0* float(parts[9])) / float(parts[8])
+                
+                if i == 1 and parts[0] in sections:
+                    dates['loc'][parts[0] + " " + parts[1]] = course_location( sections[parts[0]] )
+                    dates['time'][parts[0] + " " + parts[1]] = course_time(sections[parts[0]] )
+                    dates['start'][parts[0] + " " + parts[1]] = course_start( sections[parts[0]] )
+                    dates['teacher'][parts[0] + " " + parts[1]] = course_teacher( sections[parts[0]] )
+
+                #dates[d]['act'] = parts[9]
+                #dates[d]['nam'] = parts[5]
+                #dates[d]['onl'] = ''
+                #print parts
+                #if len(parts)>13 and parts[13]=='ONLINE': dates[d]['onl'] = 'online'
+            i += 1    
+    """for d in sorted(dates.keys()):
+        for c in d:
+            print d
+            print dates[d]['crs']"""
+                
+    df = pd.DataFrame(dates)
+    df.to_csv(out)
+    
+# In the schedule, is this a class or a continuation of the class above?    
+def categorize():
+    # todo: must we open all these files?
+    dates = {}
+    
+    files = sorted(os.listdir('.'))
+    files = list( filter( lambda x: re.search('reg(\d+)\.csv',x), files) )
+    files.reverse()
+   
+    f = files[0]
+    filein = codecs.open(f,'r','utf-8').readlines()[1:] 
+    sections = {}
+    this_section = []
+    
+    for L in filein:
+        parts = L.strip().split(',')           # crn,code,sec,cmp,cred,name,days,time,cap,act,rem,teacher,date,loc
+        parts = list(  map( lambda x: clean_funny3(x), parts  ) )
+        
+        if not re.search('(\d+)',parts[0]):    # This is a continuation
+            this_section.append(parts)
+        else:                                  # this is a new section or the first line
+            if this_section:
+                sections[ this_section[0][0] ] = this_section
+                #print "Section: " + this_section[0][0] + " is: " + str(this_section) + "\n"
+                #print this_section[0][0] + "\t", course_start(this_section)
+                #print this_section[0][0] + "\t", course_time(this_section)
+                #print this_section[0][0] + "\t", course_location(this_section)
+            this_section =  [ parts, ]
+    return sections  
+    
+    
+    
+    
+    
+    
+
+
+
+
+
+# Deprecated. call perl.
+def constructSchedule():
+    term = raw_input("Name of html file? (ex: sp18.html)  ")
+    os.chdir('make-web-sched')
+    cmd = 'perl make.pl ' + term
+    print "command: " + cmd
+    os.system(cmd)
+
+
+
+"""
+def fetch_dict(target,params={}):
+    # if there are more results, return the url for more fetching.
+    # else return false
+    #print target
+    global results_dict
+    r2 = requests.get(target, headers = header, params=params)
+    output = r2.text
+    if output.startswith('while('):
+        output = output[9:]
+    #print output
+    mycopy = results_dict.copy()
+    results_dict = {}
+    results_dict.update(json.loads(output))
+    results_dict.update(mycopy)
+    f.write(json.dumps(results_dict, indent=2))
+    #print "\n"
+    if ('link' in r2.headers):
+        links = r2.headers['link'].split(',')
+        for L in links:
+            ll = L.split(';')
+            link = ll[0].replace("<","")
+            link = link.replace(">","")
+            if re.search(r'next', ll[1]):
+                #print ll[1] + ":\t" + link
+                return link
+    return ""    
+"""
+
+def get_schedule(term='201870', sem='fall'):
+    """
+    sched_data = { 'term_in':term, 'sel_subj':'dummy', 'sel_day':'dummy', 
+        'sel_schd':'dummy', 'sel_insm':'dummy', 'sel_camp':'dummy', 'sel_levl':'dummy', 'sel_sess':'dummy',
+        'sel_instr':'dummy', 'sel_ptrm':'dummy', 'sel_attr':'dummy', 'sel_subj':'%', 'sel_crse':'', 'sel_title':'',
+        'sel_schd':'%', 'sel_from_cred':'', 'sel_to_cred':'', 'sel_camp':'%', 'sel_ptrm':'%', 'sel_sess':'%',
+        'sel_attr':'%', 'begin_hh':'0', 'begin_mi':'0', 'begin_ap':'a', 'end_hh':'0', 'end_mi':'0', 'end_ap':'a' }
+    initial_headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Encoding':'gzip, deflate, sdch, br',
+        'Accept-Language':'en-US,en;q=0.8',
+        'Connection':'keep-alive',
+        'Host':'ssb.gavilan.edu',
+        'Upgrade-Insecure-Requests':'1',
+        } #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
+    headers = { 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'Accept-Encoding':'gzip, deflate, br',
+                'Accept-Language':'en-US,en;q=0.8',
+                'Cache-Control':'max-age=0',
+                'Connection':'keep-alive',
+                'Content-Type':'application/x-www-form-urlencoded',
+                'Host':'ssb.gavilan.edu',
+                'Origin':'https://ssb.gavilan.edu',
+                'Referer':'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term='+term,
+                'Upgrade-Insecure-Requests':'1',
+                } #'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
+    initial_url = 'https://ssb.gavilan.edu/prod/bwckgens.p_proc_term_date?p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=' + term
+    sesh = requests.Session()
+    #r1 = sesh.get(initial_url,headers=initial_headers)
+    #sesh.headers.update(headers)
+    url = 'https://ssb.gavilan.edu/prod/bwckschd.p_get_crse_unsec'
+    r1 = sesh.get(initial_url)
+    r = sesh.post(url, data=sched_data)
+    print r.headers
+    data = r.text
+    out = open('data/temp/'+term+'.html','w')
+    out.write(data)
+    out.close()"""
+    os.system('perl parse_schedule.pl data/temp/' + term + '.html' + ' ' + sem)
+
+
+
+
+
+
+#####
+#####
+#####    conf.py   ?
+
+
+str="""355	985	1296
+354	730	1295
+353	319	1290
+352	985	1289
+351	813	1285
+350	281	1285
+349	267	1279
+348	981	1252
+347	994	1252
+346	26	1250
+345	757	1288
+344	368	1288
+343	1	1286
+259	703	1295
+256	693	1293
+255	660	1292
+254	1	1291
+250	482	1287
+246	2	1284
+245	333	1283
+244	27	1282
+243	703	1281
+242	730	1281
+241	482	1280
+239	211	1278
+238	794	1278
+237	2	1277
+236	297	1276
+235	831	1276
+233	482	1251"""
+
+for L in str.split("\n"):
+    (id,host,session) = L.split("\t")
+    qry = "INSERT INTO conf_signups (user,session,timestamp) VALUES (%s,%s,'2022-08-08 17:20:00');" % (host,session)
+    print(qry)
+        
+        
+        
\ No newline at end of file
diff --git a/fa19_sched.json b/fa19_sched.json
new file mode 100644
index 0000000..e69de29
diff --git a/geckodriver.log b/geckodriver.log
new file mode 100644
index 0000000..d2d02ba
--- /dev/null
+++ b/geckodriver.log
@@ -0,0 +1,32 @@
+1558125801686	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "-marionette" "-foreground" "-no-remote" "-profile" "C:\\Users\\phowell\\AppData\\Local\\Temp\\rust_mozprofile.IlmIOIgJLngr"
+1558125802078	addons.webextension.screenshots@mozilla.org	WARN	Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: mozillaAddons
+1558125802079	addons.webextension.screenshots@mozilla.org	WARN	Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: resource://pdf.js/
+1558125802079	addons.webextension.screenshots@mozilla.org	WARN	Loading extension 'screenshots@mozilla.org': Reading manifest: Invalid extension permission: about:reader*
+1558125802203	addons.xpi-utils	WARN	Add-on lsiwebhook@lakesidesoftware.com is not correctly signed.
+1558125802204	addons.xpi-utils	WARN	Add-on lsiwebhook@lakesidesoftware.com is not correctly signed.
+1558125802205	addons.xpi-utils	WARN	addMetadata: Add-on lsiwebhook@lakesidesoftware.com is invalid: Error: Extension lsiwebhook@lakesidesoftware.com is not correctly signed(resource://gre/modules/addons/XPIDatabase.jsm:2349:17) JS Stack trace: addMetadata@XPIDatabase.jsm:2349:17
+processFileChanges@XPIDatabase.jsm:2700:21
+checkForChanges@XPIProvider.jsm:2570:34
+startup@XPIProvider.jsm:2148:25
+callProvider@AddonManager.jsm:203:12
+_startProvider@AddonManager.jsm:652:5
+startup@AddonManager.jsm:805:9
+startup@AddonManager.jsm:2775:5
+observe@addonManager.js:66:9
+1558125802205	addons.xpi-utils	WARN	Could not uninstall invalid item from locked install location
+JavaScript error: resource://gre/modules/addons/XPIProvider.jsm, line 2614: TypeError: addon is null
+1558125803597	Marionette	INFO	Listening on port 59067
+1558125803830	Marionette	WARN	TLS certificate errors will be ignored for this session
+1558125819374	Marionette	INFO	Stopped listening on port 59067
+[Parent 7300, Gecko_IOThread] WARNING: pipe error: 109: file z:/build/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 332
+[Child 6396, Chrome_ChildThread] WARNING: pipe error: 109: file z:/build/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 332
+[Child 6396, Chrome_Child[Parent 7300, Gecko_IOThread] WARNING: pipe error: 109: file z:/build/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 332
+[Child 6664, Chrome_ChildThread] WARNING: pipe error: 109: file z:/build/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 332
+[Child 6664, Chrome_ChildThrea[Child 10084, Chrome_ChildThread] WARNING: pipe error: 109: file z:/build/build/src/ipc/chromium/src/chrome/common/ipc_channel_win.cc, line 332
+[Child 10084, Chrome_ChildThread] WARNING[GPU 1
+###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
+
+0312, Chr
+
+###!!! [Child][MessageChannel::SendAndWait] Error: Channel error: cannot send/recv
+
diff --git a/gpt.py b/gpt.py
new file mode 100644
index 0000000..97ebf1e
--- /dev/null
+++ b/gpt.py
@@ -0,0 +1,28 @@
+import os, json, sys
+import openai
+
+from secrets import openai_org, openai_api_key
+
+
+openai.organization = "org-66WLoZQEtBrO42Z9S8rfd10M"
+openai.api_key = "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"
+#print(openai.Model.list())
+
+my_prompt = "Write a series of texts trying to sell a pen to a stranger."
+print(sys.argv)
+exit
+
+if len(sys.argv)>1:
+    my_prompt = " ".join(sys.argv[1:])
+else:
+    print("Prompt: %s" % my_prompt)
+    
+my_model = "text-davinci-003"
+
+# create a completion
+completion = openai.Completion.create(engine=my_model, prompt=my_prompt, max_tokens=1000, temperature=1,top_p=1)
+
+#print(completion)
+#print(json.dumps(completion,indent=2))
+print(completion.choices[0].text)
+print()
\ No newline at end of file
diff --git a/graphics.py b/graphics.py
new file mode 100644
index 0000000..7417b9a
--- /dev/null
+++ b/graphics.py
@@ -0,0 +1,206 @@
+
+
+
+
+import cv2, sys, glob, os
+
+cascPath = "haarcascade_frontalface_default.xml"
+
+# Create the haar cascade
+#faceCascade = cv2.CascadeClassifier(cascPath)
+faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
+eye_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')
+
+
+folder = "cache/picsId/Pending/test/"
+outfolder = "cache/picsId/Pending/test3/"
+files=glob.glob(folder + "*.jpg")   
+i = 0
+
+for file in files:
+
+    # Read the image
+    fn = file.split("/")[-1]
+    print(file)
+    print(fn)
+    
+    image = cv2.imread(file)
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+    img_size = image.shape
+    ww = img_size[0]
+    hh = img_size[1]
+    
+    print("Image size: " + str(img_size))
+    
+    
+    
+    # Detect faces in the image
+    faces = faceCascade.detectMultiScale(gray, minNeighbors=5, minSize=(70, 70))
+    """
+        gray,
+        scaleFactor=1.1,
+        minNeighbors=5,
+        minSize=(30, 30),
+        flags = cv2.CASCADE_SCALE_IMAGE )
+    """
+        
+    print("Found %d faces!" % len(faces))
+    if len(faces)==0: exit()
+
+    # Crop Padding
+    left = 10
+    right = 10
+    top = 10
+    bottom = 10
+
+    # Draw a rectangle around the faces
+    j = 0
+    for (x, y, w, h) in faces:
+        k = j
+        if k == 0: k = ''
+        else: k = str(k) + '_'
+
+        # Dubugging boxes
+        # cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)
+        
+        new_x1 = x-left
+        new_y1 = y-top
+        
+        new_x2 = x+w+right
+        new_y2 = y+h+bottom
+        
+        x1 = max(new_x1, 0)
+        y1 = max(new_y1, 0)
+        
+        x2 = min(new_x2, ww)
+        y2 = min(new_y2, hh)
+        
+        xx1 = min(x1,x2)
+        xx2 = max(x1,x2)
+        
+        yy1 = min(y1,y2)
+        yy2 = max(y1,y2)
+        
+        print(x, y, w, h)
+        print(ww, hh)
+        
+        print(xx1,xx2,yy1,yy2)
+        
+        #image  = image[y-top:y+h+bottom, x-left:x+w+right]
+        write_image = image[yy1:yy2, xx1:xx2]
+        
+        print("Writing: " + outfolder + k + fn)
+        try:
+            cv2.imwrite(outfolder + k + fn, write_image)
+        except:
+            print("  (failed. was image too small?)")
+        j += 1
+        
+
+
+    
+
+    # print ("cropped_{1}{0}".format(str(file),str(x)))
+    
+    
+    
+    
+    
+    
+    
+
+# autocrop
+#
+
+"""
+from PIL import Image
+from autocrop import Cropper
+
+cropper = Cropper()
+
+# Get a Numpy array of the cropped image
+cropped_array = cropper.crop('portrait.png')
+
+# Save the cropped image with PIL
+cropped_image = Image.fromarray(cropped_array)
+cropped_image.save('cropped.png')
+
+
+--------------
+
+usage: [-h] [-o OUTPUT] [-i INPUT] [-w WIDTH] [-H HEIGHT] [-e EXTENSION] [-v]
+
+Automatically crops faces from batches of pictures
+
+optional arguments:
+  -h, --help
+  		Show this help message and exit
+  -o, --output, -p, --path
+		Folder where cropped images will be placed.
+		Default: current working directory
+  -r, --reject
+		Folder where images without detected faces will be placed.
+		Default: same as output directory
+  -i, --input
+		Folder where images to crop are located.
+		Default: current working directory
+  -w, --width
+		Width of cropped files in px. Default=500
+  -H, --height
+		Height of cropped files in px. Default=500
+  --facePercent
+  		Zoom factor. Percentage of face height to image height.
+  -e, --extension
+  		Enter the image extension which to save at output.
+  		Default: Your current image extension
+  -v, --version
+  		Show program's version number and exit
+        
+        
+        
+        
+autocrop -i . -o test4 -w 250 -H 333        
+
+
+"""
+
+
+
+
+
+# smartcrop
+#
+#
+# smartcrop -W 1140 -H 400 -i input.jpg -o output.jpg
+#
+#
+
+
+
+
+# imagemagick jpeg compress
+#
+# convert -strip -interlace Plane -gaussian-blur 0.05 -quality 60% -adaptive-resize 60% img_original.jpg img_resize.jpg
+#
+#
+#  convert image.jpg -define jpeg:extent=150kb result.jpg
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/interactive.py b/interactive.py
new file mode 100644
index 0000000..1bb445f
--- /dev/null
+++ b/interactive.py
@@ -0,0 +1,919 @@
+import curses
+import heapq, re, csv, os, shutil, datetime, urllib
+import itertools, time, markdown, csv, json, os.path, webbrowser, threading
+from functools import wraps
+from flask import Flask, request, send_from_directory, Response, render_template
+from flask import send_file
+from flask_socketio import SocketIO, emit
+from werkzeug.routing import PathConverter
+from queue import Queue
+
+from importlib import reload 
+
+import server
+import localcache
+from server import *
+from secrets import flask_secretkey
+
+q = Queue() 
+
+
+HOST_NAME = '127.0.0.1' # 
+HOST_NAME = '192.168.1.6' # 
+HOST_NAME = '192.168.1.6' # 
+PORT_NUMBER = 8080 # Maybe set this to 9000.
+
+datafile = 'lambda.csv'
+
+#writing_path = 'c:/users/peter/Nextcloud/Documents/writing/'
+
+
+####
+####   This little web server is going to work with the "gui" folder / vue app
+####
+####
+
+
+
+def dict_generator(indict, pre=None):
+    pre = pre[:] if pre else []
+    if isinstance(indict, dict):
+        for key, value in indict.items():
+            if isinstance(value, dict):
+                for d in dict_generator(value, pre + [key]):
+                    yield d
+            elif isinstance(value, list) or isinstance(value, tuple):
+                for v in value:
+                    for d in dict_generator(v, pre + [key]):
+                        yield d
+            else:
+                yield str(pre) + "   " + str([key, value]) + "\n"
+    else:
+        yield pre + [indict]
+        yield str(pre) + "   " + str([indict]) + "\n"
+        
+ 
+ 
+def print_dict(v, prefix='',indent=''):
+    if isinstance(v, dict):
+        return [   print_dict(v2, "{}['{}']".format(prefix, k) + "<br />", indent+" " )  for k, v2 in v.items()  ]
+    elif isinstance(v, list):
+        return [ print_dict(  v2,  "{}[{}]".format(prefix , i) + "<br />", indent+" ")  for i, v2 in enumerate(v) ]
+    else:
+        return '{} = {}'.format(prefix, repr(v)) + "\n"
+        
+
+def walk_file():
+    j = json.loads(open('cache/programs/programs_2.txt','r').read())
+    
+    return print_dict(j)
+
+
+def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+def server_save(key,value):
+    codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
+
+def flask_thread(q):
+    #app = Flask(__name__, static_url_path='/cache', 
+    #        static_folder='cache',)
+    app = Flask(__name__)
+    app.config['SECRET_KEY'] = flask_secretkey
+    app.jinja_env.auto_reload = True
+    socketio = SocketIO(app)
+
+    app.config['TEMPLATES_AUTO_RELOAD'] = True
+
+    def before_request():
+        app.jinja_env.cache = {}
+
+    app.before_request(before_request)
+    
+    
+
+    
+    @app.route('/clearscreens')
+    def clears():
+        clearscreens()
+        return homepage()
+
+    
+    @app.route('/displaypi/on')
+    def dpi():
+        displaypi_on()
+        return homepage()
+
+    
+    @app.route('/displaypi/off')
+    def dpi2():
+        displaypi_off()
+        return homepage()
+
+    
+    @app.route('/screensoff')
+    def screenoff_a():
+        screenoff()
+        return homepage()
+
+    
+    
+    @app.route('/light')
+    def light():
+        desklight()
+        return homepage()
+
+    
+    @app.route('/image/<filename>', methods=['GET','POST'])
+    def do_image(filename):
+        return image_edit(filename)
+
+    @app.route('/imagecrop/<filename>/<x>/<y>/<w>/<h>/<newname>', methods=['GET','POST'])
+    def do_image_crop(filename,x,y,w,h,newname):
+        return image_crop(filename,x,y,w,h,newname)
+
+
+
+    #
+    #  SAVING STUFF
+    #
+
+    @app.route('/save', methods=['POST'])
+    def save_post():
+        now = datetime.now().strftime('%Y%m%dT%H%M')
+        path = request.form['path']
+        txt = request.form['content']
+        
+        o3 = codecs.open(server.writing_path + path, 'r', 'utf-8')
+        orig_text = o3.read()
+        o3.close()
+        
+        bu_filename = server.writing_path + 'older_copies/' + path + '_' + now + '.md'
+        o2 = codecs.open( bu_filename, 'w', 'utf-8' )
+        o2.write(orig_text)
+        o2.close()
+        print('wrote backup to %s.' % bu_filename)
+        
+        o1 = codecs.open(server.writing_path+path, 'w', 'utf-8')
+        o1.write(txt)
+        o1.close()
+        return "<h1>Successfully Saved</h1><br>" + a('back to writing folder','/x/writing/index') + \
+            " &nbsp; &nbsp; &nbsp; " + a('back to home','/')
+
+
+    @app.route('/x/writing/images/<fname>')
+    def writing_img(fname):
+        img_path = "/media/hd2/peter_home/Documents/writing_img/"
+        print(img_path + fname + " - writing images folder")
+        img_ext = fname.split('.')[-1]
+        if img_ext == "gif":
+            return send_from_directory(img_path, fname)
+        if img_ext == "jpg":
+            return send_from_directory(img_path, fname)
+        if img_ext == "png":
+            return send_from_directory(img_path, fname)
+        return send_from_directory(img_path, fname)
+
+    # 
+    # SERVER maintenance type stuff
+    @app.route('/rl')
+    def restart():
+        reload(server)
+        reload(localcache)
+        return "Server code reloaded"
+
+    @app.route("/x/<func>/<arg>/<arrg>")
+    def dispatch3(func,arg,arrg):
+        print("2 args")
+        return "" + server_dispatch(func, arg, arrg)
+
+    @app.route("/x/<func>/<arg>")
+    def dispatch2(func,arg):
+        print("1 arg")
+        return "" + server_dispatch(func, arg)
+
+    @app.route("/x/<func>")
+    def dispatch(func):
+        print("0 arg")
+        return server_dispatch(func)
+
+    @app.route("/api/<func>/<arg>/<arrg>")
+    def dispatch3j(func,arg,arrg):
+        print("json, 3 args")
+        return Response(server_dispatch(func, arg, arrg), mimetype='text/json')
+
+    @app.route("/api/<func>/<arg>")
+    def dispatch2j(func,arg):
+        print("json, 1 arg")
+        return Response(server_dispatch(func, arg), mimetype='text/json')
+
+    @app.route("/api/<func>")
+    def dispatch1j(func):
+        print("json, 0 arg")
+        return Response(server_dispatch(func), mimetype='text/json')
+
+    @app.route("/")
+    def home():
+        return server.homepage()
+    
+    #
+    #  STATIC ROUTES
+    #
+
+    
+    @app.route('/data/<path:path>')
+    def send_cachedata(path):
+        #myfile = os.path.join('cache', path).replace('\\','/')
+        print(path)
+        #return app.send_static_file(myfile)
+        return send_from_directory('cache', path)
+
+
+
+
+    # Departments, classes in each, and students (with hits) in each of those.
+
+    """@app.route('/iii/<path:path>')
+    def send_js(path):
+        return send_from_directory('gui/dist', path)"""
+    """@app.route('/lib/<path:path>')
+    def send_jslib(path):
+        return send_from_directory('gui/lib', path)"""
+    
+    #@app.route('/hello/')
+    #@app.route('/hello/<name>')
+
+
+    @app.route("/save/<key>/<val>")
+    def s(key,val):
+        server_save(key,val)
+        return tag('h1','Saved.') + "<br />" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
+
+    @app.route("/sample")
+    def do_sample():
+        return sample()
+        
+
+    @app.route('/podcast/media/<string:file_id>')
+    def media(file_id):
+        return send_file(LECPATH + urllib.parse.unquote(file_id), attachment_filename=urllib.parse.unquote(file_id))
+ 
+    @app.route("/podcast")
+    def podcast():
+        return lectures()
+
+    @app.route("/lectures")
+    def weblec():
+        return web_lectures()
+
+ 
+    @app.route("/crazy")
+    def hello():
+        r = '<link rel="stylesheet" href="static/bootstrap.min.css">'
+        r += tag('style', 'textarea { white-space:nowrap; }')
+        r += tag('body', \
+            tagc('div','container-fluid', \
+                tagc('div','row', \
+                    tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
+                    tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
+        
+         
+        
+        return r
+    
+    @app.route("/sd")
+    def sd():
+        print('SIGINT or CTRL-C detected. Exiting gracefully')
+        func = request.environ.get('werkzeug.server.shutdown')
+        if func is None:
+            raise RuntimeError('Not running with the Werkzeug Server')
+        func()
+        return "Server has shut down."
+        
+        
+    @socketio.on('my event', namespace='/test')
+    def test_message(message):
+        print('received and event: "my event" from page. message is: %s' % message)
+        emit('my response', {'data': 'got it! it is MYEVENT'})
+
+
+
+    socketio.run(app, host= '0.0.0.0')
+    
+    
+
+def serve():
+    x = threading.Thread(target=flask_thread, args=(q,))
+    x.start()
+    #webbrowser.open_new_tab("http://localhost:5000")
+    
+    y = threading.Thread(target=mqtt_loop)
+    y.start()
+
+
+if __name__ == '__main__':
+    serve()
+
+
+
+
+
+"""class HelloWorldExample(object):
+    def make_teacher_rel(self, tchr, clss):
+        with self._driver.session() as tx:
+            tx.run("MERGE (tchr:Teacher {name: $tchr}) MERGE (tchr)-[:TEACHES]->(clss:Class {name: $clss})", \
+           tchr=tchr, clss=clss)
+
+    def __init__(self, uri, user, password):
+        self._driver = GraphDatabase.driver(uri, auth=(user, password))
+
+    def close(self):
+        self._driver.close()
+
+
+
+    def print_greeting(self, message):
+        with self._driver.session() as session:
+            greeting = session.write_transaction(self._create_and_return_greeting, message)
+            print(greeting)
+
+    @staticmethod
+    def _create_and_return_greeting(tx, message):
+        result = tx.run("CREATE (a:Greeting) "
+                        "SET a.message = $message "
+                        "RETURN a.message + ', from node ' + id(a)", message=message)
+        return result.single()[0]
+"""        
+        
+
+def make_teacher_rel(g, tchr, clss):
+    g.run("MERGE (tchr:Teacher {name: $tchr}) MERGE (tchr)-[:TEACHES]->(clss:Class {name: $clss})", \
+       tchr=tchr, clss=clss)
+           
+
+def testgraph():
+    gg = Graph("bolt://localhost:7687", auth=("neo4j", "asdf"))
+    
+    #gg.run("DROP CONSTRAINT ON (tchr:Teacher) ASSERT tchr.name IS UNIQUE")
+    #gg.run("DROP CONSTRAINT ON (clss:Class) ASSERT clss.name IS UNIQUE")
+    
+    #gg.run("CREATE INDEX ON :Teacher(name)")
+    #gg.run("CREATE INDEX ON :Class(name)")
+    
+    stuff = json.loads( open('output/semesters/2020spring/sp20_sched.json','r').read())
+    
+    # make lists of unique course code+name, teacher, locations
+    tch = {}
+    crs = {}
+    loc = {}
+    sem = Node("Semester", name="sp20")
+    for c in stuff:
+        if not c['teacher'] in tch:
+            tch[c['teacher']] = Node("Teacher", name=c['teacher'])
+            gg.create(tch[c['teacher']])
+        if not c['code'] in crs:
+            crs[ c['code'] ] = Node("Course section", name=c['name'], code=c['code'])
+            gg.create(crs[ c['code'] ])
+        if not c['loc'] in loc:
+            loc[ c['loc'] ] = Node("Location", loc=c['loc'])
+            gg.create(loc[ c['loc'] ])
+        sect = Node("Section", crn=int(c['crn']))
+        gg.create(Relationship(tch[c['teacher']], "TEACHES", sect ))
+        gg.create(Relationship(sect, "CLASS OF", crs[ c['code'] ]  ))
+        gg.create(Relationship( sect, "LOCATED AT", loc[ c['loc'] ] ))
+            
+    """
+    for c in stuff:
+        print(c['crn'])
+        q = "CREATE (section:Section { Name: "+c['name']+", Code: "+c['code']+", Crn: "+c['crn']+", Teacher: "+c['teacher']+" })"
+        q = 'CREATE (section:Section { Name: "%s", Code: "%s", Crn: "%s", Teacher: "%s" })' % \
+            (c['name'], c['code'], c['crn'], c['teacher']) 
+        gg.run(q)
+    """
+    #gg = HelloWorldExample("bolt://localhost:7687", "neo4j", "asdf")
+    #gg.print_greeting("hi there world")
+    """
+    make_teacher_rel(gg, "Peter Howell","CSIS 42")
+    make_teacher_rel(gg, "Alex Stoykov","CSIS 42")
+    make_teacher_rel(gg, "Sabrina Lawrence","CSIS 85")
+    make_teacher_rel(gg, "Peter Howell","CSIS 85")
+    """
+
+screen = 0
+
+def Memoize( func):
+    """
+    Memoize decorator
+    """
+    cache = {}
+
+    @wraps(func)
+    def wrapper(*args):
+        if args not in cache:
+            cache[args] = func(*args)
+        return cache[args]
+    return wrapper
+
+
+
+
+class MyRepl:
+    description = {
+      "switch ": "Switch stream. You can use either 'switch public' or 'switch mine'",
+      "home "  : "Show your timeline. 'home 7' will show 7 tweet.",
+      "harry "  : "a guys name.",
+      "homo "  : "means the same.",
+      "view "  : "'view @mdo' will show @mdo's home.",
+      "h "     : "Show help.",
+      "t "     : "'t opps' will tweet 'opps' immediately.",
+      "s "     : "'s #AKB48' will search for '#AKB48' and return 5 newest tweets."
+    }
+    
+    
+    def startup(self, outfile):
+        global screen   #    make it self
+        self.g = {}
+        self.buf = {}
+        screen = None
+        self.enter_ary = [curses.KEY_ENTER,10]
+        self.delete_ary = [curses.KEY_BACKSPACE,curses.KEY_DC,8,127,263]
+        self.tab_ary = [9]
+        self.up_ary = [curses.KEY_UP]
+        self.down_ary = [curses.KEY_DOWN]
+
+        # Init curses screen
+        screen = curses.initscr()
+        screen.keypad(1)
+        curses.noecho()
+        try:
+            curses.start_color()
+            curses.use_default_colors()
+            for i in range(0, curses.COLORS):
+                curses.init_pair(i + 1, i, -1)
+        except curses.error:
+            pass
+        curses.cbreak()
+        self.g['height'] , self.g['width'] = screen.getmaxyx()
+        #print("Width: %i" % self.g['width'])
+
+        # Init color function
+        s = self
+        self.white = lambda x:curses_print_word(x,7)  #0)
+        self.grey = lambda x:curses_print_word(x, 3) #3)1)
+        self.red = lambda x:curses_print_word(x,7)  #2)
+        self.green = lambda x:curses_print_word(x, 3) #3)
+        self.yellow = lambda x:curses_print_word(x,7)  #4)
+        self.blue = lambda x:curses_print_word(x,3)
+        self.magenta = lambda x:curses_print_word(x,7)  #6)
+        self.cyan = lambda x:curses_print_word(x,7)  #7)
+        self.colors_shuffle = [s.grey, s.red, s.green, s.yellow, s.blue, s.magenta, s.cyan]
+        self.cyc = itertools.cycle(s.colors_shuffle[1:])
+        self.index_cyc = itertools.cycle(range(1,8))
+        self.setup_command(outfile)
+
+
+    def set_my_dict(self,d):
+        self.description = d
+
+    @Memoize
+    def cycle_color(self, s):
+        """
+        Cycle the colors_shuffle
+        """
+        return next(self.cyc)
+
+
+    def ascii_art(self, text):
+        """
+        Draw the Ascii Art
+        """
+        fi = figlet_format(text, font='doom')
+        for i in fi.split('\n'):
+            self.curses_print_line(i,next(self.index_cyc))
+
+
+    def close_window(self, ):
+        """
+        Close screen
+        """
+        global screen
+        screen.keypad(0);
+        curses.nocbreak();
+        curses.echo()
+        curses.endwin()
+
+
+    def suggest(self, word):
+        """
+        Find suggestion
+        """
+        rel = []
+        if not word: return rel
+        word = word.lower()
+        
+        for candidate in self.description:
+            
+            ca = candidate.lower()
+            #if ca.startswith(word): rel.append(candidate)
+                
+            for eachword in ca.split(" "):
+                if eachword.startswith(word):
+                    rel.append(candidate)
+                    
+        return rel
+
+
+    def curses_print_word(self, word,color_pair_code):
+        """
+        Print a word
+        """
+        global screen
+        word = word.encode('utf8')
+        screen.addstr(word,curses.color_pair(color_pair_code))
+
+
+    def curses_print_line(self, line,color_pair_code):
+        """
+        Print a line, scroll down if need
+        """
+        global screen
+        line = line.encode('utf8')
+        y,x = screen.getyx()
+        if y - self.g['height'] == -3:
+            self.scroll_down(2,y,x)
+            screen.addstr(y,0,line,curses.color_pair(color_pair_code))
+            self.buf[y] = line, color_pair_code
+        elif y - self.g['height'] == -2:
+            self.scroll_down(3,y,x)
+            screen.addstr(y-1,0,line,curses.color_pair(color_pair_code))
+            self.buf[y-1] = line ,color_pair_code
+        else:
+            screen.addstr(y+1,0,line,curses.color_pair(color_pair_code))
+            self.buf[y+1] = line, color_pair_code
+
+
+    def redraw(self, start_y,end_y,fallback_y,fallback_x):
+        """
+        Redraw lines from buf
+        """
+        global screen
+        for cursor in range(start_y,end_y):
+            screen.move(cursor,0)
+            screen.clrtoeol()
+            try:
+                line, color_pair_code = self.buf[cursor]
+                screen.addstr(cursor,0,line,curses.color_pair(color_pair_code))
+            except:
+                pass
+        screen.move(fallback_y,fallback_x)
+
+
+    def scroll_down(self, noredraw,fallback_y,fallback_x):
+        """
+        Scroll down 1 line
+        """
+        global screen
+        # Recreate buf
+        # noredraw = n means that screen will scroll down n-1 line
+        trip_list = heapq.nlargest(noredraw-1,buf)
+        for i in buf:
+            if i not in trip_list:
+                self.buf[i] = self.buf[i+noredraw-1]
+        for j in trip_list:
+            buf.pop(j)
+
+        # Clear and redraw
+        screen.clear()
+        self.redraw(1,g['height']-noredraw,fallback_y,fallback_x)
+
+
+    def clear_upside(self, n,y,x):
+        """
+        Clear n lines upside
+        """
+        global screen
+        for i in range(1,n+1):
+            screen.move(y-i,0)
+            screen.clrtoeol()
+        screen.refresh()
+        screen.move(y,x)
+
+
+    def display_suggest(self, y,x,word):
+        """
+        Display box of suggestion
+        """
+        global screen
+        g = self.g
+        side = 2
+
+        # Check if need to print upside
+        upside = y+6 > int(g['height'])
+
+        # Redraw if suggestion is not the same as previous display
+        sug = self.suggest(word)
+        if sug != self.g['prev']:
+            # 0-line means there is no suggetions (height = 0)
+            # 3-line means there are many suggetions (height = 3)
+            # 5-line means there is only one suggetions (height = 5)
+            # Clear upside section
+            if upside:
+                # Clear upside is a bit difficult. Here it's seperate to 4 case.
+                # now: 3-lines / previous : 0 line
+                if len(sug) > 1 and not self.g['prev']:
+                    self.clear_upside(3,y,x)
+                # now: 0-lines / previous :3 lines
+                elif not sug and len(g['prev'])>1:
+                    self.redraw(y-3,y,y,x)
+                # now: 3-lines / previous :5 lines
+                elif len(sug) > 1 == len(g['prev']):
+                    self.redraw(y-5,y-3,y,x)
+                    self.clear_upside(3,y,x)
+                # now: 5-lines / previous :3 lines
+                elif len(sug) == 1 < len(g['prev']):
+                    self.clear_upside(3,y,x)
+                # now: 0-lines / previous :5 lines
+                elif not sug and len(g['prev'])==1:
+                    self.redraw(y-5,y,y,x)
+                # now: 3-lines / previous :3 lines
+                elif len(sug) == len(g['prev']) > 1:
+                    self.clear_upside(3,y,x)
+                # now: 5-lines / previous :5 lines
+                elif len(sug) == len(g['prev']) == 1:
+                    self.clear_upside(5,y,x)
+                screen.refresh()
+            else:
+            # Clear downside
+                screen.clrtobot()
+                screen.refresh()
+        self.g['prev'] = sug
+
+        if sug:
+            # More than 1 suggestion
+            if len(sug) > 1:
+                if len(sug) > 5: sug = sug[:5]
+                    
+                #needed_lenth = sum([len(i)+side for i in sug]) + side
+                needed_lenth = max( self.g['width']-5, sum([len(i)+side for i in sug]) + side)
+                print(self.g['width'])
+                print(word)
+                print(sug)
+                print(needed_lenth)
+                if upside:
+                    win = curses.newwin(3,needed_lenth,y-3,0)
+                    win.erase()
+                    win.box()
+                    win.refresh()
+                    cur_width = side
+                    for i in range(len(sug)):
+                        if cur_width+len(sug[i]) > self.g['width']: break
+                        screen.addstr(y-2,cur_width,sug[i],curses.color_pair(4))
+                        cur_width += len(sug[i]) + side
+                        if cur_width > self.g['width']:
+                            break
+                else:
+                    win = curses.newwin(3,needed_lenth,y+1,0)
+                    win.erase()
+                    win.box()
+                    win.refresh()
+                    cur_width = side
+                    for i in range(len(sug)):
+                        screen.addstr(y+2,cur_width,sug[i],curses.color_pair(4))
+                        cur_width += len(sug[i]) + side
+                        if cur_width > self.g['width']:
+                            break
+            # Only 1 suggestion
+            else:
+                can = sug[0]
+                if upside:
+                    win = curses.newwin(5,len(self.description[can])+2*side,y-5,0)
+                    win.box()
+                    win.refresh()
+                    screen.addstr(y-4,side,can,curses.color_pair(4))
+                    screen.addstr(y-2,side,self.description[can],curses.color_pair(3))
+                else:
+                    win = curses.newwin(5,len(self.description[can])+2*side,y+1,0)
+                    win.box()
+                    win.refresh()
+                    screen.addstr(y+2,side,can,curses.color_pair(4))
+                    screen.addstr(y+4,side,self.description[can],curses.color_pair(3))
+
+
+    def inputloop(self, ):
+        """
+        Main loop input
+        """
+        global screen
+        word = ''
+        screen.addstr("\n" + self.g['prefix'],curses.color_pair(7))
+
+        while True:
+            # Current position
+            y,x = screen.getyx()
+            # Get char
+            event = screen.getch()
+            try :
+                char = chr(event)
+            except:
+                char = ''
+
+            # Test curses_print_line
+            if char == '?':
+                self.buf[y] = self.g['prefix'] + '?', 0
+                self.ascii_art('dtvd88')
+
+            # TAB to complete
+            elif event in self.tab_ary:
+                # First tab
+                try:
+                    if not self.g['tab_cycle']:
+                        self.g['tab_cycle'] = itertools.cycle(self.suggest(word))
+
+                    suggestion = next(self.g['tab_cycle'])
+                    # Clear current line
+                    screen.move(y,len(self.g['prefix']))
+                    screen.clrtoeol()
+                    # Print out suggestion
+                    word = suggestion
+                    screen.addstr(y,len(self.g['prefix']),word)
+                    self.display_suggest(y,x,word)
+                    screen.move(y,len(word)+len(self.g['prefix']))
+                except:
+                    pass
+
+            # UP key
+            elif event in self.up_ary:
+                if self.g['hist']:
+                    # Clear current line
+                    screen.move(y,len(self.g['prefix']))
+                    screen.clrtoeol()
+                    # Print out previous history
+                    if self.g['hist_index'] > 0 - len(self.g['hist']):
+                        self.g['hist_index'] -= 1
+                    word = self.g['hist'][self.g['hist_index']]
+                    screen.addstr(y,len(self.g['prefix']),word)
+                    self.display_suggest(y,x,word)
+                    screen.move(y,len(word)+len(self.g['prefix']))
+
+            # DOWN key
+            elif event in self.down_ary:
+                if self.g['hist']:
+                    # clear current line
+                    screen.move(y,len(self.g['prefix']))
+                    screen.clrtoeol()
+                    # print out previous history
+                    if not self.g['hist_index']:
+                        self.g['hist_index'] = -1
+                    if self.g['hist_index'] < -1:
+                        self.g['hist_index'] += 1
+                    word = self.g['hist'][self.g['hist_index']]
+                    screen.addstr(y,len(self.g['prefix']),word)
+                    self.display_suggest(y,x,word)
+                    screen.move(y,len(word)+len(self.g['prefix']))
+
+            # Enter key                  #### I should get the command out of there?
+            #                            ####  Can I register a callback function?
+            
+            elif event in self.enter_ary:
+                self.g['tab_cycle'] = None
+                self.g['hist_index'] = 0
+                self.g['hist'].append(word)
+                if word== 'q':
+                    self.cleanup_command()
+                    break;
+                self.display_suggest(y,x,'')
+                screen.clrtobot()
+                self.handle_command(word)
+                
+                self.buf[y] = self.g['prefix'] + word, 0
+                # Touch the screen's end
+                if y - self.g['height'] > -3:
+                    self.scroll_down(2,y,x)
+                    screen.addstr(y,0,self.g['prefix'],curses.color_pair(7))     ##  SHOW NEW PROMPT
+                else:
+                    screen.addstr(y+1,0,self.g['prefix'],curses.color_pair(7))
+                word = ''
+
+            # Delete / Backspace
+            elif event in self.delete_ary:
+                self.g['tab_cycle'] = None
+                # Touch to line start
+                if x < len(self.g['prefix']) + 1:
+                    screen.move(y,x)
+                    word = ''
+                # Midle of line
+                else:
+                    word = word[:-1]
+                    screen.move(y,x-1)
+                    screen.clrtoeol()
+                    self.display_suggest(y,x,word)
+                    screen.move(y,x-1)
+
+            # Another keys
+            else:
+                self.g['tab_cycle'] = None
+            # Explicitly print char
+                try:
+                    screen.addstr(char)
+                    word += char
+                    self.display_suggest(y,x,word)
+                    screen.move(y,x+1)
+                except ValueError as e:   # got errors here when i adjusted the volume....
+                    pass
+                    
+        # Reset
+        self.close_window()
+
+    def setup_command(self,outfile):
+        self.data = open(outfile,'a')
+        
+        self.g['prev'] = None
+        self.g['tab_cycle'] = None
+        self.g['prefix'] = '[gav]: '
+        self.g['hist_index'] = 0
+        # Load history from previous session
+        try:
+            o = open('completer.hist')
+            self.g['hist'] = [i.strip() for i in o.readlines()]
+        except:
+            self.g['hist'] = []
+
+    def cleanup_command(self):
+        o = open('completer.hist','a')
+        o.write("\n".join(self.g['hist']))
+        o.close()
+        self.data.close()
+
+    def handle_command(self, cmd):
+        r1 = re.search( r'^n\s(.*)$',cmd)
+        if r1:
+            # new data collection mode
+            mode = r1.group(1)
+            self.g['prefix'] = "[" + mode + "]"
+            
+            self.data.write("\n\n# %s\n" % mode)
+        else:
+            #winsound.Beep(440,300)
+            self.data.write(cmd + "\n")
+            self.data.flush()
+        
+            
+
+def repl_staff():
+    
+    tch = json.loads( open('cache/teacherdata/teachers.json','r').read() )
+    newdict = {}
+    for T in tch:
+        newdict[T['name']] = 'teacher with id ' + T['login_id']
+    c = MyRepl()
+    
+    c.set_my_dict(newdict)
+    c.startup('cache/people_logs.txt')
+    c.inputloop()
+    
+
+def repl_degs():
+    
+    tch = csv.reader( open('cache/attainment_masterlist.csv','r'),delimiter=",")
+    
+    newdict = {}
+    num = 0
+    for row in tch:
+        if num==0:
+            pass
+        else:
+            d = ' '
+            if row[0]: d = row[0]
+            newdict[row[4]] = d
+        num += 1
+        
+    #print(newdict)
+    #input('ready')
+    c = MyRepl()
+    
+    c.set_my_dict(newdict)
+
+#c.startup('cache/g_path_cluster2020_.txt')
+#    c.inputloop()
+
+def repl():
+    repl_degs()
+
+
+    
+    #input('ready')
+    c = MyRepl()
+    
+    c.set_my_dict(newdict)
+
+#c.startup('cache/g_path_cluster2020_.txt')
+#    c.inputloop()
+
+def repl():
+    repl_degs()
+
+
+    
diff --git a/interactivex.py b/interactivex.py
new file mode 100644
index 0000000..cfd67fe
--- /dev/null
+++ b/interactivex.py
@@ -0,0 +1,759 @@
+import curses
+import heapq, re, csv
+import itertools, time, markdown, csv, json, os.path, webbrowser, threading
+from functools import wraps
+from flask import Flask, request, send_from_directory, Response
+from werkzeug.routing import PathConverter
+from queue import Queue
+from flask import render_template
+from importlib import reload 
+
+import server
+import localcache
+from server import *
+
+    
+ 
+
+
+q = Queue() 
+
+
+
+HOST_NAME = '127.0.0.1' # 
+HOST_NAME = '192.168.1.6' # 
+HOST_NAME = '192.168.1.6' # 
+PORT_NUMBER = 8080 # Maybe set this to 9000.
+
+datafile = 'lambda.csv'
+
+
+####
+####   This little web server is going to work with the "gui" folder / vue app
+####
+####
+
+
+
+def dict_generator(indict, pre=None):
+    pre = pre[:] if pre else []
+    if isinstance(indict, dict):
+        for key, value in indict.items():
+            if isinstance(value, dict):
+                for d in dict_generator(value, pre + [key]):
+                    yield d
+            elif isinstance(value, list) or isinstance(value, tuple):
+                for v in value:
+                    for d in dict_generator(v, pre + [key]):
+                        yield d
+            else:
+                yield str(pre) + "   " + str([key, value]) + "\n"
+    else:
+        yield pre + [indict]
+        yield str(pre) + "   " + str([indict]) + "\n"
+        
+ 
+ 
+def print_dict(v, prefix='',indent=''):
+    if isinstance(v, dict):
+        return [   print_dict(v2, "{}['{}']".format(prefix, k) + "<br />", indent+" " )  for k, v2 in v.items()  ]
+    elif isinstance(v, list):
+        return [ print_dict(  v2,  "{}[{}]".format(prefix , i) + "<br />", indent+" ")  for i, v2 in enumerate(v) ]
+    else:
+        return '{} = {}'.format(prefix, repr(v)) + "\n"
+        
+
+def walk_file():
+    j = json.loads(open('cache/programs/programs_2.txt','r').read())
+    
+    return print_dict(j)
+
+
+def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+
+def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+
+def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+def server_save(key,value):
+    codecs.open('cache/server_data.txt','a').write( "%s=%s\n" % (str(key),str(value)))
+
+def flask_thread(q):
+    #app = Flask(__name__, static_url_path='/cache', 
+    #        static_folder='cache',)
+    app = Flask(__name__)
+    app.jinja_env.auto_reload = True
+    app.config['TEMPLATES_AUTO_RELOAD'] = True
+
+    def before_request():
+        app.jinja_env.cache = {}
+
+    app.before_request(before_request)
+
+
+    # 
+    # SERVER maintenance type stuff
+    @app.route('/rl')
+    def restart():
+        reload(server)
+        reload(localcache)
+        return "Server code reloaded"
+
+    @app.route("/x/<func>/<arg>/<arrg>")
+    def dispatch3(func,arg,arrg):
+        print("2 args")
+        return "" + server_dispatch(func, arg, arrg)
+
+    @app.route("/x/<func>/<arg>")
+    def dispatch2(func,arg):
+        print("1 arg")
+        return "" + server_dispatch(func, arg)
+
+    @app.route("/x/<func>")
+    def dispatch(func):
+        print("0 arg")
+        return server_dispatch(func)
+
+    @app.route("/api/<func>/<arg>/<arrg>")
+    def dispatch3j(func,arg,arrg):
+        print("json, 3 args")
+        return Response(server_dispatch(func, arg, arrg), mimetype='text/json')
+
+    @app.route("/api/<func>/<arg>")
+    def dispatch2j(func,arg):
+        print("json, 1 arg")
+        return Response(server_dispatch(func, arg), mimetype='text/json')
+
+    @app.route("/api/<func>")
+    def dispatch1j(func):
+        print("json, 0 arg")
+        return Response(server_dispatch(func), mimetype='text/json')
+
+    @app.route("/")
+    def home():
+        return server.homepage()
+    
+    #
+    #  STATIC ROUTES
+    #
+
+    """@app.route('/lib/<path:path>')
+    def send_jslib(path):
+        return send_from_directory('gui/lib', path)"""
+    
+    @app.route('/data/<path:path>')
+    def send_cachedata(path):
+        #myfile = os.path.join('cache', path).replace('\\','/')
+        print(path)
+        #return app.send_static_file(myfile)
+        return send_from_directory('cache', path)
+    
+    # Departments, classes in each, and students (with hits) in each of those.
+
+    """@app.route('/iii/<path:path>')
+    def send_js(path):
+        return send_from_directory('gui/dist', path)"""
+    
+    #@app.route('/hello/')
+    #@app.route('/hello/<name>')
+
+
+    @app.route("/save/<key>/<val>")
+    def s(key,val):
+        server_save(key,val)
+        return tag('h1','Saved.') + "<br />" + tag('p', 'Saved: %s = %s' % (str(key),str(val)))
+
+    @app.route("/sample")
+    def do_sample():
+        return sample()
+
+
+    @app.route("/crazy")
+    def hello():
+        r = '<link rel="stylesheet" href="static/bootstrap.min.css">'
+        r += tag('style', 'textarea { white-space:nowrap; }')
+        r += tag('body', \
+            tagc('div','container-fluid', \
+                tagc('div','row', \
+                    tagc( 'div', 'col-md-6', tag('pre', walk_file() ) ) + \
+                    tagc( 'div', 'col-md-6', 'Column 2' + a('Shut Down','/shutdown' ) ) ) ) )
+        
+         
+        
+        return r
+    
+    @app.route("/sd")
+    def sd():
+        print('SIGINT or CTRL-C detected. Exiting gracefully')
+        func = request.environ.get('werkzeug.server.shutdown')
+        if func is None:
+            raise RuntimeError('Not running with the Werkzeug Server')
+        func()
+        return "Server has shut down."
+    app.run(host= '0.0.0.0')
+    
+    
+
+def serve():
+    x = threading.Thread(target=flask_thread, args=(q,))
+    x.start()
+    webbrowser.open_new_tab("http://localhost:5000")
+
+
+if __name__ == '__main__':
+    serve()
+
+
+
+
+
+"""class HelloWorldExample(object):
+    def make_teacher_rel(self, tchr, clss):
+        with self._driver.session() as tx:
+            tx.run("MERGE (tchr:Teacher {name: $tchr}) MERGE (tchr)-[:TEACHES]->(clss:Class {name: $clss})", \
+           tchr=tchr, clss=clss)
+
+    def __init__(self, uri, user, password):
+        self._driver = GraphDatabase.driver(uri, auth=(user, password))
+
+    def close(self):
+        self._driver.close()
+
+
+
+    def print_greeting(self, message):
+        with self._driver.session() as session:
+            greeting = session.write_transaction(self._create_and_return_greeting, message)
+            print(greeting)
+
+    @staticmethod
+    def _create_and_return_greeting(tx, message):
+        result = tx.run("CREATE (a:Greeting) "
+                        "SET a.message = $message "
+                        "RETURN a.message + ', from node ' + id(a)", message=message)
+        return result.single()[0]
+"""        
+        
+
+def make_teacher_rel(g, tchr, clss):
+    g.run("MERGE (tchr:Teacher {name: $tchr}) MERGE (tchr)-[:TEACHES]->(clss:Class {name: $clss})", \
+       tchr=tchr, clss=clss)
+           
+
+
+screen = 0
+
+def Memoize( func):
+    """
+    Memoize decorator
+    """
+    cache = {}
+
+    @wraps(func)
+    def wrapper(*args):
+        if args not in cache:
+            cache[args] = func(*args)
+        return cache[args]
+    return wrapper
+
+
+
+
+class MyRepl:
+    description = {
+      "switch ": "Switch stream. You can use either 'switch public' or 'switch mine'",
+      "home "  : "Show your timeline. 'home 7' will show 7 tweet.",
+      "harry "  : "a guys name.",
+      "homo "  : "means the same.",
+      "view "  : "'view @mdo' will show @mdo's home.",
+      "h "     : "Show help.",
+      "t "     : "'t opps' will tweet 'opps' immediately.",
+      "s "     : "'s #AKB48' will search for '#AKB48' and return 5 newest tweets."
+    }
+    
+    
+    def startup(self, outfile):
+        global screen   #    make it self
+        self.g = {}
+        self.buf = {}
+        screen = None
+        self.enter_ary = [curses.KEY_ENTER,10]
+        self.delete_ary = [curses.KEY_BACKSPACE,curses.KEY_DC,8,127,263]
+        self.tab_ary = [9]
+        self.up_ary = [curses.KEY_UP]
+        self.down_ary = [curses.KEY_DOWN]
+
+        # Init curses screen
+        screen = curses.initscr()
+        screen.keypad(1)
+        curses.noecho()
+        try:
+            curses.start_color()
+            curses.use_default_colors()
+            for i in range(0, curses.COLORS):
+                curses.init_pair(i + 1, i, -1)
+        except curses.error:
+            pass
+        curses.cbreak()
+        self.g['height'] , self.g['width'] = screen.getmaxyx()
+        #print("Width: %i" % self.g['width'])
+
+        # Init color function
+        s = self
+        self.white = lambda x:curses_print_word(x,7)  #0)
+        self.grey = lambda x:curses_print_word(x, 3) #3)1)
+        self.red = lambda x:curses_print_word(x,7)  #2)
+        self.green = lambda x:curses_print_word(x, 3) #3)
+        self.yellow = lambda x:curses_print_word(x,7)  #4)
+        self.blue = lambda x:curses_print_word(x,3)
+        self.magenta = lambda x:curses_print_word(x,7)  #6)
+        self.cyan = lambda x:curses_print_word(x,7)  #7)
+        self.colors_shuffle = [s.grey, s.red, s.green, s.yellow, s.blue, s.magenta, s.cyan]
+        self.cyc = itertools.cycle(s.colors_shuffle[1:])
+        self.index_cyc = itertools.cycle(range(1,8))
+        self.setup_command(outfile)
+
+
+    def set_my_dict(self,d):
+        self.description = d
+
+    @Memoize
+    def cycle_color(self, s):
+        """
+        Cycle the colors_shuffle
+        """
+        return next(self.cyc)
+
+
+    def ascii_art(self, text):
+        """
+        Draw the Ascii Art
+        """
+        return
+        #fi = figlet_format(text, font='doom')
+        #for i in fi.split('\n'):
+        #self.curses_print_line(i,next(self.index_cyc))
+
+
+    def close_window(self, ):
+        """
+        Close screen
+        """
+        global screen
+        screen.keypad(0);
+        curses.nocbreak();
+        curses.echo()
+        curses.endwin()
+
+
+    def suggest(self, word):
+        """
+        Find suggestion
+        """
+        rel = []
+        if not word: return rel
+        word = word.lower()
+        
+        for candidate in self.description:
+            
+            ca = candidate.lower()
+            #if ca.startswith(word): rel.append(candidate)
+                
+            for eachword in ca.split(" "):
+                if eachword.startswith(word):
+                    rel.append(candidate)
+                    
+        return rel
+
+
+    def curses_print_word(self, word,color_pair_code):
+        """
+        Print a word
+        """
+        global screen
+        word = word.encode('utf8')
+        screen.addstr(word,curses.color_pair(color_pair_code))
+
+
+    def curses_print_line(self, line,color_pair_code):
+        """
+        Print a line, scroll down if need
+        """
+        global screen
+        line = line.encode('utf8')
+        y,x = screen.getyx()
+        if y - self.g['height'] == -3:
+            self.scroll_down(2,y,x)
+            screen.addstr(y,0,line,curses.color_pair(color_pair_code))
+            self.buf[y] = line, color_pair_code
+        elif y - self.g['height'] == -2:
+            self.scroll_down(3,y,x)
+            screen.addstr(y-1,0,line,curses.color_pair(color_pair_code))
+            self.buf[y-1] = line ,color_pair_code
+        else:
+            screen.addstr(y+1,0,line,curses.color_pair(color_pair_code))
+            self.buf[y+1] = line, color_pair_code
+
+
+    def redraw(self, start_y,end_y,fallback_y,fallback_x):
+        """
+        Redraw lines from buf
+        """
+        global screen
+        for cursor in range(start_y,end_y):
+            screen.move(cursor,0)
+            screen.clrtoeol()
+            try:
+                line, color_pair_code = self.buf[cursor]
+                screen.addstr(cursor,0,line,curses.color_pair(color_pair_code))
+            except:
+                pass
+        screen.move(fallback_y,fallback_x)
+
+
+    def scroll_down(self, noredraw,fallback_y,fallback_x):
+        """
+        Scroll down 1 line
+        """
+        global screen
+        # Recreate buf
+        # noredraw = n means that screen will scroll down n-1 line
+        trip_list = heapq.nlargest(noredraw-1,buf)
+        for i in buf:
+            if i not in trip_list:
+                self.buf[i] = self.buf[i+noredraw-1]
+        for j in trip_list:
+            buf.pop(j)
+
+        # Clear and redraw
+        screen.clear()
+        self.redraw(1,g['height']-noredraw,fallback_y,fallback_x)
+
+
+    def clear_upside(self, n,y,x):
+        """
+        Clear n lines upside
+        """
+        global screen
+        for i in range(1,n+1):
+            screen.move(y-i,0)
+            screen.clrtoeol()
+        screen.refresh()
+        screen.move(y,x)
+
+
+    def display_suggest(self, y,x,word):
+        """
+        Display box of suggestion
+        """
+        global screen
+        g = self.g
+        side = 2
+
+        # Check if need to print upside
+        upside = y+6 > int(g['height'])
+
+        # Redraw if suggestion is not the same as previous display
+        sug = self.suggest(word)
+        if sug != self.g['prev']:
+            # 0-line means there is no suggetions (height = 0)
+            # 3-line means there are many suggetions (height = 3)
+            # 5-line means there is only one suggetions (height = 5)
+            # Clear upside section
+            if upside:
+                # Clear upside is a bit difficult. Here it's seperate to 4 case.
+                # now: 3-lines / previous : 0 line
+                if len(sug) > 1 and not self.g['prev']:
+                    self.clear_upside(3,y,x)
+                # now: 0-lines / previous :3 lines
+                elif not sug and len(g['prev'])>1:
+                    self.redraw(y-3,y,y,x)
+                # now: 3-lines / previous :5 lines
+                elif len(sug) > 1 == len(g['prev']):
+                    self.redraw(y-5,y-3,y,x)
+                    self.clear_upside(3,y,x)
+                # now: 5-lines / previous :3 lines
+                elif len(sug) == 1 < len(g['prev']):
+                    self.clear_upside(3,y,x)
+                # now: 0-lines / previous :5 lines
+                elif not sug and len(g['prev'])==1:
+                    self.redraw(y-5,y,y,x)
+                # now: 3-lines / previous :3 lines
+                elif len(sug) == len(g['prev']) > 1:
+                    self.clear_upside(3,y,x)
+                # now: 5-lines / previous :5 lines
+                elif len(sug) == len(g['prev']) == 1:
+                    self.clear_upside(5,y,x)
+                screen.refresh()
+            else:
+            # Clear downside
+                screen.clrtobot()
+                screen.refresh()
+        self.g['prev'] = sug
+
+        if sug:
+            # More than 1 suggestion
+            if len(sug) > 1:
+                if len(sug) > 5: sug = sug[:5]
+                    
+                #needed_lenth = sum([len(i)+side for i in sug]) + side
+                needed_lenth = max( self.g['width']-5, sum([len(i)+side for i in sug]) + side)
+                print(self.g['width'])
+                print(word)
+                print(sug)
+                print(needed_lenth)
+                if upside:
+                    win = curses.newwin(3,needed_lenth,y-3,0)
+                    win.erase()
+                    win.box()
+                    win.refresh()
+                    cur_width = side
+                    for i in range(len(sug)):
+                        if cur_width+len(sug[i]) > self.g['width']: break
+                        screen.addstr(y-2,cur_width,sug[i],curses.color_pair(4))
+                        cur_width += len(sug[i]) + side
+                        if cur_width > self.g['width']:
+                            break
+                else:
+                    win = curses.newwin(3,needed_lenth,y+1,0)
+                    win.erase()
+                    win.box()
+                    win.refresh()
+                    cur_width = side
+                    for i in range(len(sug)):
+                        screen.addstr(y+2,cur_width,sug[i],curses.color_pair(4))
+                        cur_width += len(sug[i]) + side
+                        if cur_width > self.g['width']:
+                            break
+            # Only 1 suggestion
+            else:
+                can = sug[0]
+                if upside:
+                    win = curses.newwin(5,len(self.description[can])+2*side,y-5,0)
+                    win.box()
+                    win.refresh()
+                    screen.addstr(y-4,side,can,curses.color_pair(4))
+                    screen.addstr(y-2,side,self.description[can],curses.color_pair(3))
+                else:
+                    win = curses.newwin(5,len(self.description[can])+2*side,y+1,0)
+                    win.box()
+                    win.refresh()
+                    screen.addstr(y+2,side,can,curses.color_pair(4))
+                    screen.addstr(y+4,side,self.description[can],curses.color_pair(3))
+
+
+    def inputloop(self, ):
+        """
+        Main loop input
+        """
+        global screen
+        word = ''
+        screen.addstr("\n" + self.g['prefix'],curses.color_pair(7))
+
+        while True:
+            # Current position
+            y,x = screen.getyx()
+            # Get char
+            event = screen.getch()
+            try :
+                char = chr(event)
+            except:
+                char = ''
+
+            # Test curses_print_line
+            if char == '?':
+                self.buf[y] = self.g['prefix'] + '?', 0
+                self.ascii_art('dtvd88')
+
+            # TAB to complete
+            elif event in self.tab_ary:
+                # First tab
+                try:
+                    if not self.g['tab_cycle']:
+                        self.g['tab_cycle'] = itertools.cycle(self.suggest(word))
+
+                    suggestion = next(self.g['tab_cycle'])
+                    # Clear current line
+                    screen.move(y,len(self.g['prefix']))
+                    screen.clrtoeol()
+                    # Print out suggestion
+                    word = suggestion
+                    screen.addstr(y,len(self.g['prefix']),word)
+                    self.display_suggest(y,x,word)
+                    screen.move(y,len(word)+len(self.g['prefix']))
+                except:
+                    pass
+
+            # UP key
+            elif event in self.up_ary:
+                if self.g['hist']:
+                    # Clear current line
+                    screen.move(y,len(self.g['prefix']))
+                    screen.clrtoeol()
+                    # Print out previous history
+                    if self.g['hist_index'] > 0 - len(self.g['hist']):
+                        self.g['hist_index'] -= 1
+                    word = self.g['hist'][self.g['hist_index']]
+                    screen.addstr(y,len(self.g['prefix']),word)
+                    self.display_suggest(y,x,word)
+                    screen.move(y,len(word)+len(self.g['prefix']))
+
+            # DOWN key
+            elif event in self.down_ary:
+                if self.g['hist']:
+                    # clear current line
+                    screen.move(y,len(self.g['prefix']))
+                    screen.clrtoeol()
+                    # print out previous history
+                    if not self.g['hist_index']:
+                        self.g['hist_index'] = -1
+                    if self.g['hist_index'] < -1:
+                        self.g['hist_index'] += 1
+                    word = self.g['hist'][self.g['hist_index']]
+                    screen.addstr(y,len(self.g['prefix']),word)
+                    self.display_suggest(y,x,word)
+                    screen.move(y,len(word)+len(self.g['prefix']))
+
+            # Enter key                  #### I should get the command out of there?
+            #                            ####  Can I register a callback function?
+            
+            elif event in self.enter_ary:
+                self.g['tab_cycle'] = None
+                self.g['hist_index'] = 0
+                self.g['hist'].append(word)
+                if word== 'q':
+                    self.cleanup_command()
+                    break;
+                self.display_suggest(y,x,'')
+                screen.clrtobot()
+                self.handle_command(word)
+                
+                self.buf[y] = self.g['prefix'] + word, 0
+                # Touch the screen's end
+                if y - self.g['height'] > -3:
+                    self.scroll_down(2,y,x)
+                    screen.addstr(y,0,self.g['prefix'],curses.color_pair(7))     ##  SHOW NEW PROMPT
+                else:
+                    screen.addstr(y+1,0,self.g['prefix'],curses.color_pair(7))
+                word = ''
+
+            # Delete / Backspace
+            elif event in self.delete_ary:
+                self.g['tab_cycle'] = None
+                # Touch to line start
+                if x < len(self.g['prefix']) + 1:
+                    screen.move(y,x)
+                    word = ''
+                # Midle of line
+                else:
+                    word = word[:-1]
+                    screen.move(y,x-1)
+                    screen.clrtoeol()
+                    self.display_suggest(y,x,word)
+                    screen.move(y,x-1)
+
+            # Another keys
+            else:
+                self.g['tab_cycle'] = None
+            # Explicitly print char
+                try:
+                    screen.addstr(char)
+                    word += char
+                    self.display_suggest(y,x,word)
+                    screen.move(y,x+1)
+                except ValueError as e:   # got errors here when i adjusted the volume....
+                    pass
+                    
+        # Reset
+        self.close_window()
+
+    def setup_command(self,outfile):
+        self.data = open(outfile,'a')
+        
+        self.g['prev'] = None
+        self.g['tab_cycle'] = None
+        self.g['prefix'] = '[gav]: '
+        self.g['hist_index'] = 0
+        # Load history from previous session
+        try:
+            o = open('completer.hist')
+            self.g['hist'] = [i.strip() for i in o.readlines()]
+        except:
+            self.g['hist'] = []
+
+    def cleanup_command(self):
+        o = open('completer.hist','a')
+        o.write("\n".join(self.g['hist']))
+        o.close()
+        self.data.close()
+
+    def handle_command(self, cmd):
+        r1 = re.search( r'^n\s(.*)$',cmd)
+        if r1:
+            # new data collection mode
+            mode = r1.group(1)
+            self.g['prefix'] = "[" + mode + "]"
+            
+            self.data.write("\n\n# %s\n" % mode)
+        else:
+            #winsound.Beep(440,300)
+            self.data.write(cmd + "\n")
+            self.data.flush()
+        
+            
+
+def repl_staff():
+    
+    tch = json.loads( open('cache/teacherdata/teachers.json','r').read() )
+    newdict = {}
+    for T in tch:
+        newdict[T['name']] = 'teacher with id ' + T['login_id']
+    c = MyRepl()
+    
+    c.set_my_dict(newdict)
+    c.startup('cache/people_logs.txt')
+    c.inputloop()
+    
+
+def repl_degs():
+    
+    tch = csv.reader( open('cache/attainment_masterlist.csv','r'),delimiter=",")
+    
+    newdict = {}
+    num = 0
+    for row in tch:
+        if num==0:
+            pass
+        else:
+            d = ' '
+            if row[0]: d = row[0]
+            newdict[row[4]] = d
+        num += 1
+        
+    #print(newdict)
+    #input('ready')
+    c = MyRepl()
+    
+    c.set_my_dict(newdict)
+
+#c.startup('cache/g_path_cluster2020_.txt')
+#    c.inputloop()
+
+def repl():
+    repl_degs()
+
+
+    
+    #input('ready')
+    c = MyRepl()
+    
+    c.set_my_dict(newdict)
+
+#c.startup('cache/g_path_cluster2020_.txt')
+#    c.inputloop()
+
+def repl():
+    repl_degs()
+
+
+    
diff --git a/localcache.py b/localcache.py
new file mode 100644
index 0000000..3da9fb0
--- /dev/null
+++ b/localcache.py
@@ -0,0 +1,2065 @@
+# Local data, saving and manipulating
+
+import os, re, gzip, codecs, funcy, pytz, sqlite3, json, random, functools, requests, sys, csv
+import pandas as pd
+import numpy as np
+from collections import defaultdict
+from datetime import datetime as dt
+from datetime import timedelta
+from dateutil.parser import parse
+from os.path import exists, getmtime
+from pipelines import sync_non_interactive, url, header, gp, dean
+
+#from courses import getCoursesInTerm
+#from courses import user_in_depts_live
+
+mycourses = {}
+
+local_data_folder = 'cache/canvas_data/'
+sqlite_file = local_data_folder + 'data.db'    #'data_su20_4hr_blocks.db' 
+mylog = codecs.open(local_data_folder + 'canvas_data_log.txt','w')
+
+thefiles_dat = {}
+for L in open('cache/canvas_data_index.txt','r').readlines():
+    L = L.strip()
+    (fname,start,finish) = L.split(',')
+    thefiles_dat[fname] = start
+
+thefiles = open('cache/canvas_data_index_temp.txt','a')    # rename me if nothing crashes :)
+
+
+NUM_ONLY = 1    # use numeric codes instead of strings. For mathy stuff
+
+requests_sum1_format = "id userid courseid timeblock viewcount partcount".split(" ")
+requests_sum1_types = "INTEGER PRIMARY KEY AUTOINCREMENT,text,text,INTEGER,INTEGER,INTEGER".split(",")
+requests_format = "id timestamp year month day userid courseid rootid course_acct_id quizid discussionid conversationid assignmentid url useragent httpmethod remoteip micros controller action contexttype contextid realid sessionid agentid httpstatus httpversion developer_key_id time_block".split(" ")
+users_format = "id canvasid rootactid name tz created vis school position gender locale public bd cc state sortablename globalid".split(" ")
+cc_format = "id canvasid userid address type position state created updated".split(" ")
+term_format = "id canvasid rootid name start end sis".split(" ")
+course_format = "id canvasid rootactid acctid termid name code type created start conclude visible sis state wikiid".split(" ")
+role_format = "id canvas_id root_account_id account_id name base_role_type workflow_state created_at updated_at deleted_at".split(" ")
+course_score_format = "s_id c_id a_id course_id enrol_id current final muted_current muted_final".split(" ")
+enrollment_dim_format = "id cid root course_section role type workflow created updated start end complete self sis course_id user_id last_activity".split(" ")
+communication_channel_dim_format = "id canvas_id user_id address type position workflow_state created_at updated_at".split(" ")
+pseudonym_dim_format = "id canvas_id user_id account_id workflow_state last_request_at last_login_at current_login_at last_login_ip current_login_ip position created_at updated_at password_auto_generated deleted_at sis_user_id unique_name integration_id authentication_provider_id".split(" ")
+conversation_dim_format = "id canvas_id has_media_objects subject course_id group_id account_id".split(" ")
+conversation_message_dim_format = "id canvas_id conversation_id author_id created_at generated has_attachments has_media_objects body".split(" ")
+
+
+
+
+
+unwanted_req_paths = """conversations/unread_count
+CFNetwork
+TooLegit
+lti_user_id
+brand_variables
+dashboard-sidebar
+dashboard_cards
+ping
+self/profile
+login/oauth2
+login/session_token
+self/colors
+self/profile
+images/thumbnails
+auth/login
+auth/conversations
+backup/login
+blackboard ally
+Proctorio
+image_thumbnail
+manifest.json
+launch_definitions/login
+login
+python-requests
+custom_data
+content_shares
+pandata_events
+trypronto
+users/self """.split("\n")
+
+other_interesting_events = { }
+
+DB_CON = 0
+DB_CUR = 0
+
+#########
+######### LOCAL DB
+#########
+
+def db():
+    global DB_CON, DB_CUR
+    if DB_CON:
+        return (DB_CON,DB_CUR)
+    print('grabbing db connection')
+    DB_CON = sqlite3.connect(sqlite_file)
+    DB_CUR = DB_CON.cursor()
+    
+    return (DB_CON, DB_CUR)
+
+
+def setup_table(table='requests'):
+    (con,cur) = db()
+    q = ''
+
+
+    if table=='conversation':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS conversation (\n"
+        for L in conversation_dim_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+
+    if table=='conversation_message':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS conversation_message (\n"
+        for L in conversation_message_dim_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+
+    if table=='requests_sum1':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS requests_sum1 (\n"
+        for j, L in enumerate(requests_sum1_format):
+            if j: 
+                (col,typ) = (L,requests_sum1_types[j])
+                q += ",\n\t%s %s" % (col,typ)
+            else:
+                (col,typ) = (L,requests_sum1_types[j])
+                q += "\t%s %s" % (col,typ)
+                
+        q += "\n);\n"
+        print(q)
+        cur.execute(q)
+
+        q = "CREATE UNIQUE INDEX index1 ON requests_sum1(userid,courseid,timeblock);"
+
+    if table=='requests':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS requests (\n"
+        for L in open('cache/request_table.txt','r').readlines():
+            L = L.strip()
+            #print(L)
+            (col,type) = re.split("\s\s\s\s",L)
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+    if table=='users':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS users (\n"
+        for L in users_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+    if table=='pseudonym':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS pseudonym(\n"
+        for L in pseudonym_dim_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+
+
+    if table=='courses':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS courses (\n"
+        for L in course_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+    if table=='enrollment':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS enrollment (\n"
+        for L in enrollment_dim_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+    if table=='comm_channel':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS comm_channel (\n"
+        for L in communication_channel_dim_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+    if table=='terms':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS terms (\n"
+        for L in term_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        q += "\n);"
+
+    if table=='roles':
+        first = 1
+        q = "CREATE TABLE IF NOT EXISTS roles (\n"
+        for L in role_format:
+            (col,type) = (L,'text')
+            if not first: 
+                q += ",\n"
+            first = 0
+            q += "\t%s %s" % (col,type)
+        return q + "\n);"
+    if table == 'summary':
+        q = """CREATE TABLE "summary_course_user_views" (
+                "id"	INTEGER PRIMARY KEY AUTOINCREMENT,
+                "courseid"	TEXT,
+                "course_canvasid"	TEXT,
+                "username"	TEXT,
+                "userid"	TEXT,
+                "user_canvasid"	TEXT,
+                "count"	INTEGER,
+                "time_block"	INTEGER )"""
+    if q:
+        print(q)
+        cur.execute(q)
+        con.commit()
+        return
+
+    if table == 'index':
+        for q in [ #'CREATE INDEX "idx_req_userid" ON "requests" ("id","courseid","userid" );',
+                'CREATE INDEX "idx_users_id" ON "users" ("id","canvasid"  );',
+                'CREATE INDEX "idx_term_id" ON "terms" ("id","canvasid" );',
+                'CREATE INDEX "idx_enrollment" ON "enrollment" ("cid","course_id","user_id" );',
+                'CREATE INDEX "idx_courses" ON "courses" ("id","canvasid","termid" );' ]:
+            #print(q)
+            cur.execute(q)
+        con.commit()
+
+# Help the next function to upload new users directly to conf database on gavilan.
+def employees_refresh_flex(data):
+    try:
+        data['a'] = 'set/newuser'
+        data['sis_user_id'] = data['sis_user_id'][3:]
+        print("\nUploading this: \n")
+        print(json.dumps(data, indent=2))
+        print("\n")
+        a = input("Continue (y) or skip (n) ? ")
+        if a == 'y':
+            # This is what I was missing..........
+            # req.add_header("Content-type", "application/x-www-form-urlencoded")
+            r3 = requests.post('https://www.gavilan.edu/staff/flex/2020/api.php', params=data)
+            print(r3.text)
+        #print(r3.headers)
+    except Exception as ex:
+        print("Failed on: %s\nErr: %s" % (str(data),str(ex)))
+
+
+
+# Everyone in iLearn DB with an   xyz@gavilan.edu   email address. 
+def all_gav_employees():
+    (connection,cursor) = db()
+    connection.row_factory = dict_factory
+    q = """SELECT u.canvasid, u.name, u.created, u.sortablename, h.address, h.type, h.workflow_state, 
+           h.updated_at, p.last_request_at, p.last_login_at, p.current_login_at, p.last_login_ip, 
+           p.current_login_ip, p.sis_user_id, p.unique_name FROM users AS u
+           JOIN comm_channel AS h ON u.id=h.user_id
+           JOIN pseudonym AS p ON p.user_id=u.id
+           WHERE h.address LIKE "%@gavilan.edu"
+           ORDER BY u.sortablename"""
+    cursor = connection.cursor()
+    cursor.execute(q)
+    everyone = cursor.fetchall()
+    everyone_set = set()
+    for E in everyone:
+        try:
+            everyone_set.add(    E['address'].lower()  )
+        except Exception as e:
+            print("Exception: %s\nwith: %s" % (str(e), str(E)))
+    
+    oo = open('cache/temp1.txt','w')
+    oo.write(json.dumps(list(everyone_set), indent=2))
+    existing = requests.get('https://gavilan.edu/staff/flex/2020/api.php?a=get/users')
+    ex = json.loads( existing.text )
+    already_enrolled = set()
+    for usr in ex['users']:
+        try:
+            #already_enrolled.add(   (usr['goo'], usr['email'].lower(), usr['name'])  )
+            already_enrolled.add(  usr['email'].lower()   )
+        except Exception as e:
+            print("Exception: %s\nWith: %s" % (str(e),str(usr)))
+
+    oo.write( "\n"*20 + '------------------------------------------\n'*20 + '------ - - - - - - ' )
+    oo.write(json.dumps(list(already_enrolled), indent=2))
+
+    # conf_users wants:   goo, email, name, active
+    #   and emails have random capitalization
+    #   name is First Last,   and sometimes with Middle in there. 
+    #
+    
+    # using sets:     to_enroll = [ x for x in students if x not in already_enrolled ]     
+    new_emp = [ x for x in everyone_set if x not in already_enrolled ]
+
+    # take the all_employee list,   filter -> anyone who's in 'existing' is removed
+
+    # funcy.where( lambda x: x['email'] == ae[4]   ,  existing )
+
+    #new_emp = list(funcy.filter( lambda ae: funcy.where( existing, email=ae['email'] ),  all_emp ))
+    #new_emp = list(funcy.where( existing, email=b'phowell@gavilan.edu')) #ae['email'] ))
+    print(new_emp)
+    oo.write( "\n"*20 + '------------------------------------------\n'*20 + '------ - - - - - - ' )
+    oo.write(json.dumps(list(new_emp), indent=2))
+
+    # Now, iLearn db (everyone)... find the rows that match the email addresses
+    # that we've decided we need to add (new_emp)
+    
+    #print(everyone)
+    #print( "searching for %s" % j )
+    #print( "searched for %s, found: %s" % (j, str(to_add) ))
+    #print("\nUploading...\n")
+    for j in new_emp:
+        #j = new_emp[0]
+        print(j)
+        to_add = list(funcy.where( everyone, address=j ))
+        if to_add:
+            employees_refresh_flex(to_add[0])
+        else:
+            print("Didn't find an entry for that account.")
+    print("done uploading")
+
+#
+def teachers_courses_semester():
+    q = """SELECT c.id, c.canvasid AS course_cid, c.name, c.code, u.name, u.sortablename, u.canvasid AS user_cid  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.sis LIKE "202070-%"
+AND NOT c.state="deleted"
+AND e."type"="TeacherEnrollment"
+ORDER BY u.sortablename"""
+    (connection,cursor) = db()
+    cursor.execute(q)
+    all_teachers = cursor.fetchall()
+    return all_teachers
+# 
+def teachers_by_term():
+    q = """SELECT c.id as course_id, c.canvasid as course_c_id, c.name, c.code, c.created as course_created, c.start, c.visible, c.state, e.last_activity, 
+u.id as user_id, u.canvasid as user_c_id, u.sortablename, u.created as user_created 
+FROM courses AS c 
+JOIN enrollment AS e ON e.course_id=c.id 
+JOIN users AS u ON u.id=e.user_id
+WHERE c.sis LIKE "202070%"
+AND e."type"="TeacherEnrollment"
+ORDER BY c.code"""
+    (connection,cursor) = db()
+    cursor.execute(q)
+    all_teachers = cursor.fetchall()
+
+
+
+# Report for AEC
+def aec_su20_report():
+    global mycourses
+    #AE 600 (80040; 80045; 80047)  10945
+    #AE 602 (80048; 80049; 80050)  10746
+    #AE 636 (80332; 80381)         10783
+    #CSIS 571A (80428)             10956
+    #GUID 558A (80429)             10957
+    import csv
+
+    course_id = "10957"
+    course_label = "GUID 558A 80429"
+
+    (connection,cursor) = db()
+    sections = "10945 10746 10783 10956 10957".split(" ")
+
+    for course_id in sections:
+        if 0:
+            for course_id in sections:
+                q = """SELECT c.code, u.sortablename, c.id, e.user_id, 
+                        c.canvasid FROM courses AS c JOIN enrollment AS e ON e.course_id=c.id
+                        JOIN users AS u ON u.id=e.user_id 
+                        WHERE c.canvasid=%s""" % course_id
+                cursor.execute(q)
+
+                for row in cursor:
+                    print(row)
+                    mycourses[row[2]] = ''
+            return
+
+
+        grp_sum_qry = """SELECT u.sortablename, r.timeblock, SUM(r.viewcount), u.canvasid AS user, c.canvasid  
+            FROM requests_sum1 AS r
+            JOIN courses AS c ON r.courseid=c.id
+            JOIN enrollment as e ON e.course_id=c.id
+            JOIN users AS u ON u.id=r.userid 
+            WHERE c.canvasid=%s
+            GROUP BY r.userid,c.id,r.timeblock 
+            ORDER BY u.sortablename ,r.timeblock """ % course_id
+
+        cursor.execute( grp_sum_qry )
+        with codecs.open("cache/aec_%s.csv" % course_id, "w", "utf-8") as write_file:
+            c_out = csv.writer(write_file)
+            c_out.writerow( ['name','timeblock','viewcount','timestamp','minutes'] )
+
+            rows = [list(row) for row in cursor]
+            print("Got %i records" % len(rows))
+            compressed_rows = []
+
+            last_timeblock = -1
+            last_R = []
+            current_minute = 0
+            current_name = ""
+            uptodate = 1
+            for R in rows:
+                print(" %s\t%s " % (R[0], current_name) )
+                if R[0] != current_name:
+                    if not uptodate: 
+                        last_R.append(current_minute)
+                        last_R.pop(1)
+                        last_R.pop(2)
+                        last_R.pop(2)
+                        compressed_rows.append(last_R)
+                    uptodate = 1
+                    last_timeblock = -1
+                    last_R = []
+                    current_minute = 0
+                    current_name = R[0]
+
+                if R[2] < 3: continue 
+                if R[1] != last_timeblock+1 and len(last_R):
+                    # non contiguous timeblock, save the last row and reset counters
+                    last_timeblock = R[1]
+
+                    R.append( str(dt_from_timeblock( R[1] )) )
+
+                    last_R.append(current_minute)
+                    current_minute = 15
+
+                    #last_R.pop(1)
+                    last_R.pop(3)
+                    last_R.pop(3)
+
+                    compressed_rows.append(last_R)    # makes a copy of list. dunno if thats necessary
+                    #print(last_R)
+
+                    last_R = R
+                    uptodate = 1
+                else:
+                    # contiguous or first timeblock
+                    current_minute += 15
+                    last_timeblock = R[1]
+                    if len(last_R):
+                        last_R[2] = int(last_R[2]) + int(R[2])  # add the views 
+                        # its contiguous, so we already have a last_R we're building on
+                    else:
+                        last_R = R[:]   # clone it. 
+                    uptodate = 0
+            if not uptodate: 
+                last_R.append(current_minute)
+                last_R.pop(1)
+                last_R.pop(2)
+                last_R.pop(2)
+                compressed_rows.append(last_R)
+
+
+            for R in compressed_rows:
+                c_out.writerow(R)
+
+            # Build up a report for everyone
+            outfile = codecs.open('cache/positive_attendance_%s.csv' % course_id , 'w', 'utf-8')
+            pa_out = csv.writer(outfile)
+            pa_out.writerow( ['name','date','viewcount','minutes'] )
+            
+            people = funcy.group_by(lambda x: x[0], compressed_rows)
+            for P in people:
+                if P in ['Ally','Burgman, Lorraine','Horta, Gilbert','Mendez, Frank','Student, Test']:
+                    continue
+                outrows = [ [P,''] ]
+                try:
+                    
+                    #print(P)
+                    #print(people[P])
+                    for x in people[P][1:]:
+                        outrows.append( [ '', x[3], x[2],x[4] ] ) 
+                    mins = list(map( lambda x: x[4], people[P][1:]))
+                    print(mins)
+                    total_min = functools.reduce( lambda x, y: int(x)+int(y), mins)
+                    outrows.append( ['Total minutes', total_min] )
+                    print("Total minutes is %i." % total_min)
+                    hours = total_min / 60.0
+                    outrows.append( ['Total hours', hours] )
+                    print("Total hours is %0.1f." % hours)
+                    outrows.append( [] )
+                    outrows.append( [] )
+
+                    for x in outrows:
+                        print(x)
+                        pa_out.writerow(x)
+                except Exception as e:
+                    print("Some sort of error: %s" % str(e))
+
+
+
+    connection.close()
+    print("Wrote output file to: %s" % "cache/aec_%s.csv" % course_label)
+
+
+
+
+
+    """
+    HELPERS
+
+    Whos in a course?
+    SELECT * FROM enrollment as e JOIN courses AS c ON e.course_id=c.id WHERE c.canvasid=10957    ; AND c.worflow=active
+
+    """
+
+
+##########
+##########
+##########    JUST LOADING FROM FILE
+##########
+########## 
+
+
+######################
+
+# Return the most up do date version of the given file. Useful for 'dimensions'.
+def most_recent_file_of( target ):
+
+    def finder(st):
+        return re.search(target,st)
+
+    all = os.listdir(local_data_folder)
+    all.sort(key=lambda x: os.stat(os.path.join(local_data_folder,x)).st_mtime)
+    all.reverse()
+    all = list(funcy.filter( finder, all ))
+
+    #print("file list is: " + str(all))
+    if not all:
+        return ''
+    return all[0]
+
+# Given a table schema, parse log file, return a list of dicts. Optionally remove some columns.
+def parse_file_with(  file, format, with_gid=0  ):
+    if not file: return []
+    all_users = []
+    for line in gzip.open(local_data_folder + file,'r'):
+        line = line.strip()
+        line_dict = dict(list(zip(format, line.decode('utf-8').split("\t"))))
+        if with_gid: line_dict['globalid'] = line_dict['globalid'].rstrip()
+
+        remove = []
+        for k,v in line_dict.items():
+            if v == '\\N' or v == b'\\N': remove.append(k)
+        for k in remove: line_dict.pop(k, None)
+        all_users.append(line_dict)
+    return all_users
+
+
+# I keep my own cache. 
+# I return a list of the read lines if the log dates in the file are within dates (top of this file), or FALSE
+def is_requestfile_interesting(fname):
+    global thefiles, thefiles_dat
+    #begin_month = ['2020-01','2020-02','2020-03','2020-04','2020-05','2020-06','2020-07']
+    #begin_month = ['2020-09','2020-10','2020-08']
+    begin_month = ['2021-02','2021-03']
+
+    #AE 600 (80040; 80045; 80047)  10945
+    #AE 602 (80048; 80049; 80050)  10746
+    #AE 636 (80332; 80381)         10783
+    #CSIS 571A (80428)             10956
+    #GUID 558A (80429)             10957
+
+    # The AEC sections of interest.
+    sections = '10945 10746 1783 10956 10957'.split(' ')
+    # Just once, to get the people
+    #[ course_enrollment(x) for x in sections ]
+    
+    
+    
+
+
+    first = {}
+    lines = False
+    if fname in thefiles_dat:
+        f_date = parse(thefiles_dat[fname])
+        #print("\t\t+ %s" % str(f_date))
+        first = {'year':str(f_date.year), 'month':"%i-%02i" % (f_date.year,f_date.month) }
+        #print("\t\t- %s" % str(first))
+        #print("\t\t* From: %s (%s)" % (first['month'], thefiles_dat[fname]) )
+        print("+ %s" % first['month'])
+    else:
+        filei = 0
+        #thefiles.write(fname + ',')
+
+        g_file = gzip.open(local_data_folder+fname,'r')
+        lines = g_file.readlines()
+
+        last = 0
+        i = 0
+        j = -1
+        while not last:
+            last = requests_line(lines[i].decode('utf-8'))
+            i += 1
+        first = 0
+        while not first:
+            first = requests_line(lines[j].decode('utf-8'))
+            j -= 1
+
+        print("- %s" % first['month'])
+
+        thefiles.write(fname + "," + str(first['date']) + ',' + str(last['date']) + '\n')
+        thefiles.flush()
+
+    # TODO more robust here
+    if first['month'] in begin_month:
+        print("++ Using it.")
+        if lines: return lines
+        return gzip.open(local_data_folder+fname,'r').readlines()
+    return False
+
+
+# This is it:
+# e670d58a-25cb-4666-9675-8438615a5a4a	2019-05-18 13:01:03.558	2019	2019-05	2019-05-18	-256911301467799527	94250000000003187	94250000000000001	94250000000000001	\N	\N	\N	\N	/api/v1/courses/3187/assignments?page=4573781&per_page=30	Java/1.8.0_191	GET	35.173.111.106	81639	assignments_api	index	Course	3187	\N	6dad4c59c75a3492b830fb3b1136e1bc	-553092862543029181	200	HTTP/1.1	170000000000376
+
+
+# TODO - investigate pearson, developer key: 170000000000376 and their ridiculous amounts of hits.
+# and all these others:  https://ilearn.gavilan.edu/accounts/1/developer_keys
+
+#from dateutil.relativedelta import relativedelta
+#diff = relativedelta(start, ends)
+
+secs_in_a_24hr_block = 60 * 60 * 24    #  24 HOUR BLOCK
+secs_in_a_4hr_block = 60 * 60 * 4    #  4 HOUR BLOCK
+secs_in_a_block = 60 * 15    #  15 MINUTE BLOCK
+start_of_time = '2020-08-23 00:00:00'
+
+# Why is this 7 minutes off?
+# start = dt.strptime(start_of_time, '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.timezone('US/Pacific'))
+
+pst = pytz.timezone('US/Pacific')
+start = pst.localize(dt.strptime(start_of_time, '%Y-%m-%d %H:%M:%S'))
+start_seconds = start.timestamp()
+
+# epoch slot: imagine time starts on Jan 1 of 20xx, and is counted off in xxxxxxxxxxxxx4 hour slots, so
+# xxxxxx time 0 = jan 1, 12am - 3:59am,   time 1 = 4am - 8am,... and so on. 
+# xxxxxx So there's 6 of these per day. 
+#
+# In this version I'm doing 15 minute slots, 4 per hour, 96 per day. 
+#
+# Return a 'timeblock'. An integer number of 15 minute blocks from my epoch. Expects a datetime object in PST timezone.
+def timeblock_from_dt(dt_obj):
+    global start, start_seconds
+    secs = dt_obj.timestamp() - start_seconds
+    return int(  secs / secs_in_a_block )
+
+# Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+def dt_from_timeblock(tb):
+    delta = timedelta(seconds=tb*secs_in_a_block)
+    return start + delta
+
+####
+# Twenty Four hour timeblocks
+def timeblock_24hr_from_dt(dt_obj):
+    global start, start_seconds
+    secs = dt_obj.timestamp() - start_seconds
+    return int(  secs / secs_in_a_24hr_block )
+
+# Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+def dt_from_24hr_timeblock(tb):
+    delta = timedelta(seconds=tb*secs_in_a_24hr_block)
+    return start + delta
+
+
+
+####
+# Four hour timeblocks
+def timeblock_4hr_from_dt(dt_obj):
+    global start, start_seconds
+    secs = dt_obj.timestamp() - start_seconds
+    return int(  secs / secs_in_a_4hr_block )
+
+# Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+def dt_from_4hr_timeblock(tb):
+    delta = timedelta(seconds=tb*secs_in_a_4hr_block)
+    return start + delta
+
+
+# I make the line into a dict, erase keys with no data, make a DT field called date, make a time_block (int) field.
+def requests_line(line,i=0):
+    L = line  # strip?
+    if type(L) == type(b'abc'): L = line.decode('utf-8')
+    line_parts = L.split("\t")
+    for pattern in unwanted_req_paths:
+        if pattern in L: 
+            return 0
+    d = dict(list(zip(requests_format, L.split("\t"))))
+    remove = []
+    for k,v in d.items():
+        if v == '\\N' or v == b'\\N': remove.append(k)
+    for k in remove: d.pop(k, None)
+    d['date'] = dt.strptime( d['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
+    d['date'] = d['date'].replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
+    d['time_block'] = timeblock_from_dt(d['date'])
+    #if i % 1000 == 1: print(d)
+    return d
+
+import time
+
+# Take all the requests.gz files and index them in some useful fashion.
+# Bulk insert of requests logs. Too much data to be useful. 
+def requests_file(fname_list):
+    global mycourses
+    samples = codecs.open('cache/request_samples.txt', 'a', 'utf-8')
+    conn,cur = db()
+
+    folderi = 0
+    filei = 0
+    last_time = time.process_time()
+
+    q = "INSERT INTO requests_sum1 (userid, courseid, timeblock, viewcount) VALUES (?,?,?,?) ON CONFLICT (userid,courseid,timeblock) DO UPDATE SET viewcount=viewcount+1" 
+    
+    for fname in fname_list:
+        #if folderi > 2: return
+        print("\n%i\t%s   \t" % (folderi, fname), end='', flush=True)
+        folderi += 1
+        filei = 0
+
+        lines = is_requestfile_interesting(fname)
+        if lines:
+            vals_cache = []
+            for L in lines:
+                thisline = requests_line(L,filei)    #TODO select if timeblock exists
+                if not thisline:
+                    continue
+                if random.random() > 0.9999:
+                    #L = str(L)
+                    if type(L) == type(b'abc'): L = L.decode('utf-8')
+                    parts = L.split('\t')
+                    if len(parts)>17:
+                        samples.write( "\t".join( [parts[13] , parts[14], parts[15], parts[16], parts[18], parts[19]]))
+                        
+                #q,v = dict_to_insert(thisline,'requests')
+                if not 'courseid' in thisline: continue
+                if not 'userid' in thisline: continue
+                
+                # Limit this database to certain courses?
+                # if thisline['courseid'] not in mycourses: continue
+
+                v = ( thisline['userid'], thisline['courseid'], thisline['time_block'], 1 )
+                vals_cache.append(  [ str(x) for x in v ] )
+                try:
+                    #cur.execute(q)
+                    if filei % 5000 == 0:
+                        conn.executemany(q, vals_cache)
+                        conn.commit()
+                        t = time.process_time()
+                        delta = t - last_time
+                        last_time = t
+                        print("\nLoop %i - committed to db in %0.1fs. " % (filei,delta), end='', flush=True)
+                        samples.flush()
+                    filei += 1
+                except Exception as e:
+                    print(thisline)
+                    print(e)
+                    print(q)
+                    print(v)
+            # do the commit on the entire file... 
+            conn.executemany(q, vals_cache)
+            conn.commit()
+            t = time.process_time()
+            delta = t - last_time
+            last_time = t
+            print("\nLoop %i - committed to db in %0.1fs. " % (filei,delta), end='', flush=True)
+
+
+# Insert or update a request line.
+def upsert_request(line, vals):
+    # "id userid courseid timeblock viewcount partcount"
+
+    # is it a view or a participation?
+    q = "INSERT INTO requests_sum1 (userid, courseid, timeblock, viewcount) VALUES ('%s','%s',%s,%s) ON CONFLICT (userid,courseid,timeblock) DO UPDATE SET viewcount=viewcount+1" % (  str(vals[0]), str(vals[1]), str(vals[2]), str(vals[3]) )
+    return q
+
+
+
+# Generic insert of a dict into a table. Keys of dict must match table columns.
+def dict_to_insert(thisline,table):  # a dict
+    vals = []
+    v_str = ''
+    first = 1
+    q = "INSERT INTO %s (" % table
+
+    for k in thisline.keys():
+        #print(k)
+        if k == 'date': continue
+        if not first: 
+            q += ","
+            v_str += ","
+        q += k
+        v_str += "?"
+        vals.append(str(thisline[k]))
+        first = 0
+    q +=  ") VALUES (" + v_str + ")"
+    return q, vals
+
+# This and the following merge functions do direct inserts without further tallying.
+# This now does tallying by timeblock. 
+def merge_requests():
+    req = []
+    i = 0
+    max = 2000
+
+    for f in os.listdir(local_data_folder):
+        if re.search(r'requests',f) and i < max: 
+            req.append(f)
+            i += 1
+    #req = ['requests-00000-afc834d1.gz',]
+    print("Checking %i request log files." % len(req))
+    requests_file(req)
+
+def merge_comm_channel():
+    setup_table('comm_channel')
+    (conn,cur) = db()
+    count = 0 
+
+    cfile = most_recent_file_of('communication_channel_dim')
+    cm = parse_file_with( cfile, communication_channel_dim_format)
+    for U in cm:
+        q,v = dict_to_insert(U,'comm_channel')
+        try:
+            cur.execute(q,v)
+            count += 1
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+    print("Processed %i comm channel entries" % count)
+
+
+def merge_pseudonym():
+    setup_table('pseudonym')
+    (conn,cur) = db()
+    count = 0 
+
+    cfile = most_recent_file_of('pseudonym_dim')
+    cm = parse_file_with( cfile, pseudonym_dim_format)
+    for U in cm:
+        q,v = dict_to_insert(U,'pseudonym')
+        try:
+            cur.execute(q,v)
+            count += 1
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+    print("Processed %i pseudonym entries" % count)
+
+
+
+
+def merge_users():
+    setup_table('users')
+    (conn,cur) = db()
+
+    user_file = most_recent_file_of('user_dim')
+    users = parse_file_with( user_file, users_format)
+    for U in users:
+        q,v = dict_to_insert(U,'users')
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+
+def merge_courses():
+    setup_table('courses')
+    (conn,cur) = db()
+
+    c_file = most_recent_file_of('course_dim')
+    courses = parse_file_with( c_file, course_format)
+    for U in courses:
+        q,v = dict_to_insert(U,'courses')
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+
+def merge_enrollment():
+    setup_table('enrollment')
+    (conn,cur) = db()
+
+    c_file = most_recent_file_of('enrollment_dim')
+    print("Using enrollments from: %s" % c_file)
+    courses = parse_file_with( c_file, enrollment_dim_format)
+    count = 0
+    for U in courses:
+        q,v = dict_to_insert(U,'enrollment')
+        count += 1
+        #if count % 1000 == 0:
+        #    print( "%i - " % count + q + " " + str(v) )
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+    print("Processed %i enrollments" % count)
+
+
+def merge_term():
+    setup_table('terms')
+    (conn,cur) = db()
+
+    c_file = most_recent_file_of('enrollment_term_dim')
+    courses = parse_file_with( c_file, term_format)
+    for U in courses:
+        q,v = dict_to_insert(U,'terms')
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+
+def merge_roles():
+    (conn,cur) = db()
+    cur.execute(setup_table('roles'))
+    conn.commit()
+
+    c_file = most_recent_file_of('role_dim')
+    courses = parse_file_with( c_file, role_format)
+    for U in courses:
+        q,v = dict_to_insert(U,'roles')
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+
+def merge_convos():
+    setup_table('conversation')
+    setup_table('conversation_message')
+
+    (conn,cur) = db()
+    c_file = most_recent_file_of('conversation_dim')
+    ccc = parse_file_with( c_file, conversation_dim_format)
+    for U in ccc:
+        q,v = dict_to_insert(U,'conversation')
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+
+    c_file = most_recent_file_of('conversation_message_dim')
+    ccc = parse_file_with( c_file, conversation_message_dim_format)
+    for U in ccc:
+        q,v = dict_to_insert(U,'conversation_message')
+        try:
+            cur.execute(q,v)
+        except Exception as e:
+            print(e)
+            print(q)
+    conn.commit()
+
+# For returning sqlite results as dicts
+def dict_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+    return d
+
+
+# TODO... approaches to all this data... list requests in order descending time, unique users, and just 
+# file stats on them...?
+
+# people's maxs, with time block window:
+# select *,count(course_canvasid),sum(count),max(time_block),min(time_block) from summary_course_user_views group by username order by min(time_block)
+
+# get the time back:  dt_from_timeblock(11296)
+
+
+# Attempt to do tallying
+def make_views_summarys():
+    connection = sqlite3.connect(sqlite_file)
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+    q1 = """select courses.id, courses.code, courses.name, courses.visible, courses.state, courses.sis from courses 
+            join terms on courses.termid=terms.id 
+            where terms.name="2021 Spring" and courses.state="available";
+            """
+
+    cursor.execute(q1)
+    sp2020_courses = cursor.fetchall()
+    #print(json.dumps(sp2020_courses,indent=2))
+
+    print("Summarizing views...     ", end='')
+    for C in sp2020_courses:
+        print("%s,  " % C['name'], end='', flush=True)
+
+        #if input('enter to go, q to quit') == 'q': break
+        q2 = """select sum(requests_sum1.viewcount) as views, requests_sum1.timeblock as block, courses.code, courses.canvasid as ccid, 
+            users.name, users.id, users.canvasid from requests_sum1
+            join users on users.id = requests_sum1.userid
+            join courses on courses.id=requests_sum1.courseid  
+            where courses.id="%s"
+            group by users.name, block """ % C['id']
+        cursor.execute(q2)
+        views = cursor.fetchall()
+        #print(json.dumps(views,indent=2))
+        for U in views:
+            q3 = """INSERT INTO summary_course_user_views ("courseid","course_canvasid", "username","userid","user_canvasid","count","time_block") VALUES (?,?,?,?,?,?,?);"""
+            vals = [C['id'], U['ccid'], U['name'], U['id'], U['canvasid'], U['views'], U['block']]
+            #print( q3 )
+            #print( vals )
+            #print('')
+            cursor.execute(q3,vals)
+        connection.commit()
+    connection.close()
+
+# original without time_blocks info.
+def make_views_summarys_v1():
+    connection = sqlite3.connect(sqlite_file)
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+    q1 = """select courses.id, courses.code, courses.name, courses.visible, courses.state, courses.sis from courses 
+            join terms on courses.termid=terms.id 
+            where terms.name="2020 Spring " and courses.state="available";
+            """
+
+    cursor.execute(q1)
+    sp2020_courses = cursor.fetchall()
+    #print(json.dumps(sp2020_courses,indent=2))
+
+    for C in sp2020_courses:
+        print("Summarizing views for " + C['name'])
+
+        #if input('enter to go, q to quit') == 'q': break
+        q2 = """select count(requests.id) as views, courses.code, courses.canvasid as ccid, users.name, users.id, users.canvasid from requests
+            join users on users.id = requests.userid
+            join courses on courses.id=requests.courseid  
+            where requests.courseid="%s" 
+            group by users.name;""" % C['id']
+        cursor.execute(q2)
+        views = cursor.fetchall()
+        #print(json.dumps(views,indent=2))
+        for U in views:
+            q3 = """INSERT INTO summary_course_user_views ("courseid","course_canvasid", "username","userid","user_canvasid","count") VALUES (?,?,?,?,?,?);"""
+            vals = [C['id'], U['ccid'], U['name'], U['id'], U['canvasid'], U['views'] ]
+            print( q3 )
+            print( vals )
+            print('')
+            cursor.execute(q3,vals)
+        connection.commit()
+    connection.close()
+
+
+# Setup my basic db stats base from scratch
+def full_reload():
+    
+    path = "cache/canvas_data/"
+    file = "data.db"
+    if exists(path + file):
+        time = date_time = dt.fromtimestamp( getmtime(path + file) )
+        newname = 'data'+ time.strftime('%Y%m%d') + ".db"
+        print("renaming old data file to %s" % newname)
+        os.rename(path+file, path + newname)
+
+    sync_non_interactive()
+
+    setup_table('requests_sum1')
+    setup_table('courses')
+    setup_table('users')
+    setup_table('roles')
+    setup_table('enrollment')
+    setup_table('terms')
+    setup_table('conversation')
+    setup_table('conversation_message')
+    setup_table('summary')
+    setup_table('index')
+    
+    
+    
+    
+
+    merge_users()
+    merge_comm_channel()
+    merge_convos()
+    merge_courses()
+    merge_pseudonym()
+    merge_enrollment()
+    merge_term()
+    merge_roles()
+
+    #merge_requests()
+
+    #make_views_summarys()
+
+def guess_dept(t):
+    #print(t)
+    method = 1  # crosslisted courses get their own dept
+    method = 2  # xlisted takes dept first listed
+
+    if method==1:
+        p = "^([A-Z/]+)\d+"
+        m = re.search(p, t['code'])
+        if m:
+            return m.group(1)
+        return '?'
+    if method==2:
+        p = "^([A-Z]+)[\d/]+"
+        m = re.search(p, t['code'])
+        if m:
+            return m.group(1)
+        return '?'
+
+
+# Main view of all class / all user overview...
+def dept_with_studentviews(dept="", sem=''):
+    if not sem:
+        sem = input("which semester? (ex: 2020 Fall) ")
+        
+    connection = sqlite3.connect(sqlite_file)
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+    q1 = """select courses.id, courses.canvasid, courses.code, courses.name, courses.visible, courses.state, courses.sis from courses 
+            join terms on courses.termid=terms.id 
+            where terms.name="%s" and courses.state="available" """ % sem
+    if dept:
+        q1 += " AND courses.code LIKE '%" + dept + "%';"
+    
+    print(q1)
+    cursor.execute(q1)
+    courses = cursor.fetchall()
+    return courses
+    #print(json.dumps(sp2020_courses,indent=2))
+
+    # version 1 of this got as high as 208 MB. Removed names, other unused columns.
+
+    qry = "select suv.user_canvasid, suv.courseid, suv.count, suv.time_block, courses.code from summary_course_user_views as suv join courses on courses.id=suv.courseid where suv.courseid=%s"
+
+    if dept == 'all':
+        views_records = list( funcy.flatten( [ cursor.execute(qry% x['id']).fetchall() for x in sp2020_courses ] ) )
+        by_course = funcy.group_by( lambda x: x['code'], views_records)
+        for k,v in by_course.items():
+            by_course[k] = funcy.group_by( lambda x: x['user_canvasid'], v)
+        return by_course
+
+
+    def f(x):
+        return x['code']
+    this_dept = filter( lambda x: guess_dept(x)==dept, sp2020_courses ) 
+
+
+    views_records = list( funcy.flatten( [ cursor.execute(qry% x['id']).fetchall() for x in this_dept ] ) )
+
+    return funcy.group_by( lambda x: x['courseid'], views_records)
+    return "Couldn't find that department: %s" % dept
+
+
+def get_courses_in_term_local(term="172"):
+    q = """SELECT c.code, c.name, c.state, c.canvasid, c.id  FROM courses AS c JOIN terms AS t ON c.termid=t.id WHERE t.canvasid==%s""" % term
+    (connection,cursor) = db()
+    cursor.execute(q)
+    allrows = cursor.fetchall()
+    return allrows
+
+# get student count
+def course_student_stats(canvasid):
+    q = """SELECT u.name FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.canvasid=%s
+AND e.type="StudentEnrollment"
+AND e.workflow="active" """ % (canvasid)
+    (connection,cursor) = db()
+    cursor.execute(q)
+    allrows = cursor.fetchall()
+    a = [ len(allrows), ]
+    b = []
+    for x in allrows: b.append(x[0])
+    return [a,b]
+    return [x[0] for x in allrows]
+
+
+# get teacher name from local db
+def course_quick_stats(canvasid):
+    q = """SELECT c.id AS courseid, c.code, tt.name, c.state, COUNT(u.id) AS student_count  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+JOIN (
+SELECT c.id AS courseid, u.id AS userid, c.code, u.name  FROM courses AS c
+	JOIN enrollment AS e ON e.course_id=c.id
+	JOIN users AS u ON u.id=e.user_id
+	WHERE c.canvasid=%s
+	AND e."type"="TeacherEnrollment"
+) AS tt ON c.id=tt.courseid
+WHERE c.canvasid=%s
+AND e."type"="StudentEnrollment"
+GROUP BY c.code
+ORDER BY c.code""" % (canvasid,canvasid)
+    (connection,cursor) = db()
+    cursor.execute(q)
+    allrows = cursor.fetchall()
+    return allrows
+
+
+# What a student has taken / teacher has taught
+def user_enrolled_in(userid):
+    q = """SELECT u.canvasid as user_id, c.canvasid AS course_id, u.name, u.sortablename, c.code, c.name AS course_name, c.sis, t.name, p.current_login_at, p.current_login_ip, p.sis_user_id FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON e.user_id=u.id
+JOIN pseudonym AS p ON p.user_id=u.id
+JOIN terms AS t ON c.termid=t.id
+WHERE u.canvasid=%s ORDER BY t.name ASC""" % userid
+#AND e.workflow="active"
+#GROUP BY u.canvasid"""               ## AND e."type"="StudentEnrollment"
+    (connection,cursor) = db()
+    cursor.execute(q)
+    return cursor.fetchall()
+
+
+# All students in this semester ...
+def users_this_semester_db(sem=''):
+    if not sem:
+        sem = input("which semester? (ex: 202150) ")
+        
+    q = """SELECT u.canvasid, u.name, u.sortablename, COUNT(e.id) AS num FROM enrollment AS e
+JOIN users AS u ON e.user_id=u.id
+JOIN courses AS c ON e.course_id=c.id
+WHERE c.sis LIKE "%s-%%" 
+AND e.workflow="active"
+GROUP BY u.canvasid"""  % sem             ## AND e."type"="StudentEnrollment"
+    (connection,cursor) = db()
+    cursor.execute(q)
+    all_u = set()
+    for u in cursor:
+        print(u) 
+        all_u.add(str(u[0]))
+    print("%i users this semester." % len(all_u))
+    return all_u
+    
+
+# Everyone whose first semester is .....
+def users_new_this_semester(sem=''):
+    if not sem:
+        sem = input("which semester? (ex: 202150) ")
+    users_to_enroll = set()
+
+    q = """SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e
+JOIN users AS u ON e.user_id=u.id
+JOIN courses AS c ON e.course_id=c.id
+WHERE c.sis LIKE "%s-%%" 
+AND e.workflow="active"
+AND e."type"="StudentEnrollment"
+AND u.canvasid NOT IN (
+	SELECT u.canvasid FROM enrollment AS e
+	JOIN users AS u ON e.user_id=u.id
+	JOIN courses AS c ON e.course_id=c.id
+	WHERE c.sis NOT LIKE "%s-%%" 
+	AND e.workflow="active"
+	AND e."type"="StudentEnrollment"
+	GROUP BY u.canvasid
+)
+GROUP BY u.canvasid
+ORDER BY num DESC, u.sortablename""" % (sem,sem)
+
+
+    (connection,cursor) = db()
+    cursor.execute(q)
+    #s = cursor.fetchall()
+    #if s:
+    for u in cursor:
+        users_to_enroll.add(str(u[0]))
+        #print(s)
+    print("%i new users this semester." % len(users_to_enroll))
+    return users_to_enroll
+
+
+# All student users in STEM - from local db
+def user_in_stem():
+    enrolled = set()
+    q = """SELECT c.id, c.canvasid, c.name, c.code, c.start, c.visible, c.state, 
+u.id AS userid, u.canvasid AS user_c_id, u.sortablename FROM courses AS c 
+JOIN enrollment AS e ON c.id=e.course_id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.canvasid="11015" AND e."type"="StudentEnrollment"
+AND e."workflow"='active'
+ORDER BY c.code, u.sortablename """
+    (connection,cursor) = db()
+    cursor.execute(q)
+    results = cursor.fetchall()
+    for u in results:
+        enrolled.add( (u[9], u[8] ) )
+    return enrolled
+
+
+
+# Get all the classes in one dept
+def dept_classes(dept,sem=''):
+    if not sem:
+        sem = input("which semester? (ex: 202150) ")
+        
+
+    q = """SELECT c.id, c.canvasid, c.name, c.code, c.start, c.visible, c.state, 
+u.id AS userid, u.canvasid AS user_c_id, u.sortablename FROM courses AS c 
+JOIN enrollment AS e ON c.id=e.course_id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.name LIKE """ + '"' + dept + """%" AND c.sis LIKE """ + '"' + sem + """%" AND e."type"="StudentEnrollment"
+AND e."workflow"='active'
+ORDER BY c.code, u.sortablename """
+    
+    users = set()
+    (connection,cursor) = db()
+
+    cursor.execute(q)
+    results = cursor.fetchall()
+    for u in results:
+        users.add(  (u[9], u[8]) )
+    return users
+
+
+
+# TODO
+#
+# depts -> courses -> count students... 1 structure...     display as 1 grid?  treeview?
+# afterwards: views by student / by week, row of grids per class...
+
+def depts_with_classcounts(sem=''):
+    if not sem:
+        sem = input("which semester? (ex: 202150) ")
+
+    # This is messier cause i don't have depts in database 
+    # should I add that? Or just use python. TODO
+
+    q = """select users.canvasid, courses.code, courses.id, users.name, roles.name as role, 
+        enrollment.workflow as user_status, courses.state as course_state  
+        from courses join terms on courses.termid = terms.id
+        join enrollment on enrollment.course_id=courses.id 
+        join users on enrollment.user_id = users.id 
+        join roles on roles.id=enrollment.role
+        where terms.sis='%s' and enrollment.workflow='active'
+        order by courses.code"""  %   sem
+        
+    connection = sqlite3.connect(sqlite_file)
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+    cursor.execute(q)
+    results = cursor.fetchall()
+    connection.close()
+
+    def f(x):
+        return x['code']
+    by_dept_ = funcy.group_by( guess_dept, results )
+    by_dept = {}
+
+    def name_with_count(name,li):
+        count = len(li)
+        return (name,count,li[0]['id'])
+
+    for d,li in by_dept_.items():
+        classes_in_dept = funcy.group_by( f, li )
+        #print(classes_in_dept)
+        by_dept[d] = [ name_with_count(c,v) for c,v in classes_in_dept.items() ]
+
+
+
+    return by_dept
+
+def arrange_data_for_web(dept='', sem=''):
+    if not sem:
+        sem = input("which semester? (ex: 202150) ")
+
+    # I want:
+    # - structure of dicts, 1 file per class
+    # - class -> teacher [ teacher1_cid: {name:nnn, week1:hits,week2:hits...],   
+    #            student [stuent1_cid: {name:nnn, week1:hits,week2:hits...],
+    #
+
+    q = "select * from courses join terms on courses.termid = terms.id where terms.sis='%s' and courses.state='claimed'" % sem
+
+    # three... seconds:
+    
+    q2 = """select courses.code, users.name, roles.name as role from courses join terms on courses.termid = terms.id
+        join enrollment on enrollment.course_id=courses.id 
+        join users on enrollment.user_id = users.id 
+        join roles on roles.id=enrollment.role
+        where terms.sis='%s' and courses.state='claimed'
+        order by code, role """ % sem
+        
+
+    # courses with users - need it as hierarchy - by course, or by user... or 1 user... (with logs...?)
+    
+    q3 = """select users.canvasid, courses.code, users.name, roles.name as role, 
+        enrollment.workflow as user_status, courses.state as course_state 
+        from courses 
+        join terms on courses.termid = terms.id
+        join enrollment on enrollment.course_id=courses.id 
+        join users on enrollment.user_id = users.id 
+        join roles on roles.id=enrollment.role
+        where terms.sis='%s' 
+        order by courses.code""" % sem
+
+
+
+
+
+    connection = sqlite3.connect(sqlite_file)
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+
+    cursor.execute(q3)
+
+    # fetch all or one we'll go for all.
+    results = cursor.fetchall()
+    #print(results)
+    connection.close()
+
+    def f(x):
+        return x['code']
+    by_dept_ = funcy.group_by( guess_dept, results )
+    by_dept = {}
+
+    for d,li in by_dept_.items():
+        by_dept[d] = funcy.group_by( f, li )
+    #by_course = funcy.group_by( f, results )
+    #return by_course
+    #print(json.dumps(by_dept,indent=2))
+    if not dept:
+        return by_dept  # list(by_dept.keys())
+
+    if dept in by_dept:
+        return by_dept[d]
+    return "Error"
+
+#
+#
+#
+#
+#
+#
+#
+# This csv loading code isn't really necessary cause i get it all from the canvas_data files.
+# Except that the enrollments don't seem to be there so this works.
+#
+# Saved to mine in the future.....
+
+# Get enrollments. (Best to freshly run pipelines/get_rosters) and put them into DB
+def build_tables(headers,name):
+    first = 1
+    query = "CREATE TABLE IF NOT EXISTS %s (\n" % name
+    for L in headers:
+        if not first: 
+            query += ",\n"
+        first = 0
+        query += "\t%s %s" % (L,"text")
+    return query + "\n);"
+
+def load_tables(table,headers,row,verbose=0):
+    (conn,cur) = db()
+    vals = []
+    v_str = ''
+    i = 0
+    q = "INSERT INTO %s (" % table
+    for L in headers:
+        if i: 
+            q += ","
+            v_str += ","
+        q += L
+        v_str += "?"
+        vals.append(str(row[i]))
+        i += 1
+    q +=  ") VALUES (" + v_str + ")"
+    try:
+        cur.execute(q,vals)
+        if verbose: 
+            print(q)
+            print(vals)
+    except Exception as e:
+        print(e)
+        print(q)
+    conn.commit()
+
+def semester_enrollments(verbose=0):
+    def qstrip(txt): return txt.strip('"')
+
+    epath = "cache/rosters/enrollments-2020-08-02-19-49-36.csv"
+    #cpath = "cache/rosters/spring2020/courses.2020-02-25T15-57.csv"
+    #upath = "cache/rosters/spring2020/users.2020-02-25T15-57.csv"
+
+    enrollments =  [ list( map( qstrip, L.strip().split(','))) for L in open(epath,'r').readlines() ]
+    #classes = [ list( map( qstrip, L.strip().split(','))) for L in open(cpath,'r').readlines() ]
+    #users = [ list( map( qstrip, L.strip().split(','))) for L in open(upath,'r').readlines() ]
+
+    e = build_tables(enrollments[0],"enrollments")
+    #c = build_tables(classes[0],"classes")
+    #u = build_tables(users[0],"users")
+
+    if verbose:
+        #for x in [e,c,u]: print(x)
+        print(enrollments[0])
+        print(enrollments[5])
+        #print(classes[0])
+        #print(classes[5])
+        #print(users[0])
+        #print(users[5])
+
+    (conn,cur) = db()
+    q = e
+    try:
+        cur.execute(q)
+        if verbose: print(q)
+    except Exception as ex:
+        print(ex)
+        print(q)
+    conn.commit()
+
+    headers = enrollments[0]
+    rows = enrollments[1:]
+    # Probably don't want to commit on every row?
+    for row in rows:
+        load_tables("enrollments",headers,row,verbose)
+
+# Show this as a big grid? D3? CSV?
+
+# Ultimately we need session calcs too. When we get 15 minute chunks, then just add them up....
+
+# Overview of student hits in a course. Return a (pandas??) table student/timeblock/hits  6 * 7 * 7 items per student. 
+
+    """e_qry = "CREATE TABLE IF NOT EXISTS enrollments (
+        id integer PRIMARY KEY,
+        name text NOT NULL,
+        begin_date text,
+        end_date text
+    );"""
+
+"""    
+
+['CREATE INDEX "idx_req_userid" ON "requests" ("id","courseid","userid" );',
+ 'CREATE INDEX "idx_users_id" ON "users" ("id","canvasid",  );',
+ 'CREATE INDEX "idx_term_id" ON "terms" ("id","canvasid" );',
+ 'CREATE INDEX "idx_enrollment" ON "enrollment" ("cid","course_id","user_id" );',
+ 'CREATE INDEX "idx_courses" ON "courses" ("id","canvasid","termid","code","name" );' ]
+
+
+took 6 seconds
+
+
+select * from users where name = "Peter Howell"
+
+select * from users join requests on users.id = requests.userid where name = "Peter Howell"
+20k rows in 1.014 seconds!!   with index above
+
+without: killed it after 120 seconds
+
+select timestamp, url, useragent, httpmethod, remoteip, controller from users join requests on users.id = requests.userid where name = "Peter Howell" order by requests.timestamp
+
+
+
+select courses.name, courses.code, terms.name, requests.url from courses 
+join terms on courses.termid = terms.id
+join requests on courses.id = requests.courseid
+where terms.name='2020 Spring '   and courses.code='ACCT20 SP20 40039'
+order by courses.code
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+def more_unused_xreferencing():
+    """continue
+
+        for line in lines:
+            r = requests_line(line.decode('utf-8'),filei)
+            if filei < 5: 
+                print(r)
+            else:
+                break
+            filei += 1
+
+
+    by_date_course = defaultdict( lambda: defaultdict(int) )
+    by_date_user   = defaultdict( lambda: defaultdict(int) )
+    df_list = []
+    df_list_crs = []
+    users = defaultdict( lambda: defaultdict(int) )
+    #by_user = {}
+    #by_course = {}
+    i = 0
+
+    limit = 300
+
+            #print(r)
+            date = dt.strptime( r['timestamp'], "%Y-%m-%d %H:%M:%S.%f" )
+            if r['userid'] in users:
+                users[r['userid']]['freq'] += 1
+                if users[r['userid']]['lastseen'] < date:
+                    users[r['userid']]['lastseen'] = date
+            else:
+                users[r['userid']] = {"id":r['userid'], "lastseen":date, "freq":1}
+            by_date_course[ r['day'] ][ r['courseid'] ] += 1
+            by_date_user[ r['day'] ][ r['userid'] ] += 1
+            #if r['userid'] in by_user: by_user[r['userid']] += 1
+            #else: by_user[r['userid']] = 1
+            #if r['courseid'] in by_course: by_course[r['courseid']] += 1
+            #else: by_course[r['courseid']] = 1
+        #mylog.write("by_user = " + str(by_user))
+        df_list.append(pd.DataFrame(data=by_date_user))
+        df_list_crs.append(pd.DataFrame(data=by_date_course))
+        i += 1
+        if i > limit: break
+    #mylog.write("by_date_course = ")
+    result = pd.concat(df_list, axis=1,join='outer')
+    result_crs = pd.concat(df_list_crs, axis=1,join='outer')
+    #print result_crs
+    mylog.write(result.to_csv())
+    # get users
+    usersf = user_role_and_online()
+    merged = pd.merge(result,usersf,left_index=True,right_on='id', how='left')
+    #dropkeys = "rootactid tz created vis school position gender locale public bd cc state".split(" ")
+    #merged.drop(dropkeys, inplace=True, axis=1)
+    mglog = open(local_data_folder+'userlogs.csv','w')
+    mglog.write(merged.to_csv())
+
+    # get courses
+    courses = courses_file()
+    merged2 = pd.merge(result_crs,courses,left_index=True,right_on='id', how='left')
+    dropkeys = "rootactid wikiid".split(" ")
+    merged2.drop(dropkeys, inplace=True, axis=1)
+    mglogc = open(local_data_folder + 'courselogs.csv','w')
+    mglogc.write(merged2.to_csv())
+
+    # a users / freq / lastseen file
+    ufl = open(local_data_folder + "user_freq.json","w")
+    today = datetime.datetime.today()
+    for U in list(users.keys()):
+        date = users[U]['lastseen']
+        users[U]['lastseen'] = date.strftime("%Y-%m-%d")
+        diff = today - date
+        users[U]['daysago'] = str(diff.days)
+        users[U]['hoursago'] = str(int(diff.total_seconds()/3600))
+    us_frame = pd.DataFrame.from_dict(users,orient='index')
+    us_with_names = pd.merge(us_frame,usersf,left_index=True,right_on='id', how='left')
+    #dropkeys = "id id_x id_y globalid rootactid tz created vis school position gender locale public bd cc state".split(" ")
+    #us_with_names.drop(dropkeys, inplace=True, axis=1)
+    print(us_with_names)
+    ufl.write( json.dumps(users, indent=4) )
+    ufl.close()
+    mglogd = open('canvas_data/user_freq.csv','w')
+    mglogd.write(us_with_names.to_csv())
+    """
+
+    """    -- projects table
+    CREATE TABLE IF NOT EXISTS projects (
+        id integer PRIMARY KEY,
+        name text NOT NULL,
+        begin_date text,
+        end_date text
+    );
+    """
+    pass
+
+def user_role_and_online():
+    # cross list users, classes enrolled, and their roles
+    global role_table, term_courses
+
+    role_table = enrollment_file()
+    user_table = users_file()
+    user_table = user_table[ user_table['name']!="Test Student" ]
+    term_table = term_file()
+    current = term_table[lambda d: d.course_section=='2020 Spring']            # current semester from canvas
+    term_id = current['id'].values[0]
+    course_table = courses_file()                                              # from canvas
+    schedule = current_schedule()                                              # from banner... 
+
+    term_courses = course_table[lambda d: d.termid==term_id]                   # courses this semester ... now add a crn column
+    term_courses['crn'] = term_courses['code'].map( lambda x: get_crn_from_name(x) )
+                                                                               # add is_online flag (for courses listed in schedule as online-only)
+    term_courses['is_online'] = term_courses['crn'].map( lambda x: course_is_online( x ) )  # kinda redundant
+    ban_can = term_courses.merge(schedule,on='crn',how='left')                 #join the schedule from banner to the courses from canvas
+
+    role_table = role_table.where(lambda x: x.workflow=='active')
+
+    # this join limits to current semester if 'inner', or all semesters if 'left'
+    courses_and_enrol = role_table.merge(ban_can,left_on='course_id',right_on='id', how='left')
+
+    user_table = user_table.drop(columns="rootactid tz created vis school position gender locale public bd cc state".split(" "))
+    c_e_user = courses_and_enrol.merge(user_table,left_on='user_id',right_on='id',how='left')
+
+
+    prop_online = pd.DataFrame(c_e_user.groupby(['user_id'])['is_online'].aggregate(summarize_proportion_online_classes).rename('proportion_online'))
+    num_trm_crs = pd.DataFrame(c_e_user.groupby(['user_id'])['is_online'].aggregate(summarize_num_term_classes).rename('num_term_crs'))
+    stu_tch_rol = pd.DataFrame(c_e_user.groupby(['user_id'])['type'].aggregate(summarize_student_teacher_role).rename('main_role'))
+    user_table = user_table.merge(prop_online,left_on='id',right_index=True)
+    user_table = user_table.merge(num_trm_crs,left_on='id',right_index=True)
+    user_table = user_table.merge(stu_tch_rol,left_on='id',right_index=True)
+
+    # remove name-less entries
+    user_table = user_table.where(lambda x: (x.canvasid!='') ) # math.isnan(x.canvasid))
+
+    return user_table
+
+#print user_table.query('proportion_online=="online-only"')
+    #print user_table.query('main_role=="teacher"')
+    #user_table.to_csv('canvas_data/users_online.csv')
+
+
+
+
+
+
+def comm_channel_file():
+    """all = os.listdir(local_data_folder)
+    all.sort(key=lambda x: os.stat(os.path.join(local_data_folder,x)).st_mtime)
+    all.reverse()
+    #print "sorted file list:"
+    #print all
+    for F in all:
+        if re.search('communication_channel_dim',F):
+            user_file = F
+            break
+    print("most recent comm channel file is " + user_file)"""
+
+
+    user_file = most_recent_file_of('communication_channel_dim')
+
+    all_commchannels = []
+    for line in gzip.open(local_data_folder+user_file,'r'):
+        line_dict = dict(list(zip(communication_channel_dim_format, line.split("\t"))))
+        line_dict['globalid'] = line_dict['globalid'].rstrip()
+        all_commchannels.append(line_dict)
+    df = pd.DataFrame(all_commchannels)
+    return df
+
+def pseudonym_file():
+    all = os.listdir(local_data_folder)
+    all.sort(key=lambda x: os.stat(os.path.join(local_data_folder,x)).st_mtime)
+    all.reverse()
+    #print "sorted file list:"
+    #print all
+    for F in all:
+        if re.search('pseudonym_dim',F):
+            p_file = F
+            break
+    print("most recent pseudonym file is " + p_file)
+    all_users = []
+    for line in gzip.open(local_data_folder + p_file,'r'):
+        line_dict = dict(list(zip(pseudonym_dim_format, line.split("\t"))))
+        line_dict['authentication_provider_id'] = line_dict['authentication_provider_id'].rstrip()
+        all_users.append(line_dict)
+    df = pd.DataFrame(all_users)
+    return df
+
+def users_p_file():
+    uf = users_file()
+    pf = pseudonym_file()
+    #print pf
+    upf = uf.merge(pf,left_on='id',right_on='user_id',how='left')
+    return upf
+
+    """
+    def com_channel_dim():
+        all = os.listdir(local_data_folder)
+        all.sort(key=lambda x: os.stat(os.path.join(local_data_folder,x)).st_mtime)
+        all.reverse()
+        #print "sorted file list:"
+        #print all
+        for F in all:
+            if re.search('communication_channel_dim',F):
+                cc_file = F
+                break
+        print("most recent communication channel file is " + cc_file)
+        cc_users = []
+        for line in gzip.open(local_data_folder + cc_file,'r'):
+            line_dict = dict(list(zip(cc_format, line.split("\t"))))
+            #line_dict['globalid'] = line_dict['globalid'].rstrip()
+            cc_users.append(line_dict)
+        df = pd.DataFrame(cc_users)
+        return df
+    """
+
+
+    """grp_sum_qry = ""SELECT u.sortablename, r.timeblock, SUM(r.viewcount), u.canvasid AS user, c.canvasid AS course 
+            FROM requests_sum1 AS r 
+            JOIN courses AS c ON e.course_id=c.id 
+            JOIN enrollment as e ON r.courseid=c.id
+            JOIN users AS u ON u.id=e.user_id 
+            WHERE c.canvasid=%s AND e."type"="StudentEnrollment" 
+            GROUP BY u.id,c.id,r.timeblock
+            ORDER BY u.sortablename DESC, r.timeblock"" % course_id
+
+    q = ""SELECT u.sortablename, r.timeblock, r.viewcount, u.canvasid AS user, c.canvasid AS course 
+            FROM requests_sum1 AS r 
+            JOIN courses AS c ON e.course_id=c.id 
+            JOIN enrollment as e ON r.courseid=c.id
+            JOIN users AS u ON u.id=e.user_id 
+            WHERE c.canvasid=%s AND e."type"="StudentEnrollment" AND u.canvasid=810
+            ORDER BY u.sortablename DESC, r.timeblock"" % course_id
+    
+    
+    q = ""SELECT u.sortablename, r.timeblock, r.viewcount, u.canvasid AS user, c.canvasid AS course FROM enrollment as e JOIN courses AS c ON e.course_id=c.id 
+JOIN requests_sum1 AS r ON r.courseid=c.id
+JOIN users AS u ON u.id=e.user_id 
+WHERE c.canvasid=%s AND e."type"="StudentEnrollment"
+ORDER BY u.sortablename, r.timeblock"" % course_id"""
+
+def abcd():
+    setup_table('index')
+
+
+def crns_to_teachers():
+    semester = '202070'
+    (connection,cursor) = db()
+    emails = set()
+    crns = codecs.open('cache/eval_teachers_2020fa.txt','r').readlines()
+    q = """SELECT c.id, c.canvasid AS course_cid, c.name, c.code, u.name, u.sortablename, u.canvasid AS user_cid, c.sis, h.address  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+JOIN comm_channel AS h ON u.id=h.user_id 
+WHERE h."type"="email" 
+AND c.sis = "%s-%s"
+AND NOT c.state="deleted"
+AND e."type"="TeacherEnrollment"
+GROUP BY h.address;"""
+    for c in crns:
+        c = c.strip()
+        print(c)
+        cursor.execute(q % (semester,c))
+        r = cursor.fetchall()
+        for inst in r:
+            emails.add(inst[8])
+            print(inst)
+    open('cache/eval_emails.txt','w').write( ';'.join(emails))
+    return emails
+   
+
+
+def all_sem_courses_teachers():
+    q = """SELECT c.id, c.canvasid AS course_cid, c.name, c.code, u.name, u.sortablename, u.canvasid AS user_cid, p.sis_user_id  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+JOIN pseudonym AS p ON p.user_id=u.id
+WHERE c.sis LIKE "202170-%" 
+AND NOT c.state="deleted"
+AND e."type"="TeacherEnrollment"
+ORDER BY u.sortablename;"""
+    (connection,cursor) = db()
+    cursor.execute(q)
+    courses = cursor.fetchall()
+    #print(courses)
+    return courses
+
+
+def to_sis_sem(s):
+    season = s[0:2]
+    year = "20" + s[2:5]
+    a = {'sp':'30','su':'50','fa':'70'}
+    season = a[season]
+    return year+season
+
+def build_db_schedule():
+    # from the schedule json files
+    target = "\_sched\_expanded\.json"
+    def finder(st):
+        return re.search(target,st)
+    
+    fields = 'sem,sem_sis,crn,dept,num,gp,dean,code,name,teacher,type,cap,act,loc,site,date,days,time,cred,ztc,partofday'.split(',')
+    fff = codecs.open('cache/schedule_db_version.sql', 'w', 'utf-8')
+    fff.write("CREATE TABLE IF NOT EXISTS schedule ( id text, sem text, sem_sis text, dept text, num text, gp text, dean text, code text, crn text, name text, teacher text,mode text, loc text, cap text, act text, site text, date text, cred text, ztc text, days text, time text, partofday text);\n")
+    all = os.listdir('cache/')
+    all = list(funcy.filter( finder, all ))
+    all.sort()
+    for F in all:
+        print("\n\n" + F)
+        sched = json.loads(codecs.open('cache/'+F,'r','utf-8').read())
+        for S in sched:
+            parts = S['code'].split(' ')
+            S['dept'] = parts[0]
+            S['num'] = parts[1]
+            S['gp'] = gp[parts[0]]
+            S['dean'] = dean[parts[0]]
+            S['sem'] = F[0:4]
+            S['sem_sis'] = to_sis_sem(F[0:4])
+            if not 'partofday' in S:
+                S['partofday'] = ''
+            str = "INSERT INTO schedule (sem,sem_sis,crn,dept,num,gp,dean,code,name,teacher,mode,cap,act,loc,site,date,days,time,cred,ztc,partofday) VALUES (%s);\n" % \
+                  ", ".join( [ "'" + re.sub(r"'", "", S[x]) + "'" for x in fields ] )
+            #print(str)
+            fff.write(str)
+
+def process_enrollment_data():
+    
+    sem_index = {'201830':0, '201850':1, '201870':2, '201930':3, '201950':4, '201970':5, '202030':6, '202050':7, '202070':8, '202130':9, '202150':10, '202170':11, '202230':12, '202250':13, '202270':14, '202330':15}
+    
+    def sem_to_idx(s): 
+        return sem_index[str(s)]
+    
+    p = pd.read_csv('cache/20221207_all_enrollments_by_student.csv')
+    p = p.fillna('')
+    p['sem_idx'] = p['sem'].map(sem_to_idx)
+    print(p)
+    print(sorted(p['sem'].unique()) )
+    print(sorted(p['mode'].unique()) )
+    print(sorted(p['site'].unique()) )
+    print(sorted(p['partofday'].unique()) )
+    print(sorted(p['days'].unique()) )
+    print(sorted(p['dept'].unique()) )
+    print(sorted(p['num'].unique()) )
+    print(len(p['num'].unique()) )
+    
+    print("I see this many student/semester rows: ", len(p))
+    
+    #q = p.groupby(["canvasid","sem"])
+    
+    q = p.groupby(["canvasid"])
+    print("I see this many students: ", len(q))
+    #print(q.size())
+    r = pd.DataFrame(q.size())
+    print("Summary of course counts")
+    print(r.iloc[:,0].value_counts())
+    
+    out = codecs.open('cache/20221207_all_enrollments_by_student_with_sems.csv','w','utf-8')
+    out.write('"canvasid","sem","mode","site","partofday","days","dept","num","cred","sem_idx","local_sem_idx"\n')
+    
+    # convert absolute semester to sequence,
+    # ie: student's 1st, 2nd, 3rd, etc
+    for name,group in q:
+        # drop students with only a single semester -- no predictive value here
+        if len(group['sem_idx'].unique())<2:
+            continue
+        mn = group['sem_idx'].min()
+        group.loc[:,'local_sem_idx'] = group['sem_idx'].map(lambda x: x - mn)
+        out.write(group.to_csv(index=False, header=False))
+    
+    s = p.groupby(by="sem")
+    #print("I see this many semesters: ", len(s))
+    #print(s.size())
+    
+
+    
+    # todo
+    
+def do_encoding():
+    # one shot encoding of each field
+    
+    modes = {'hybrid':[0,0,0,1], 'in-person':[1,0,0,0], 'online':[0,1,0,0], 'online live':[0,0,1,0]}
+    sites = {'Coyote Valley':[1,0,0,0,0,0], 'Gilroy':[0,1,0,0,0,0], 'Hollister':[0,0,1,0,0,0], 'Morgan Hill':[0,0,0,1,0,0], 'Online':[0,0,0,0,1,0], 'Other':[0,0,0,0,0,0], 'San Martin Airport':[0,0,0,0,0,1], 'TBA':[0,0,0,0,0,0]}
+    times = {'':[0,0,0,0,0], 'Afternoon':[0,0,1,0,0], 'Evening':[0,0,0,1,0], 'Midday':[0,1,0,0,0], 'Morning':[1,0,0,0,0]}
+    days  = {'':[0,0,0,0,0,0], 'F':[0,0,0,0,1,0], 'FS':[0,0,0,0,1,1], 'M':[1,0,0,0,0,0], 'MF':[1,0,0,0,1,0], 'MR':[1,0,0,1,0,0], 'MT':[1,1,0,0,0,0], 'MTR':[1,1,0,1,0,0], 'MTRF':[1,1,0,1,1,0], 'MTW':[1,1,1,0,0,0], 'MTWF':[1,1,0,1,1,0], 'MTWR':[1,1,1,1,0,0], 'MTWRF':[1,1,1,1,1,0], 'MW':[1,0,1,0,0,0], 'MWF':[1,0,1,0,1,0], 'MWR':[1,0,1,1,0,0], 'R':[0,0,0,1,0,0], 'RF':[0,0,0,1,1,0], 'S':[0,0,0,0,0,1], 'T':[0,1,0,0,0,0], 'TBA':[0,0,0,0,0,0], 'TF':[0,1,0,0,1,0], 'TR':[0,1,0,1,0,0], 'TRF':[0,1,0,1,1,0], 'TW':[0,1,1,0,0,0], 'TWR':[0,1,1,1,0,0], 'TWRF':[0,1,1,1,1,0], 'U':[0,0,0,0,0,0], 'W':[0,0,1,0,0,0], 'WF':[0,0,1,0,1,0], 'WR':[0,0,1,1,0,0]}
+    
+    deptslist = ['ACCT', 'AE', 'AH', 'AJ', 'AMT', 'ANTH', 'APE', 'ART', 'ASTR', 'ATH', 'BIO', 'BOT', 'BUS', 'CD', 'CHEM', 'CMGT', 'CMUN', 'COS', 'CSIS', 'CWE', 'DM', 'ECOL', 'ECON', 'ENGL', 'ENGR', 'ENVS', 'ESL', 'ETHN', 'FRNH', 'GEOG', 'GEOL', 'GUID', 'HE', 'HIST', 'HTM', 'HUM', 'HVAC', 'JFT', 'JLE', 'JOUR', 'JPN', 'KIN', 'LIB', 'LIFE', 'MATH', 'MCTV', 'MGMT', 'MUS', 'PHIL', 'PHYS', 'POLS', 'PSCI', 'PSYC', 'RE', 'SJS', 'SOC', 'SPAN', 'THEA', 'WELD', 'WTRM']
+    d_len = len(deptslist)
+    d_template = [ 0 for i in range(d_len) ]
+    depts = {}
+    for i in range(d_len):
+        depts[  deptslist[i] ] = d_template.copy()
+        depts[  deptslist[i] ][i] = 1
+        
+    numslist =  ['1', '10', '100', '100A', '101', '102', '103', '104', '105', '107', '107A', '109', '10A', '10B', '11', '110', '111', '112', '113', '114', '118', '119', '11A', '11B', '11C', '12', '120', '121', '122', '124', '126', '128', '129', '12A', '12B', '12L', '13', '130', '131', '132', '133', '135', '13A', '13B', '13C', '13D', '14', '140', '142', '143', '144', '14A', '14B', '15', '150', '152', '154', '156', '157', '158', '15A', '15B', '16', '160', '162', '164', '166', '16A', '16B', '16C', '17', '171', '173', '175', '176', '178', '179', '17A', '17B', '17C', '18', '180', '181', '182', '183', '184', '186', '187', '189', '18A', '18B', '19', '190', '191A', '192', '19A', '19B', '19C', '1A', '1B', '1C', '1L', '2', '20', '200', '201', '202', '203', '204', '205', '206', '207', '208', '209', '20A', '20B', '20C', '21', '210', '211', '212', '213', '213A', '214', '215', '216', '217', '218', '219', '22', '220', '221', '223', '225', '226', '227', '228', '229', '229A', '23', '230', '231', '231A', '232', '233', '235', '236', '24', '240', '242', '24A', '24B', '24C', '24D', '25', '250', '25A', '25B', '26', '260', '27', '270', '28', '280', '281', '282', '283', '290', '291A', '2A', '2B', '2C', '2F', '2J', '2L', '3', '301', '30A', '30B', '32', '33A', '33B', '33C', '34', '34A', '34B', '35', '36', '37', '38', '3A', '3B', '3C', '3D', '4', '40', '400', '402', '41', '411', '412', '412A', '412B', '413', '414', '415', '416', '42', '420', '43', '430', '44', '440', '44A', '44B', '44C', '45', '46', '47', '48', '4A', '4B', '4C', '5', '51', '52', '527', '528', '53', '530', '531', '534', '535', '536', '537', '538', '539', '54', '541', '542', '543', '547', '548', '549', '54L', '55', '550', '552', '553', '554', '557', '558A', '56', '560', '562', '563', '564', '565', '569', '570A', '570B', '571A', '571B', '571C', '575', '5A', '5B', '6', '60', '600', '601', '602', '603', '61A', '61B', '61C', '62A', '62B', '62C', '636', '638', '64A', '64B', '64C', '64D', '65', '66A', '66B', '66C', '68A', '68B', '68C', '7', '700', '701', '702A', '702B', '703', '704A', '705', '706', '707', '709', '70A', '71', '710', '71A', '71B', '727', '728', '73', '731', '732', '737', '738', '74', '740', '741', '742', '743', '744', '746', '747', '748', '749', '74A', '74B', '75', '752', '753', '754', '756', '76', '762', '763', '764', '77', '775', '776', '78', '784', '785', '786', '787', '788', '789', '79', '793', '7A', '7B', '7C', '8', '80', '81A', '81C', '83', '83A', '84', '85', '88A', '88B', '8A', '8B', '8C', '9', '90', '91A', '91B', '92', '97', '9A', '9B']
+    n_len = len(numslist)
+    n_template = [ 0 for i in range(n_len) ]
+    nums = {}
+    for i in range(n_len):
+        nums[  numslist[i] ] = n_template.copy()
+        nums[  numslist[i] ][i] = 1
+    
+    return [modes,sites,times,days,depts,nums]
+    
+    for x in [modes,sites,times,days,depts,nums]:
+        print('var')
+        for k,v in x.items():
+            print("\t",k,":",v)
+                
+
+if __name__ == "__main__":
+
+    print ('')
+    options = { 
+            1: ['Read and join communications channels.', merge_comm_channel], 
+            2: ['Read and join users files.', merge_users ],
+            3: ['Read and join courses files.', merge_courses ],
+            4: ['Read and join enrollment files.', merge_enrollment ],
+            5: ['Read and join terms files.', merge_term ],
+            6: ['Read and join roles files.', merge_roles ],
+            7: ['Read and join conversation files', merge_convos],
+            8: ['Read all courses', semester_enrollments],
+            9: ['Load requests files. Merge into 15min blocks.', merge_requests ], 
+            10: ['Full reload. Rename current db.', full_reload],
+            11: ['test setup index', abcd],
+            12: ['Test web version of data files (json)', make_views_summarys],   #depts_with_classcounts],   # arrange_data_for_web],
+            13: ['Test web version of data files (?)', depts_with_classcounts],  
+            14: ['Student views, classes in 1 dept', dept_with_studentviews],
+            15: ['AEC report positive attendance', aec_su20_report],
+            16: ['Create list of all employees', all_gav_employees],
+            17: ['List emails of evaluated instructors this semester', crns_to_teachers],
+            18: ['Fetch this semester shells with teachers', all_sem_courses_teachers],
+            19: ['Build DB schedule from json files', build_db_schedule],
+            20: ['Process enrollment data', process_enrollment_data],
+            21: ['Encode data', do_encoding],
+            #19: ['add evals for a whole semester', instructor_list_to_activate_evals],
+            #16: ['Upload new employees to flex app', employees_refresh_flex],
+            }
+
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
+
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..ee23bb2
--- /dev/null
+++ b/main.py
@@ -0,0 +1,141 @@
+
+"""
+
+Main entry point for Gavilan Canvas Tools App. 
+
+"""
+
+
+
+
+#import datetime
+from HTMLParser import HTMLParseError
+from bs4 import BeautifulSoup as bs
+from bs4 import Comment
+from collections import defaultdict
+from datetime import date
+from datetime import datetime
+from datetime import timedelta
+from dateutil import parser
+from dateutil import tz
+from itertools import groupby
+from sets import Set
+from time import strptime, mktime
+import base64
+import codecs
+import collections
+import csv
+import gzip
+import hashlib
+import hmac
+import html
+import htmlentitydefs
+import html2markdown as h2m
+import imghdr
+import isoweek
+import json
+import math
+import numpy as np
+import os
+import sys
+import glob
+import pandas as pd
+import pdb
+import pysftp
+import pytz
+import re
+import requests
+import shutil
+import sqlite3
+import subprocess
+import time
+import urllib
+import webbrowser
+import xlwt
+
+import checker
+from pipelines import *
+from stats import *
+from users import *
+from util import *
+from courses import *
+from tasks import *
+from outcomes import *
+from content import *
+    
+#from upload_to_web import put_file
+       
+    
+    
+
+               
+   
+        
+if __name__ == "__main__":
+    
+    print ("")
+    options = {     
+                # Basic info & random questions
+                39:['List all terms', getTerms],
+                1: ['Current Activity',getCurrentActivity] ,  
+                3: ['List Course Info', getCourses] ,
+                5: ['List users in a course', getUsersInCourse] ,
+                8: ['List courses in a term', getCoursesInTerm] ,
+                12:['Get current classes', class_logs],
+                13:['Logs for one user', user_logs],
+                14:['Recent logins, last 5 min', recent_logins],
+                
+                # User tracking 
+                #4: ['List all users with a gavilan.edu address (employees)', getTeacherRoles] ,
+                27:['Grades summary of a semester', grades_rundown] ,
+                30:['List inactive teachers in term', getInactiveTeachersInTerm] ,
+                6: ['List all teachers', getAllTeachers] ,
+                15:['All teachers in a term',getAllTeachersInTerm],
+                16:['Make log of teacher activity',teacherActivityLog],
+                
+                # Sched or Semester info
+                #17:['Construct schedule from html file',constructSchedule],
+                18:['List late-start classes',list_latestarts],                         ###
+                19:['External tools',externaltool],
+                
+                # Tasks
+                9: ['Upload a photo', uploadPhoto],
+                10:['Download new photos', downloadPhoto],
+                11:['Check for avatar',checkForAvatar],
+                25:['X-List 190 sections', xlist_cwe] ,                                 ###
+                28:['Check accessibility of a course', accessible_check] ,
+                29:['Switch enrollments of a shell to all teachers', switch_enrol] ,
+                35:['Enroll user to all active courses in a semester', enroll_accred],
+                36:['Fix an older course so it can be enrolled again, add accred', unrestrict_course],
+                38:['Modify external tool', modify_x_tool],
+               
+                # Post semester
+                2: ['Positive Attendance Report', pos_atten] ,
+                26:['20/60 hours calculation', hours_calc] ,
+                
+                # Outcomes
+                20:['List outcomes and groups at account level',outcome_groups],
+                21:['Get outcome results',outcome_report2],
+                22:['List outcomes attached to classes', outcomes_attached_to_courses] ,
+                23:['Read the SLOs local file', read_slo_source] ,
+                24:['Outcome overview and sync', outcome_overview] ,
+                
+                # Content editing or pulling
+                31:['Auto update a canvas page to remove fancy html', update_page] ,
+                32:['Download a courses pages for offline updating', grab_course_pages] ,
+                33:['Upload course pages back to a class', put_course_pages],
+                34:['Swap course youtube embeds', swap_youtube_subtitles],
+                37:['Test the iframe swap', test_swap],
+                
+                
+              }
+    
+    for key in options:
+        print str(key) + '.\t' + options[key][0]
+    resp = raw_input('\nChoose: ')
+    
+    results = []
+    results_dict = {}
+    
+    # Call the function in the options dict
+    x = options[ int(resp)][1]() 
diff --git a/myconsole.py b/myconsole.py
new file mode 100644
index 0000000..f48fdb5
--- /dev/null
+++ b/myconsole.py
@@ -0,0 +1,57 @@
+
+
+import importlib, signal
+
+
+
+
+import outcomes
+import pipelines
+import curric2022
+
+def handler(signum, frame):
+    print("\n\nCancelled.\n\n")
+    exit(1)
+ 
+ 
+    
+    
+def mainloop():
+    
+    print ('')
+    options = { 1: ['run sample course', curric2022.sampleclass],
+                2: ['pattern matcher style', curric2022.matchstyle],
+                3: ['pattern matcher - test on all classes', curric2022.match_style_test],
+                4: ['process all classes', curric2022.path_style_test],
+                5: ['process all programs', curric2022.path_style_prog],
+                6: ['show course outcomes', curric2022.all_outcomes],
+                7: ['Main outcome show & modify for a semester', outcomes.outcome_overview],
+                8: ['The outcome groups and links in iLearn',     outcomes.outcome_groups],
+                9: ['Outcome report #2 sample', outcomes.outcome_report2],
+               10: ['Outcome groups dump', outcomes.outcome_groups_dump],
+               11: ['All outcomes attached to courses', outcomes.outcomes_attached_to_courses],
+                0: ['exit', exit],
+              }
+    
+    for key in options:
+        print((str(key) + '.\t' + options[key][0]))
+    
+    print('')
+    #resp = eval(input('Choose: '))
+    resp = input('Choose: ')
+    
+    importlib.reload(outcomes)
+    importlib.reload(pipelines)
+    importlib.reload(curric2022)
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+    
+    print("\n\n\n\n")
+    mainloop()
+
+
+
+signal.signal(signal.SIGINT, handler)
+
+mainloop()
+
diff --git a/new flex app.md b/new flex app.md
new file mode 100644
index 0000000..0d3d283
--- /dev/null
+++ b/new flex app.md	
@@ -0,0 +1,68 @@
+
+
+
+## Ideas and links
+
+This, spacy: https://www.analyticsvidhya.com/blog/2020/06/nlp-project-information-extraction/
+
+Stackexchange: https://datascience.stackexchange.com/questions/12053/a-few-ideas-to-parse-events-from-a-text-document
+https://stackoverflow.com/questions/2587663/natural-language-parsing-of-an-appointment
+
+
+Sherlock (javascript): https://github.com/neilgupta/Sherlock
+
+Plain python:  dateutil.parser.parse("today is 21 jan 2016", fuzzy=True)
+
+NLTK reference: https://stackoverflow.com/questions/10340540/using-the-nltk-to-recognise-dates-as-named-entities?noredirect=1&lq=1
+
+Multiple plain python packages: https://stackoverflow.com/questions/19994396/best-way-to-identify-and-extract-dates-from-text-python?noredirect=1&lq=1
+
+Chrono - nat lang date parser javascript - https://github.com/wanasit/chrono
+https://github.com/wanasit/chrono-python
+
+Natty - java nat language date parser - https://github.com/joestelmach/natty
+
+
+
+## Snippets that I'd like my program to find the fields and put them in the right slots automatically (separated by --)
+
+
+Join Better Together Labs for an exclusive Zoom workshop:
+
+Creative and Playful Ways to Engage Learners on Zoom, Part 3
+
+Tuesday, August 11, 2020, 1:00pm Pacific/4:00pm Eastern, 90 minutes
+
+This hands-on workshop will explore Zoom games and activities that foster engagement, provide a sense of community, and leave your learners looking forward to more.
+
+--
+
+Multiple Dates
+https://livetraining.zoom.us/webinar/register/8915786869708/WN_Qkc7KpkNSFOdlTwpZkGFlQ
+Topic
+Getting Started with Zoom Meetings
+Description
+Ready to start using Zoom, but need some help? Drop-in for our daily (Mon-Fri) quick starts! A Zoom expert will take you through a 45-minute high-level tour of Zoom and cover the basics to get you up and running. It’s as simple as logging in, scheduling a meeting, and finding the controls. Start Zooming today! Stick around to get all your burning questions answered through live Q&A!
+
+--
+
+Monday @ 2pm
+
+Tuesday @ 10am and 2pm
+
+Thursday @ 10am and 2pm
+
+Friday @ 10am
+
+(All times in PST)
+
+Join Zoom expert Raul Montes to learn the Zoom basics: scheduling, recording, screen sharing, and more.
+
+Register Now: https://zoom.us/webinar/register/weeklylivedemo
+
+--
+
+
+
+
+
diff --git a/notebook.ipynb b/notebook.ipynb
new file mode 100644
index 0000000..6ffd95d
--- /dev/null
+++ b/notebook.ipynb
@@ -0,0 +1,1577 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "https://api.inshosteddata.com/api/account/self/file/sync\n",
+      "{'Authorization': 'HMACAuth 948e67098a8822cf37104f35e88b019144c7c39a:QDdp4T9CqLw35FXM2qsC7BIspK39oNBINQusfXD2pq8=', 'Date': 'Sat, 12 Dec 20 11:37:47 GMT', 'Content-type': 'application/json'}\n",
+      "0.\tLocal: No \tRemote: account_dim-00000-f5038498.gz\n",
+      "1.\tLocal: No \tRemote: assignment_dim-00000-c36a7689.gz\n",
+      "2.\tLocal: No \tRemote: assignment_fact-00000-2e0dd051.gz\n",
+      "3.\tLocal: No \tRemote: assignment_group_dim-00000-27db5aaf.gz\n",
+      "4.\tLocal: No \tRemote: assignment_group_fact-00000-f1b5fe92.gz\n",
+      "5.\tLocal: No \tRemote: assignment_group_rule_dim-00000-20680b49.gz\n",
+      "6.\tLocal: No \tRemote: assignment_group_score_dim-00000-7ae8f32b.gz\n",
+      "7.\tLocal: No \tRemote: assignment_group_score_fact-00000-9074ea0f.gz\n",
+      "8.\tLocal: No \tRemote: assignment_override_dim-00000-47d7db7e.gz\n",
+      "9.\tLocal: No \tRemote: assignment_override_fact-00000-1915e098.gz\n",
+      "10.\tLocal: No \tRemote: assignment_override_user_dim-00000-77eb4caa.gz\n",
+      "11.\tLocal: No \tRemote: assignment_override_user_fact-00000-7159d514.gz\n",
+      "12.\tLocal: No \tRemote: assignment_override_user_rollup_fact-00000-5f6dd787.gz\n",
+      "13.\tLocal: No \tRemote: assignment_rule_dim-00000-7225a11b.gz\n",
+      "14.\tLocal: No \tRemote: communication_channel_dim-00000-eb7d1170.gz\n",
+      "15.\tLocal: No \tRemote: communication_channel_fact-00000-958a1852.gz\n",
+      "16.\tLocal: No \tRemote: conference_dim-00000-ef5be0e5.gz\n",
+      "17.\tLocal: No \tRemote: conference_fact-00000-6d5c2eb2.gz\n",
+      "18.\tLocal: No \tRemote: conference_participant_dim-00000-0719c5b0.gz\n",
+      "19.\tLocal: No \tRemote: conference_participant_fact-00000-ecb9c07d.gz\n",
+      "20.\tLocal: No \tRemote: conversation_dim-00000-56c32424.gz\n",
+      "21.\tLocal: No \tRemote: conversation_message_dim-00000-da96a9e4.gz\n",
+      "22.\tLocal: No \tRemote: conversation_message_participant_fact-00000-02075b76.gz\n",
+      "23.\tLocal: No \tRemote: conversation_message_participant_fact-00001-3990abd5.gz\n",
+      "24.\tLocal: No \tRemote: course_dim-00000-5b48adc7.gz\n",
+      "25.\tLocal: No \tRemote: course_score_dim-00000-8eaf6477.gz\n",
+      "26.\tLocal: No \tRemote: course_score_fact-00000-62e7aecc.gz\n",
+      "27.\tLocal: No \tRemote: course_section_dim-00000-63a92a38.gz\n",
+      "28.\tLocal: No \tRemote: course_ui_canvas_navigation_dim-00000-b419a2ad.gz\n",
+      "29.\tLocal: No \tRemote: course_ui_navigation_item_dim-00000-098a3d0c.gz\n",
+      "30.\tLocal: No \tRemote: course_ui_navigation_item_fact-00000-96eaa771.gz\n",
+      "31.\tLocal: No \tRemote: discussion_entry_dim-00000-4b41808c.gz\n",
+      "32.\tLocal: No \tRemote: discussion_entry_fact-00000-2f037f74.gz\n",
+      "33.\tLocal: No \tRemote: discussion_topic_dim-00000-c6c192d6.gz\n",
+      "34.\tLocal: No \tRemote: discussion_topic_fact-00000-55cfebee.gz\n",
+      "35.\tLocal: No \tRemote: enrollment_dim-00000-37136d59.gz\n",
+      "36.\tLocal: No \tRemote: enrollment_fact-00000-552e14f4.gz\n",
+      "37.\tLocal: No \tRemote: enrollment_rollup_dim-00000-5405ffd0.gz\n",
+      "38.\tLocal: No \tRemote: enrollment_term_dim-00000-bad49cc0.gz\n",
+      "39.\tLocal: No \tRemote: external_tool_activation_dim-00000-903f1257.gz\n",
+      "40.\tLocal: No \tRemote: external_tool_activation_fact-00000-7b6f11ad.gz\n",
+      "41.\tLocal: No \tRemote: file_dim-00000-dda644c6.gz\n",
+      "42.\tLocal: No \tRemote: file_fact-00000-1702d3ba.gz\n",
+      "43.\tLocal: No \tRemote: group_dim-00000-6481df05.gz\n",
+      "44.\tLocal: No \tRemote: group_fact-00000-d0c78601.gz\n",
+      "45.\tLocal: No \tRemote: group_membership_dim-00000-10516d99.gz\n",
+      "46.\tLocal: No \tRemote: group_membership_fact-00000-dd72bd7e.gz\n",
+      "47.\tLocal: No \tRemote: learning_outcome_dim-00000-b66f806b.gz\n",
+      "48.\tLocal: No \tRemote: learning_outcome_fact-00000-d835f148.gz\n",
+      "49.\tLocal: No \tRemote: learning_outcome_group_association_fact-00000-a1c27ba1.gz\n",
+      "50.\tLocal: No \tRemote: learning_outcome_group_dim-00000-f60e46ea.gz\n",
+      "51.\tLocal: No \tRemote: learning_outcome_group_fact-00000-5b138b0a.gz\n",
+      "52.\tLocal: No \tRemote: learning_outcome_question_result_dim-00000-48c8d782.gz\n",
+      "53.\tLocal: No \tRemote: learning_outcome_question_result_fact-00000-c5cbb003.gz\n",
+      "54.\tLocal: No \tRemote: learning_outcome_result_dim-00000-31c0018d.gz\n",
+      "55.\tLocal: No \tRemote: learning_outcome_result_fact-00000-b043439d.gz\n",
+      "56.\tLocal: No \tRemote: learning_outcome_rubric_criterion_dim-00000-ec2d4638.gz\n",
+      "57.\tLocal: No \tRemote: learning_outcome_rubric_criterion_fact-00000-0746a294.gz\n",
+      "58.\tLocal: No \tRemote: module_completion_requirement_dim-00000-690c6eb0.gz\n",
+      "59.\tLocal: No \tRemote: module_completion_requirement_fact-00000-79cd62f9.gz\n",
+      "60.\tLocal: No \tRemote: module_dim-00000-29e24c02.gz\n",
+      "61.\tLocal: No \tRemote: module_fact-00000-e832627e.gz\n",
+      "62.\tLocal: No \tRemote: module_item_dim-00000-4d586235.gz\n",
+      "63.\tLocal: No \tRemote: module_item_fact-00000-63d1a901.gz\n",
+      "64.\tLocal: No \tRemote: module_prerequisite_dim-00000-a0757c3c.gz\n",
+      "65.\tLocal: No \tRemote: module_prerequisite_fact-00000-5aedc7a7.gz\n",
+      "66.\tLocal: No \tRemote: module_progression_completion_requirement_dim-00000-74c4a53f.gz\n",
+      "67.\tLocal: No \tRemote: module_progression_completion_requirement_fact-00000-b5f22af6.gz\n",
+      "68.\tLocal: No \tRemote: module_progression_dim-00000-9fea8745.gz\n",
+      "69.\tLocal: No \tRemote: module_progression_dim-00001-aaeab561.gz\n",
+      "70.\tLocal: No \tRemote: module_progression_fact-00000-10b5fc11.gz\n",
+      "71.\tLocal: No \tRemote: module_progression_fact-00001-04026d35.gz\n",
+      "72.\tLocal: No \tRemote: pseudonym_dim-00000-da451682.gz\n",
+      "73.\tLocal: No \tRemote: pseudonym_fact-00000-68410591.gz\n",
+      "74.\tLocal: No \tRemote: quiz_dim-00000-e92dac6c.gz\n",
+      "75.\tLocal: No \tRemote: quiz_fact-00000-c173e87e.gz\n",
+      "76.\tLocal: No \tRemote: quiz_question_answer_dim-00000-804f40ab.gz\n",
+      "77.\tLocal: No \tRemote: quiz_question_answer_dim-00001-68b50f49.gz\n",
+      "78.\tLocal: No \tRemote: quiz_question_answer_dim-00002-bfffd265.gz\n",
+      "79.\tLocal: No \tRemote: quiz_question_answer_fact-00000-c82d06a6.gz\n",
+      "80.\tLocal: No \tRemote: quiz_question_answer_fact-00001-2b0d9381.gz\n",
+      "81.\tLocal: No \tRemote: quiz_question_answer_fact-00002-0bbee5f7.gz\n",
+      "82.\tLocal: No \tRemote: quiz_question_dim-00000-533bac25.gz\n",
+      "83.\tLocal: No \tRemote: quiz_question_fact-00000-ebb35794.gz\n",
+      "84.\tLocal: No \tRemote: quiz_question_group_dim-00000-f6fffb18.gz\n",
+      "85.\tLocal: No \tRemote: quiz_question_group_fact-00000-e00c4ae7.gz\n",
+      "86.\tLocal: No \tRemote: quiz_submission_dim-00000-48e0508f.gz\n",
+      "87.\tLocal: No \tRemote: quiz_submission_fact-00000-9124331b.gz\n",
+      "88.\tLocal: No \tRemote: quiz_submission_historical_dim-00000-d426c6e7.gz\n",
+      "89.\tLocal: No \tRemote: quiz_submission_historical_fact-00000-b0df0425.gz\n",
+      "90.\tLocal: No \tRemote: requests-00000-8e6af56f.gz\n",
+      "91.\tLocal: No \tRemote: requests-00001-790a397b.gz\n",
+      "92.\tLocal: No \tRemote: role_dim-00000-63fdee5b.gz\n",
+      "93.\tLocal: No \tRemote: submission_comment_dim-00000-aaf1fa9a.gz\n",
+      "94.\tLocal: No \tRemote: submission_comment_fact-00000-fceab0e2.gz\n",
+      "95.\tLocal: No \tRemote: submission_dim-00000-201559a3.gz\n",
+      "96.\tLocal: No \tRemote: submission_dim-00001-b85b854b.gz\n",
+      "97.\tLocal: No \tRemote: submission_dim-00002-7370540c.gz\n",
+      "98.\tLocal: No \tRemote: submission_dim-00003-d8a31bf9.gz\n",
+      "99.\tLocal: No \tRemote: submission_dim-00004-e1862304.gz\n",
+      "100.\tLocal: No \tRemote: submission_fact-00000-2744be21.gz\n",
+      "101.\tLocal: No \tRemote: submission_fact-00001-84354ca8.gz\n",
+      "102.\tLocal: No \tRemote: submission_fact-00002-a9c00364.gz\n",
+      "103.\tLocal: No \tRemote: submission_fact-00003-082011f6.gz\n",
+      "104.\tLocal: No \tRemote: submission_fact-00004-5ff7a0d4.gz\n",
+      "105.\tLocal: No \tRemote: submission_file_fact-00000-bc8d0ad1.gz\n",
+      "106.\tLocal: No \tRemote: user_dim-00000-6eee8f8c.gz\n",
+      "107.\tLocal: No \tRemote: wiki_dim-00000-07eef6e0.gz\n",
+      "108.\tLocal: No \tRemote: wiki_fact-00000-252b3fbb.gz\n",
+      "109.\tLocal: No \tRemote: wiki_page_dim-00000-45b46d1b.gz\n",
+      "110.\tLocal: No \tRemote: wiki_page_fact-00000-4e658212.gz\n",
+      "111.\tLocal: No \tRemote: requests-00000-60d4d55e.gz\n",
+      "112.\tLocal: No \tRemote: requests-00001-b9f02622.gz\n",
+      "113.\tLocal: No \tRemote: requests-00000-a1e875d5.gz\n",
+      "114.\tLocal: No \tRemote: requests-00001-0c8a9e5e.gz\n",
+      "115.\tLocal: No \tRemote: requests-00000-e1d4452b.gz\n",
+      "116.\tLocal: No \tRemote: requests-00001-a8ff7754.gz\n",
+      "117.\tLocal: No \tRemote: requests-00000-953a589b.gz\n",
+      "118.\tLocal: No \tRemote: requests-00001-9e152530.gz\n",
+      "119.\tLocal: Yes\tRemote: requests-00000-ab4ff4dc.gz\n",
+      "120.\tLocal: Yes\tRemote: requests-00001-ec4b38a3.gz\n",
+      "121.\tLocal: Yes\tRemote: requests-00000-d6acd272.gz\n",
+      "122.\tLocal: Yes\tRemote: requests-00001-9bd12ea2.gz\n",
+      "123.\tLocal: Yes\tRemote: requests-00000-3561c7c6.gz\n",
+      "124.\tLocal: Yes\tRemote: requests-00001-6e190c85.gz\n",
+      "125.\tLocal: Yes\tRemote: requests-00000-de0e1602.gz\n",
+      "126.\tLocal: Yes\tRemote: requests-00001-bab761a4.gz\n",
+      "127.\tLocal: Yes\tRemote: requests-00000-77dd2074.gz\n",
+      "128.\tLocal: Yes\tRemote: requests-00001-3e20bcc9.gz\n",
+      "129.\tLocal: Yes\tRemote: requests-00000-9840c677.gz\n",
+      "130.\tLocal: Yes\tRemote: requests-00000-5f4dc4aa.gz\n",
+      "131.\tLocal: Yes\tRemote: requests-00000-6d3a3537.gz\n",
+      "132.\tLocal: Yes\tRemote: requests-00000-fa43743f.gz\n",
+      "133.\tLocal: Yes\tRemote: requests-00000-41e5cad0.gz\n",
+      "134.\tLocal: Yes\tRemote: requests-00000-c8736ee0.gz\n",
+      "135.\tLocal: Yes\tRemote: requests-00000-59141c8a.gz\n",
+      "136.\tLocal: Yes\tRemote: requests-00001-195357b3.gz\n",
+      "137.\tLocal: Yes\tRemote: requests-00000-028d0ed6.gz\n",
+      "138.\tLocal: Yes\tRemote: requests-00001-380f6bf7.gz\n",
+      "139.\tLocal: Yes\tRemote: requests-00002-2b41ae7f.gz\n",
+      "140.\tLocal: Yes\tRemote: requests-00003-f976534f.gz\n",
+      "141.\tLocal: Yes\tRemote: requests-00004-ddd831f1.gz\n",
+      "142.\tLocal: Yes\tRemote: requests-00005-c0ab0033.gz\n",
+      "143.\tLocal: Yes\tRemote: requests-00006-45f3e13d.gz\n",
+      "144.\tLocal: Yes\tRemote: requests-00007-764add86.gz\n",
+      "145.\tLocal: Yes\tRemote: requests-00008-35a4bf69.gz\n",
+      "146.\tLocal: Yes\tRemote: requests-00009-af283d36.gz\n",
+      "147.\tLocal: Yes\tRemote: requests-00010-b9c55786.gz\n",
+      "148.\tLocal: Yes\tRemote: requests-00011-b4d9ac41.gz\n",
+      "149.\tLocal: Yes\tRemote: requests-00012-4d17e1a8.gz\n",
+      "150.\tLocal: Yes\tRemote: requests-00013-dd0ee685.gz\n",
+      "151.\tLocal: Yes\tRemote: requests-00014-3a12b5f9.gz\n",
+      "152.\tLocal: Yes\tRemote: requests-00015-6bc1a59c.gz\n",
+      "153.\tLocal: Yes\tRemote: requests-00016-4c935f2a.gz\n",
+      "154.\tLocal: Yes\tRemote: requests-00017-1aa47b9a.gz\n",
+      "155.\tLocal: Yes\tRemote: requests-00018-a0332f05.gz\n",
+      "156.\tLocal: Yes\tRemote: requests-00019-541d78ca.gz\n",
+      "157.\tLocal: Yes\tRemote: requests-00020-b2ad81c2.gz\n",
+      "158.\tLocal: Yes\tRemote: requests-00021-1f849c66.gz\n",
+      "159.\tLocal: Yes\tRemote: requests-00022-2c0d5a9b.gz\n",
+      "160.\tLocal: Yes\tRemote: requests-00023-7afdecf3.gz\n",
+      "161.\tLocal: Yes\tRemote: requests-00024-26779167.gz\n",
+      "162.\tLocal: Yes\tRemote: requests-00025-15d6dee5.gz\n",
+      "163.\tLocal: Yes\tRemote: requests-00026-e8e087b5.gz\n",
+      "164.\tLocal: Yes\tRemote: requests-00027-875befdf.gz\n",
+      "165.\tLocal: Yes\tRemote: requests-00028-1ef3dc94.gz\n",
+      "166.\tLocal: Yes\tRemote: requests-00029-49bf949f.gz\n",
+      "167.\tLocal: Yes\tRemote: requests-00030-cbfd3cc2.gz\n",
+      "168.\tLocal: Yes\tRemote: requests-00031-9e3cc1cf.gz\n",
+      "169.\tLocal: Yes\tRemote: requests-00032-6aae6156.gz\n",
+      "170.\tLocal: Yes\tRemote: requests-00033-c4a8ca1b.gz\n",
+      "171.\tLocal: Yes\tRemote: requests-00034-fa7ea2cb.gz\n",
+      "172.\tLocal: Yes\tRemote: requests-00035-41588475.gz\n",
+      "173.\tLocal: Yes\tRemote: requests-00000-9f6389f7.gz\n",
+      "174.\tLocal: Yes\tRemote: requests-00001-14470a89.gz\n",
+      "175.\tLocal: Yes\tRemote: requests-00002-454f24fc.gz\n",
+      "176.\tLocal: Yes\tRemote: requests-00003-6c8ef0ac.gz\n",
+      "177.\tLocal: Yes\tRemote: requests-00004-c826ed1b.gz\n",
+      "178.\tLocal: Yes\tRemote: requests-00005-c2263599.gz\n",
+      "179.\tLocal: Yes\tRemote: requests-00006-062102ef.gz\n",
+      "180.\tLocal: Yes\tRemote: requests-00007-bd31c799.gz\n",
+      "181.\tLocal: Yes\tRemote: requests-00008-7f720852.gz\n",
+      "182.\tLocal: Yes\tRemote: requests-00009-e7b243fc.gz\n",
+      "183.\tLocal: Yes\tRemote: requests-00010-56b4920f.gz\n",
+      "184.\tLocal: Yes\tRemote: requests-00011-0b89a399.gz\n",
+      "185.\tLocal: Yes\tRemote: requests-00012-6e12b8cf.gz\n",
+      "186.\tLocal: Yes\tRemote: requests-00013-1229c23e.gz\n",
+      "187.\tLocal: Yes\tRemote: requests-00014-9c6aa214.gz\n",
+      "188.\tLocal: Yes\tRemote: requests-00015-ff5d6064.gz\n",
+      "189.\tLocal: Yes\tRemote: requests-00016-b61b2eba.gz\n",
+      "190.\tLocal: Yes\tRemote: requests-00017-0a63eb19.gz\n",
+      "191.\tLocal: Yes\tRemote: requests-00018-0a575cb3.gz\n",
+      "192.\tLocal: Yes\tRemote: requests-00019-71221fc7.gz\n",
+      "193.\tLocal: Yes\tRemote: requests-00020-2d482e48.gz\n",
+      "194.\tLocal: Yes\tRemote: requests-00021-b3069fd3.gz\n",
+      "195.\tLocal: Yes\tRemote: requests-00022-8eea0a09.gz\n",
+      "196.\tLocal: Yes\tRemote: requests-00023-ac653335.gz\n",
+      "197.\tLocal: Yes\tRemote: requests-00024-4e12397c.gz\n",
+      "198.\tLocal: Yes\tRemote: requests-00025-9f953d90.gz\n",
+      "199.\tLocal: Yes\tRemote: requests-00026-20494cbe.gz\n",
+      "200.\tLocal: Yes\tRemote: requests-00027-269b81c4.gz\n",
+      "201.\tLocal: Yes\tRemote: requests-00028-e54def28.gz\n",
+      "202.\tLocal: Yes\tRemote: requests-00029-040e684f.gz\n",
+      "203.\tLocal: Yes\tRemote: requests-00030-8e8b61e6.gz\n",
+      "204.\tLocal: Yes\tRemote: requests-00031-335f860e.gz\n",
+      "205.\tLocal: Yes\tRemote: requests-00032-19f8afbb.gz\n",
+      "206.\tLocal: Yes\tRemote: requests-00033-191a285f.gz\n",
+      "207.\tLocal: Yes\tRemote: requests-00034-d48ee544.gz\n",
+      "208.\tLocal: Yes\tRemote: requests-00035-d0a79b90.gz\n",
+      "209.\tLocal: Yes\tRemote: requests-00036-345bd7f2.gz\n",
+      "210.\tLocal: Yes\tRemote: requests-00037-ef6ca8e2.gz\n",
+      "211.\tLocal: Yes\tRemote: requests-00038-af387aca.gz\n",
+      "212.\tLocal: Yes\tRemote: requests-00039-4e80aaba.gz\n",
+      "213.\tLocal: Yes\tRemote: requests-00040-b4af9795.gz\n",
+      "214.\tLocal: Yes\tRemote: requests-00041-ddd0c206.gz\n",
+      "215.\tLocal: Yes\tRemote: requests-00042-a50caf3b.gz\n",
+      "216.\tLocal: Yes\tRemote: requests-00043-70975754.gz\n",
+      "217.\tLocal: Yes\tRemote: requests-00044-f11697d4.gz\n",
+      "218.\tLocal: Yes\tRemote: requests-00045-1365c170.gz\n",
+      "219.\tLocal: Yes\tRemote: requests-00046-cce13fe4.gz\n",
+      "220.\tLocal: Yes\tRemote: requests-00047-0d9a991d.gz\n",
+      "221.\tLocal: Yes\tRemote: requests-00048-0e4f34eb.gz\n",
+      "222.\tLocal: Yes\tRemote: requests-00049-de813df6.gz\n",
+      "223.\tLocal: Yes\tRemote: requests-00050-0ac6e79c.gz\n",
+      "224.\tLocal: Yes\tRemote: requests-00051-a32f9e03.gz\n",
+      "225.\tLocal: Yes\tRemote: requests-00052-0763e23f.gz\n",
+      "226.\tLocal: Yes\tRemote: requests-00053-d4ea6b5c.gz\n",
+      "227.\tLocal: Yes\tRemote: requests-00054-8a904b77.gz\n",
+      "228.\tLocal: Yes\tRemote: requests-00055-bd3c7d48.gz\n",
+      "229.\tLocal: Yes\tRemote: requests-00056-6c0d5cdd.gz\n",
+      "230.\tLocal: Yes\tRemote: requests-00057-cb9e472d.gz\n",
+      "231.\tLocal: Yes\tRemote: requests-00058-71406f22.gz\n",
+      "232.\tLocal: Yes\tRemote: requests-00059-451f0901.gz\n",
+      "233.\tLocal: Yes\tRemote: requests-00060-3817bba3.gz\n",
+      "234.\tLocal: Yes\tRemote: requests-00061-ca694ace.gz\n",
+      "235.\tLocal: Yes\tRemote: requests-00062-59c90b35.gz\n",
+      "236.\tLocal: Yes\tRemote: requests-00063-8e48581e.gz\n",
+      "237.\tLocal: Yes\tRemote: requests-00064-2bdfbb81.gz\n",
+      "238.\tLocal: Yes\tRemote: requests-00065-106ecc41.gz\n",
+      "239.\tLocal: Yes\tRemote: requests-00066-b4a9ec32.gz\n",
+      "240.\tLocal: Yes\tRemote: requests-00067-a8475a17.gz\n",
+      "241.\tLocal: Yes\tRemote: requests-00068-40409ef7.gz\n",
+      "242.\tLocal: Yes\tRemote: requests-00069-9815d51c.gz\n",
+      "243.\tLocal: Yes\tRemote: requests-00070-3b799ae7.gz\n",
+      "244.\tLocal: Yes\tRemote: requests-00071-e8d779d0.gz\n",
+      "245.\tLocal: Yes\tRemote: requests-00072-1c7d07d7.gz\n",
+      "246.\tLocal: Yes\tRemote: requests-00073-80e92e4f.gz\n",
+      "247.\tLocal: Yes\tRemote: requests-00074-5e7b3fa1.gz\n",
+      "248.\tLocal: Yes\tRemote: requests-00075-1a3ba30a.gz\n",
+      "249.\tLocal: Yes\tRemote: requests-00076-a10d9889.gz\n",
+      "250.\tLocal: Yes\tRemote: requests-00077-05190bb4.gz\n",
+      "251.\tLocal: Yes\tRemote: requests-00078-8411186a.gz\n",
+      "252.\tLocal: Yes\tRemote: requests-00079-d606199b.gz\n",
+      "253.\tLocal: Yes\tRemote: requests-00080-a1e6c1b5.gz\n",
+      "254.\tLocal: Yes\tRemote: requests-00081-4892f2c3.gz\n",
+      "255.\tLocal: Yes\tRemote: requests-00082-3171a8ff.gz\n",
+      "256.\tLocal: Yes\tRemote: requests-00083-c8e21587.gz\n",
+      "257.\tLocal: Yes\tRemote: requests-00084-05f9f546.gz\n",
+      "258.\tLocal: Yes\tRemote: requests-00085-ae6f3390.gz\n",
+      "259.\tLocal: Yes\tRemote: requests-00086-f84303cf.gz\n",
+      "260.\tLocal: Yes\tRemote: requests-00087-314eee0d.gz\n",
+      "261.\tLocal: Yes\tRemote: requests-00088-173be602.gz\n",
+      "262.\tLocal: Yes\tRemote: requests-00089-06c26ad8.gz\n",
+      "263.\tLocal: Yes\tRemote: requests-00090-5f6187c1.gz\n",
+      "264.\tLocal: Yes\tRemote: requests-00091-85226755.gz\n",
+      "265.\tLocal: Yes\tRemote: requests-00092-0d5a262e.gz\n",
+      "266.\tLocal: Yes\tRemote: requests-00093-98cd8d18.gz\n",
+      "267.\tLocal: Yes\tRemote: requests-00094-13d7aafd.gz\n",
+      "268.\tLocal: Yes\tRemote: requests-00095-92a12c52.gz\n",
+      "269.\tLocal: Yes\tRemote: requests-00096-5959ae67.gz\n",
+      "270.\tLocal: Yes\tRemote: requests-00097-1d7e48d4.gz\n",
+      "271.\tLocal: Yes\tRemote: requests-00098-29a1fac5.gz\n",
+      "272.\tLocal: Yes\tRemote: requests-00099-643f4bd0.gz\n",
+      "273.\tLocal: Yes\tRemote: requests-00100-4f3e7cd6.gz\n",
+      "274.\tLocal: Yes\tRemote: requests-00101-e7f5802a.gz\n",
+      "275.\tLocal: Yes\tRemote: requests-00102-689dd503.gz\n",
+      "276.\tLocal: Yes\tRemote: requests-00103-89fd08a8.gz\n",
+      "277.\tLocal: Yes\tRemote: requests-00104-7a066ff5.gz\n",
+      "278.\tLocal: Yes\tRemote: requests-00105-7b321ff4.gz\n",
+      "279.\tLocal: Yes\tRemote: requests-00106-b3709d01.gz\n",
+      "280.\tLocal: Yes\tRemote: requests-00107-2b1dcc22.gz\n",
+      "281.\tLocal: Yes\tRemote: requests-00108-2752aefc.gz\n",
+      "282.\tLocal: Yes\tRemote: requests-00109-12e94d3a.gz\n",
+      "283.\tLocal: Yes\tRemote: requests-00110-02b732db.gz\n",
+      "284.\tLocal: Yes\tRemote: requests-00111-df1d205d.gz\n",
+      "285.\tLocal: Yes\tRemote: requests-00112-ce4aca0c.gz\n",
+      "286.\tLocal: Yes\tRemote: requests-00113-4a5c2730.gz\n",
+      "287.\tLocal: Yes\tRemote: requests-00114-df499f04.gz\n",
+      "288.\tLocal: Yes\tRemote: requests-00115-6f190643.gz\n",
+      "289.\tLocal: Yes\tRemote: requests-00116-99f25a77.gz\n",
+      "290.\tLocal: Yes\tRemote: requests-00117-67f23ea3.gz\n",
+      "291.\tLocal: Yes\tRemote: requests-00118-b21de634.gz\n",
+      "292.\tLocal: Yes\tRemote: requests-00119-b9b186fb.gz\n",
+      "293.\tLocal: Yes\tRemote: requests-00120-d36b8377.gz\n",
+      "294.\tLocal: Yes\tRemote: requests-00121-e12b23a5.gz\n",
+      "295.\tLocal: Yes\tRemote: requests-00122-882b6df6.gz\n",
+      "296.\tLocal: Yes\tRemote: requests-00123-bcea9bd5.gz\n",
+      "297.\tLocal: Yes\tRemote: requests-00124-f087b451.gz\n",
+      "298.\tLocal: Yes\tRemote: requests-00125-38f8d26c.gz\n",
+      "299.\tLocal: Yes\tRemote: requests-00126-ee99a33e.gz\n",
+      "300.\tLocal: Yes\tRemote: requests-00127-f7c8cb58.gz\n",
+      "301.\tLocal: Yes\tRemote: requests-00128-035e81c9.gz\n",
+      "302.\tLocal: Yes\tRemote: requests-00129-9931568e.gz\n",
+      "303.\tLocal: Yes\tRemote: requests-00130-ff1d3d1b.gz\n",
+      "304.\tLocal: Yes\tRemote: requests-00131-73641c71.gz\n",
+      "305.\tLocal: Yes\tRemote: requests-00132-e7f8f0c1.gz\n",
+      "306.\tLocal: Yes\tRemote: requests-00133-32a91904.gz\n",
+      "307.\tLocal: Yes\tRemote: requests-00134-cdfd29bd.gz\n",
+      "308.\tLocal: Yes\tRemote: requests-00135-526a2306.gz\n",
+      "309.\tLocal: Yes\tRemote: requests-00136-90e91c51.gz\n",
+      "310.\tLocal: Yes\tRemote: requests-00137-543244db.gz\n",
+      "311.\tLocal: Yes\tRemote: requests-00138-872784d7.gz\n",
+      "312.\tLocal: Yes\tRemote: requests-00139-fcd0a963.gz\n",
+      "313.\tLocal: Yes\tRemote: requests-00140-7849378d.gz\n",
+      "314.\tLocal: Yes\tRemote: requests-00141-e917b105.gz\n",
+      "315.\tLocal: Yes\tRemote: requests-00142-f489d288.gz\n",
+      "316.\tLocal: Yes\tRemote: requests-00143-8b0c6850.gz\n",
+      "317.\tLocal: Yes\tRemote: requests-00144-2e948c95.gz\n",
+      "318.\tLocal: Yes\tRemote: requests-00145-efadbb83.gz\n",
+      "319.\tLocal: Yes\tRemote: requests-00146-01e1d7de.gz\n",
+      "320.\tLocal: Yes\tRemote: requests-00147-2e31c5e9.gz\n",
+      "321.\tLocal: Yes\tRemote: requests-00148-4ea3f910.gz\n",
+      "322.\tLocal: Yes\tRemote: requests-00149-a9c96dd2.gz\n",
+      "323.\tLocal: Yes\tRemote: requests-00150-d8fc85e1.gz\n",
+      "324.\tLocal: Yes\tRemote: requests-00151-1d73895c.gz\n",
+      "325.\tLocal: Yes\tRemote: requests-00152-e5dc9a13.gz\n",
+      "326.\tLocal: Yes\tRemote: requests-00153-3781c23a.gz\n",
+      "327.\tLocal: Yes\tRemote: requests-00154-908b4cdb.gz\n",
+      "328.\tLocal: Yes\tRemote: requests-00155-8b50ad6d.gz\n",
+      "329.\tLocal: Yes\tRemote: requests-00156-476ad9b3.gz\n",
+      "330.\tLocal: Yes\tRemote: requests-00157-3b19a82d.gz\n",
+      "331.\tLocal: Yes\tRemote: requests-00158-ad546750.gz\n",
+      "332.\tLocal: Yes\tRemote: requests-00159-5fa72f8a.gz\n",
+      "333.\tLocal: Yes\tRemote: requests-00160-17862eea.gz\n",
+      "334.\tLocal: Yes\tRemote: requests-00161-c6d6e8cc.gz\n",
+      "335.\tLocal: Yes\tRemote: requests-00162-3114202b.gz\n",
+      "336.\tLocal: Yes\tRemote: requests-00163-a419a0a6.gz\n",
+      "337.\tLocal: Yes\tRemote: requests-00164-12c6acc7.gz\n",
+      "338.\tLocal: Yes\tRemote: requests-00165-48b7b0d0.gz\n",
+      "339.\tLocal: Yes\tRemote: requests-00166-dcc758cb.gz\n",
+      "340.\tLocal: Yes\tRemote: requests-00167-6084265f.gz\n",
+      "341.\tLocal: Yes\tRemote: requests-00168-fd58b1bd.gz\n",
+      "342.\tLocal: Yes\tRemote: requests-00169-d1b9994a.gz\n",
+      "343.\tLocal: Yes\tRemote: requests-00170-851058de.gz\n",
+      "344.\tLocal: Yes\tRemote: requests-00171-3057dc31.gz\n",
+      "345.\tLocal: Yes\tRemote: requests-00172-9344c8c3.gz\n",
+      "346.\tLocal: Yes\tRemote: requests-00173-30203835.gz\n",
+      "347.\tLocal: Yes\tRemote: requests-00174-6a4a6fdd.gz\n",
+      "348.\tLocal: Yes\tRemote: requests-00175-d93e9288.gz\n",
+      "349.\tLocal: Yes\tRemote: requests-00176-87840acb.gz\n",
+      "350.\tLocal: Yes\tRemote: requests-00177-82112004.gz\n",
+      "351.\tLocal: Yes\tRemote: requests-00178-2552486c.gz\n",
+      "352.\tLocal: Yes\tRemote: requests-00179-65570ea8.gz\n",
+      "353.\tLocal: Yes\tRemote: requests-00180-6a714641.gz\n",
+      "354.\tLocal: Yes\tRemote: requests-00181-a96310e2.gz\n",
+      "355.\tLocal: Yes\tRemote: requests-00182-a965c011.gz\n",
+      "356.\tLocal: Yes\tRemote: requests-00183-234df5a3.gz\n",
+      "357.\tLocal: Yes\tRemote: requests-00184-dd312f8e.gz\n",
+      "358.\tLocal: Yes\tRemote: requests-00185-dd3de567.gz\n",
+      "359.\tLocal: Yes\tRemote: requests-00186-bd8801a2.gz\n",
+      "360.\tLocal: Yes\tRemote: requests-00187-31f9f6b2.gz\n",
+      "361.\tLocal: Yes\tRemote: requests-00188-f0cb866c.gz\n",
+      "362.\tLocal: Yes\tRemote: requests-00189-715c6a6c.gz\n",
+      "363.\tLocal: Yes\tRemote: requests-00190-f00fd4a1.gz\n",
+      "364.\tLocal: Yes\tRemote: requests-00191-0ec02ac7.gz\n",
+      "365.\tLocal: Yes\tRemote: requests-00192-a8caaeea.gz\n",
+      "366.\tLocal: Yes\tRemote: requests-00193-fa50817a.gz\n",
+      "367.\tLocal: Yes\tRemote: requests-00194-96175dcc.gz\n",
+      "368.\tLocal: Yes\tRemote: requests-00195-293f8461.gz\n",
+      "369.\tLocal: Yes\tRemote: requests-00196-101ce99b.gz\n",
+      "370.\tLocal: Yes\tRemote: requests-00197-8d3f2c38.gz\n",
+      "371.\tLocal: Yes\tRemote: requests-00198-e27e5973.gz\n",
+      "372.\tLocal: Yes\tRemote: requests-00199-f2e1f75c.gz\n",
+      "373.\tLocal: Yes\tRemote: requests-00200-857c38a4.gz\n",
+      "374.\tLocal: Yes\tRemote: requests-00201-666bf629.gz\n",
+      "375.\tLocal: Yes\tRemote: requests-00202-3f665fc0.gz\n",
+      "376.\tLocal: Yes\tRemote: requests-00203-866b8d63.gz\n",
+      "377.\tLocal: Yes\tRemote: requests-00204-8d53c8d9.gz\n",
+      "378.\tLocal: Yes\tRemote: requests-00205-5310b7dc.gz\n",
+      "379.\tLocal: Yes\tRemote: requests-00206-15f1b7f8.gz\n",
+      "380.\tLocal: Yes\tRemote: requests-00207-9f637230.gz\n",
+      "381.\tLocal: Yes\tRemote: requests-00208-94425ac6.gz\n",
+      "382.\tLocal: Yes\tRemote: requests-00209-d55a3ce1.gz\n",
+      "383.\tLocal: Yes\tRemote: requests-00210-a46bbe09.gz\n",
+      "384.\tLocal: Yes\tRemote: requests-00211-58fb724e.gz\n",
+      "385.\tLocal: Yes\tRemote: requests-00212-2aec7198.gz\n",
+      "386.\tLocal: Yes\tRemote: requests-00213-8134f568.gz\n",
+      "387.\tLocal: Yes\tRemote: requests-00214-ac158d91.gz\n",
+      "388.\tLocal: Yes\tRemote: requests-00215-49f9bc26.gz\n",
+      "389.\tLocal: Yes\tRemote: requests-00216-eced3806.gz\n",
+      "390.\tLocal: Yes\tRemote: requests-00217-63eee3f7.gz\n",
+      "391.\tLocal: Yes\tRemote: requests-00218-760124c5.gz\n",
+      "392.\tLocal: Yes\tRemote: requests-00219-dfcb20ee.gz\n",
+      "393.\tLocal: Yes\tRemote: requests-00220-e3369b16.gz\n",
+      "394.\tLocal: Yes\tRemote: requests-00221-61b009fc.gz\n",
+      "395.\tLocal: Yes\tRemote: requests-00222-0f64b099.gz\n",
+      "396.\tLocal: Yes\tRemote: requests-00223-3f6d3a26.gz\n",
+      "397.\tLocal: Yes\tRemote: requests-00224-c0266749.gz\n",
+      "398.\tLocal: Yes\tRemote: requests-00225-5ffdd567.gz\n",
+      "399.\tLocal: Yes\tRemote: requests-00226-55ecac63.gz\n",
+      "400.\tLocal: Yes\tRemote: requests-00227-1eec974b.gz\n",
+      "401.\tLocal: Yes\tRemote: requests-00228-eb9b2433.gz\n",
+      "402.\tLocal: Yes\tRemote: requests-00229-ceee7c27.gz\n",
+      "403.\tLocal: Yes\tRemote: requests-00230-b63e7db5.gz\n",
+      "404.\tLocal: Yes\tRemote: requests-00231-fcc1a65a.gz\n",
+      "405.\tLocal: Yes\tRemote: requests-00232-53a79ff6.gz\n",
+      "406.\tLocal: Yes\tRemote: requests-00233-84b802c3.gz\n",
+      "407.\tLocal: Yes\tRemote: requests-00234-5bfbf33d.gz\n",
+      "408.\tLocal: Yes\tRemote: requests-00235-f9df1c80.gz\n",
+      "409.\tLocal: Yes\tRemote: requests-00236-83c68d74.gz\n",
+      "410.\tLocal: Yes\tRemote: requests-00237-5e4cc792.gz\n",
+      "411.\tLocal: Yes\tRemote: requests-00238-f8e61a96.gz\n",
+      "412.\tLocal: Yes\tRemote: requests-00239-f95865e8.gz\n",
+      "413.\tLocal: Yes\tRemote: requests-00240-e47ea374.gz\n",
+      "414.\tLocal: Yes\tRemote: requests-00241-a43ae778.gz\n",
+      "415.\tLocal: Yes\tRemote: requests-00242-66fcb04a.gz\n",
+      "416.\tLocal: Yes\tRemote: requests-00243-b6bf76b2.gz\n",
+      "417.\tLocal: Yes\tRemote: requests-00244-bf8e20b7.gz\n",
+      "418.\tLocal: Yes\tRemote: requests-00245-fad687e1.gz\n",
+      "419.\tLocal: Yes\tRemote: requests-00246-e083a683.gz\n",
+      "420.\tLocal: Yes\tRemote: requests-00247-27de1377.gz\n",
+      "421.\tLocal: Yes\tRemote: requests-00248-165cdfb8.gz\n",
+      "422.\tLocal: Yes\tRemote: requests-00249-ab20bd64.gz\n",
+      "423.\tLocal: Yes\tRemote: requests-00250-000b90ae.gz\n",
+      "424.\tLocal: Yes\tRemote: requests-00251-6cfe157e.gz\n",
+      "425.\tLocal: Yes\tRemote: requests-00252-ad487020.gz\n",
+      "426.\tLocal: Yes\tRemote: requests-00253-7c053852.gz\n",
+      "427.\tLocal: Yes\tRemote: requests-00254-4a50421c.gz\n",
+      "428.\tLocal: Yes\tRemote: requests-00255-81e0c59f.gz\n",
+      "429.\tLocal: Yes\tRemote: requests-00256-ce55913c.gz\n",
+      "430.\tLocal: Yes\tRemote: requests-00257-e5aaa79f.gz\n",
+      "431.\tLocal: Yes\tRemote: requests-00258-27b0ccf1.gz\n",
+      "432.\tLocal: Yes\tRemote: requests-00259-78e494fe.gz\n",
+      "433.\tLocal: Yes\tRemote: requests-00260-70b7affc.gz\n",
+      "434.\tLocal: Yes\tRemote: requests-00261-53e7857b.gz\n",
+      "435.\tLocal: Yes\tRemote: requests-00262-ad5da50d.gz\n",
+      "436.\tLocal: Yes\tRemote: requests-00263-9dc8715f.gz\n",
+      "437.\tLocal: Yes\tRemote: requests-00264-cde1f076.gz\n",
+      "438.\tLocal: Yes\tRemote: requests-00265-7f593bac.gz\n",
+      "439.\tLocal: Yes\tRemote: requests-00266-8973cf13.gz\n",
+      "440.\tLocal: Yes\tRemote: requests-00267-6f71143f.gz\n",
+      "441.\tLocal: Yes\tRemote: requests-00268-60dcb3a9.gz\n",
+      "442.\tLocal: Yes\tRemote: requests-00269-8bcf6614.gz\n",
+      "443.\tLocal: Yes\tRemote: requests-00270-ad670a61.gz\n",
+      "444.\tLocal: Yes\tRemote: requests-00271-7230455c.gz\n",
+      "445.\tLocal: Yes\tRemote: requests-00272-e77e2834.gz\n",
+      "446.\tLocal: Yes\tRemote: requests-00273-e5cc2241.gz\n",
+      "447.\tLocal: Yes\tRemote: requests-00274-b49e0feb.gz\n",
+      "448.\tLocal: Yes\tRemote: requests-00275-6611ef9d.gz\n",
+      "449.\tLocal: Yes\tRemote: requests-00276-c43254f2.gz\n",
+      "450.\tLocal: Yes\tRemote: requests-00277-aee057da.gz\n",
+      "451.\tLocal: Yes\tRemote: requests-00278-7383d9fb.gz\n",
+      "452.\tLocal: Yes\tRemote: requests-00279-828d977a.gz\n",
+      "453.\tLocal: Yes\tRemote: requests-00280-44b047f0.gz\n",
+      "454.\tLocal: Yes\tRemote: requests-00281-4726914d.gz\n",
+      "455.\tLocal: Yes\tRemote: requests-00282-4bb82add.gz\n",
+      "456.\tLocal: Yes\tRemote: requests-00283-63f89f10.gz\n",
+      "457.\tLocal: Yes\tRemote: requests-00284-eee7063c.gz\n",
+      "458.\tLocal: Yes\tRemote: requests-00285-b10c0b70.gz\n",
+      "459.\tLocal: Yes\tRemote: requests-00286-d04a043f.gz\n",
+      "460.\tLocal: Yes\tRemote: requests-00287-27bf7ba5.gz\n",
+      "461.\tLocal: Yes\tRemote: requests-00288-a47934bd.gz\n",
+      "462.\tLocal: Yes\tRemote: requests-00289-aa7d99ce.gz\n",
+      "463.\tLocal: Yes\tRemote: requests-00290-d291a0af.gz\n",
+      "464.\tLocal: Yes\tRemote: requests-00291-bc916f2d.gz\n",
+      "465.\tLocal: Yes\tRemote: requests-00292-5869d044.gz\n",
+      "466.\tLocal: Yes\tRemote: requests-00293-c35a31b6.gz\n",
+      "467.\tLocal: Yes\tRemote: requests-00294-d065de31.gz\n",
+      "468.\tLocal: Yes\tRemote: requests-00295-67d0e1f6.gz\n",
+      "469.\tLocal: Yes\tRemote: requests-00296-8a3c61ea.gz\n",
+      "470.\tLocal: Yes\tRemote: requests-00297-8db33309.gz\n",
+      "471.\tLocal: Yes\tRemote: requests-00298-12a4d2c3.gz\n",
+      "472.\tLocal: Yes\tRemote: requests-00299-0ce766a8.gz\n",
+      "473.\tLocal: Yes\tRemote: requests-00300-b7d3f82d.gz\n",
+      "474.\tLocal: Yes\tRemote: requests-00301-cce7da47.gz\n",
+      "475.\tLocal: Yes\tRemote: requests-00302-99b64b15.gz\n",
+      "476.\tLocal: Yes\tRemote: requests-00303-b7139f07.gz\n",
+      "477.\tLocal: Yes\tRemote: requests-00304-d0b8585f.gz\n",
+      "478.\tLocal: Yes\tRemote: requests-00305-61afdcbb.gz\n",
+      "479.\tLocal: Yes\tRemote: requests-00306-aad4a9f8.gz\n",
+      "480.\tLocal: Yes\tRemote: requests-00307-7da6b0e0.gz\n",
+      "481.\tLocal: Yes\tRemote: requests-00308-eec4ce62.gz\n",
+      "482.\tLocal: Yes\tRemote: requests-00309-f2198660.gz\n",
+      "483.\tLocal: Yes\tRemote: requests-00310-793f5b28.gz\n",
+      "484.\tLocal: Yes\tRemote: requests-00311-1b72491d.gz\n",
+      "485.\tLocal: Yes\tRemote: requests-00312-245b5a43.gz\n",
+      "486.\tLocal: Yes\tRemote: requests-00313-77a72f18.gz\n",
+      "487.\tLocal: Yes\tRemote: requests-00314-02841e3a.gz\n",
+      "488.\tLocal: Yes\tRemote: requests-00315-00d85a16.gz\n",
+      "489.\tLocal: Yes\tRemote: requests-00316-97f6683b.gz\n",
+      "490.\tLocal: Yes\tRemote: requests-00317-e5154ae3.gz\n",
+      "491.\tLocal: Yes\tRemote: requests-00318-62ba0516.gz\n",
+      "492.\tLocal: Yes\tRemote: requests-00319-925a14e2.gz\n",
+      "493.\tLocal: Yes\tRemote: requests-00320-9673fe8c.gz\n",
+      "494.\tLocal: Yes\tRemote: requests-00321-093d79fa.gz\n",
+      "495.\tLocal: Yes\tRemote: requests-00322-ab1b485a.gz\n",
+      "496.\tLocal: Yes\tRemote: requests-00323-6f5e452e.gz\n",
+      "497.\tLocal: Yes\tRemote: requests-00324-1deb3cce.gz\n",
+      "498.\tLocal: Yes\tRemote: requests-00325-be2c88b2.gz\n",
+      "499.\tLocal: Yes\tRemote: requests-00326-7c7315a9.gz\n",
+      "500.\tLocal: Yes\tRemote: requests-00327-15484b32.gz\n",
+      "501.\tLocal: Yes\tRemote: requests-00328-0a0c0ead.gz\n",
+      "502.\tLocal: Yes\tRemote: requests-00329-4be093d7.gz\n",
+      "503.\tLocal: Yes\tRemote: requests-00330-10456bf8.gz\n",
+      "504.\tLocal: Yes\tRemote: requests-00331-2fceba86.gz\n",
+      "505.\tLocal: Yes\tRemote: requests-00332-0c6933e7.gz\n",
+      "506.\tLocal: Yes\tRemote: requests-00333-e0a68e8a.gz\n",
+      "507.\tLocal: Yes\tRemote: requests-00334-da2a0280.gz\n",
+      "508.\tLocal: Yes\tRemote: requests-00335-5b8ec391.gz\n",
+      "509.\tLocal: Yes\tRemote: requests-00336-211b9f06.gz\n",
+      "510.\tLocal: Yes\tRemote: requests-00337-a6a2d70f.gz\n",
+      "511.\tLocal: Yes\tRemote: requests-00338-b14dd08c.gz\n",
+      "512.\tLocal: Yes\tRemote: requests-00339-a78480c4.gz\n",
+      "513.\tLocal: Yes\tRemote: requests-00340-38222b6f.gz\n",
+      "514.\tLocal: Yes\tRemote: requests-00341-5a3fd351.gz\n",
+      "515.\tLocal: Yes\tRemote: requests-00342-bac9cbd6.gz\n",
+      "516.\tLocal: Yes\tRemote: requests-00343-e239eb13.gz\n",
+      "517.\tLocal: Yes\tRemote: requests-00344-75608bcb.gz\n",
+      "518.\tLocal: Yes\tRemote: requests-00345-64fe4e70.gz\n",
+      "519.\tLocal: Yes\tRemote: requests-00346-6eb7e916.gz\n",
+      "520.\tLocal: Yes\tRemote: requests-00347-060da8d6.gz\n",
+      "521.\tLocal: Yes\tRemote: requests-00348-dda2340b.gz\n",
+      "522.\tLocal: Yes\tRemote: requests-00349-064d2aa5.gz\n",
+      "523.\tLocal: Yes\tRemote: requests-00350-e26fbad6.gz\n",
+      "524.\tLocal: Yes\tRemote: requests-00351-e1a57fb5.gz\n",
+      "525.\tLocal: Yes\tRemote: requests-00352-db5abaa2.gz\n",
+      "526.\tLocal: Yes\tRemote: requests-00353-63a0a134.gz\n",
+      "527.\tLocal: Yes\tRemote: requests-00354-ba648d9e.gz\n",
+      "528.\tLocal: Yes\tRemote: requests-00355-9e063d17.gz\n",
+      "529.\tLocal: Yes\tRemote: requests-00356-17e99ef5.gz\n",
+      "530.\tLocal: Yes\tRemote: requests-00357-3ab93f26.gz\n",
+      "531.\tLocal: Yes\tRemote: requests-00358-921167b5.gz\n",
+      "532.\tLocal: Yes\tRemote: requests-00000-48027597.gz\n",
+      "533.\tLocal: Yes\tRemote: requests-00001-3e78c0f6.gz\n",
+      "534.\tLocal: Yes\tRemote: requests-00002-202e9d88.gz\n",
+      "535.\tLocal: Yes\tRemote: requests-00003-d1ac8cfc.gz\n",
+      "536.\tLocal: Yes\tRemote: requests-00004-3b24230a.gz\n",
+      "537.\tLocal: Yes\tRemote: requests-00005-5ca8ff34.gz\n",
+      "538.\tLocal: Yes\tRemote: requests-00006-299ed827.gz\n",
+      "539.\tLocal: Yes\tRemote: requests-00007-59652777.gz\n",
+      "540.\tLocal: Yes\tRemote: requests-00008-8b3e7ad7.gz\n",
+      "541.\tLocal: Yes\tRemote: requests-00009-c63bff74.gz\n",
+      "542.\tLocal: Yes\tRemote: requests-00010-f74e02d0.gz\n",
+      "543.\tLocal: Yes\tRemote: requests-00011-de6ed953.gz\n",
+      "544.\tLocal: Yes\tRemote: requests-00012-cbba2ab2.gz\n",
+      "545.\tLocal: Yes\tRemote: requests-00013-1053a397.gz\n",
+      "546.\tLocal: Yes\tRemote: requests-00014-b2e52a95.gz\n",
+      "547.\tLocal: Yes\tRemote: requests-00015-6a0622e7.gz\n",
+      "548.\tLocal: Yes\tRemote: requests-00016-e0839294.gz\n",
+      "549.\tLocal: Yes\tRemote: requests-00017-ae621a1b.gz\n",
+      "550.\tLocal: Yes\tRemote: requests-00018-070cb58f.gz\n",
+      "551.\tLocal: Yes\tRemote: requests-00019-286977f5.gz\n",
+      "552.\tLocal: Yes\tRemote: requests-00020-35c5a50a.gz\n",
+      "553.\tLocal: Yes\tRemote: requests-00021-955a5d93.gz\n",
+      "554.\tLocal: Yes\tRemote: requests-00022-0476f98a.gz\n",
+      "555.\tLocal: Yes\tRemote: requests-00023-be9013b2.gz\n",
+      "556.\tLocal: Yes\tRemote: requests-00024-a48c55bc.gz\n",
+      "557.\tLocal: Yes\tRemote: requests-00025-7599e3b0.gz\n",
+      "558.\tLocal: Yes\tRemote: requests-00026-712c6a37.gz\n",
+      "559.\tLocal: Yes\tRemote: requests-00027-18b31f26.gz\n",
+      "560.\tLocal: Yes\tRemote: requests-00000-d1e690c0.gz\n",
+      "561.\tLocal: Yes\tRemote: requests-00001-9fbd0a0a.gz\n",
+      "562.\tLocal: Yes\tRemote: requests-00002-44e46e23.gz\n",
+      "563.\tLocal: Yes\tRemote: requests-00003-448f8a4a.gz\n",
+      "564.\tLocal: Yes\tRemote: requests-00004-af3b8b3d.gz\n",
+      "565.\tLocal: Yes\tRemote: requests-00005-b9b9ea41.gz\n",
+      "566.\tLocal: Yes\tRemote: requests-00006-f17303de.gz\n",
+      "567.\tLocal: Yes\tRemote: requests-00007-8450e36d.gz\n",
+      "568.\tLocal: Yes\tRemote: requests-00008-66fa5deb.gz\n",
+      "569.\tLocal: Yes\tRemote: requests-00009-d5436412.gz\n",
+      "570.\tLocal: Yes\tRemote: requests-00010-e722e8b8.gz\n",
+      "571.\tLocal: Yes\tRemote: requests-00011-e4ebe14e.gz\n",
+      "572.\tLocal: Yes\tRemote: requests-00012-6dec07a7.gz\n",
+      "573.\tLocal: Yes\tRemote: requests-00013-ed53a319.gz\n",
+      "574.\tLocal: Yes\tRemote: requests-00014-ac3943d0.gz\n",
+      "575.\tLocal: Yes\tRemote: requests-00015-00e65d2c.gz\n",
+      "576.\tLocal: Yes\tRemote: requests-00016-0f8980df.gz\n",
+      "577.\tLocal: Yes\tRemote: requests-00017-3c4a9f16.gz\n",
+      "578.\tLocal: Yes\tRemote: requests-00018-d88372c0.gz\n",
+      "579.\tLocal: Yes\tRemote: requests-00019-0f15839f.gz\n",
+      "580.\tLocal: Yes\tRemote: requests-00020-97006bd8.gz\n",
+      "581.\tLocal: Yes\tRemote: requests-00021-6363cd6e.gz\n",
+      "582.\tLocal: Yes\tRemote: requests-00022-84a95ec0.gz\n",
+      "583.\tLocal: Yes\tRemote: requests-00023-3f6ae2fa.gz\n",
+      "584.\tLocal: Yes\tRemote: requests-00024-8a7ebecb.gz\n",
+      "585.\tLocal: Yes\tRemote: requests-00025-146145dc.gz\n",
+      "586.\tLocal: Yes\tRemote: requests-00026-dcbe9e21.gz\n",
+      "587.\tLocal: Yes\tRemote: requests-00000-492144e6.gz\n",
+      "588.\tLocal: Yes\tRemote: requests-00001-229aaa9a.gz\n",
+      "589.\tLocal: Yes\tRemote: requests-00002-13749a80.gz\n",
+      "590.\tLocal: Yes\tRemote: requests-00003-7dc240b8.gz\n",
+      "591.\tLocal: Yes\tRemote: requests-00004-9c2b9662.gz\n",
+      "592.\tLocal: Yes\tRemote: requests-00005-779ab454.gz\n",
+      "593.\tLocal: Yes\tRemote: requests-00006-72e4e422.gz\n",
+      "594.\tLocal: Yes\tRemote: requests-00007-de8cdd80.gz\n",
+      "595.\tLocal: Yes\tRemote: requests-00008-a0850229.gz\n",
+      "596.\tLocal: Yes\tRemote: requests-00009-b6c6feb0.gz\n",
+      "597.\tLocal: Yes\tRemote: requests-00010-7eeed98a.gz\n",
+      "598.\tLocal: Yes\tRemote: requests-00011-8e720e01.gz\n",
+      "599.\tLocal: Yes\tRemote: requests-00012-51379939.gz\n",
+      "600.\tLocal: Yes\tRemote: requests-00000-1bc2feed.gz\n",
+      "601.\tLocal: Yes\tRemote: requests-00001-1994483c.gz\n",
+      "602.\tLocal: Yes\tRemote: requests-00002-ce26b68d.gz\n",
+      "603.\tLocal: Yes\tRemote: requests-00003-0f0a2db4.gz\n",
+      "604.\tLocal: Yes\tRemote: requests-00004-bea256d9.gz\n",
+      "605.\tLocal: Yes\tRemote: requests-00005-7bc6e9eb.gz\n",
+      "606.\tLocal: Yes\tRemote: requests-00006-603e05e0.gz\n",
+      "607.\tLocal: Yes\tRemote: requests-00007-8cbb8c79.gz\n",
+      "608.\tLocal: Yes\tRemote: requests-00008-2471a75a.gz\n",
+      "609.\tLocal: Yes\tRemote: requests-00009-27de8b0d.gz\n",
+      "610.\tLocal: Yes\tRemote: requests-00010-bc056d87.gz\n",
+      "611.\tLocal: Yes\tRemote: requests-00011-29fc82ef.gz\n",
+      "612.\tLocal: Yes\tRemote: requests-00012-97574421.gz\n",
+      "613.\tLocal: Yes\tRemote: requests-00013-a757b9fa.gz\n",
+      "614.\tLocal: Yes\tRemote: requests-00014-793e7e90.gz\n",
+      "615.\tLocal: Yes\tRemote: requests-00015-ee04b64e.gz\n",
+      "616.\tLocal: Yes\tRemote: requests-00016-7bbaffdb.gz\n",
+      "617.\tLocal: Yes\tRemote: requests-00017-84dc3abc.gz\n",
+      "618.\tLocal: Yes\tRemote: requests-00018-3d03d287.gz\n",
+      "619.\tLocal: Yes\tRemote: requests-00019-904ce62a.gz\n",
+      "620.\tLocal: Yes\tRemote: requests-00020-613e5d8f.gz\n",
+      "621.\tLocal: Yes\tRemote: requests-00021-7f61205f.gz\n",
+      "622.\tLocal: Yes\tRemote: requests-00022-9bb667a2.gz\n",
+      "623.\tLocal: Yes\tRemote: requests-00023-b5b1de3a.gz\n",
+      "624.\tLocal: Yes\tRemote: requests-00024-8a22057f.gz\n",
+      "625.\tLocal: Yes\tRemote: requests-00025-e0da029c.gz\n",
+      "626.\tLocal: Yes\tRemote: requests-00026-fc629505.gz\n",
+      "627.\tLocal: Yes\tRemote: requests-00027-b0f98e0d.gz\n",
+      "628.\tLocal: Yes\tRemote: requests-00028-626588d4.gz\n",
+      "629.\tLocal: Yes\tRemote: requests-00029-68f4413b.gz\n",
+      "630.\tLocal: Yes\tRemote: requests-00030-f40e2b3c.gz\n",
+      "631.\tLocal: Yes\tRemote: requests-00031-7ad25492.gz\n",
+      "632.\tLocal: Yes\tRemote: requests-00032-b886ae2e.gz\n",
+      "633.\tLocal: Yes\tRemote: requests-00033-48fc1cd8.gz\n",
+      "634.\tLocal: Yes\tRemote: requests-00034-e1e3ceb8.gz\n",
+      "635.\tLocal: Yes\tRemote: requests-00035-21229f46.gz\n",
+      "636.\tLocal: Yes\tRemote: requests-00036-e14f3cc7.gz\n",
+      "637.\tLocal: Yes\tRemote: requests-00037-ff8f0334.gz\n",
+      "638.\tLocal: Yes\tRemote: requests-00038-e7aca4fa.gz\n",
+      "639.\tLocal: Yes\tRemote: requests-00039-24454eb2.gz\n",
+      "640.\tLocal: Yes\tRemote: requests-00000-20bd54e8.gz\n",
+      "641.\tLocal: Yes\tRemote: requests-00001-5b33ad70.gz\n",
+      "642.\tLocal: Yes\tRemote: requests-00002-ab4a910f.gz\n",
+      "643.\tLocal: Yes\tRemote: requests-00003-61d4a4ce.gz\n",
+      "644.\tLocal: Yes\tRemote: requests-00004-97b9ec9b.gz\n",
+      "645.\tLocal: Yes\tRemote: requests-00005-fa012852.gz\n",
+      "646.\tLocal: Yes\tRemote: requests-00006-4549a1c0.gz\n",
+      "647.\tLocal: Yes\tRemote: requests-00007-c40b86ac.gz\n",
+      "648.\tLocal: Yes\tRemote: requests-00008-8b708db3.gz\n",
+      "649.\tLocal: Yes\tRemote: requests-00009-12d2fc1d.gz\n",
+      "650.\tLocal: Yes\tRemote: requests-00010-eea3f8fb.gz\n",
+      "651.\tLocal: Yes\tRemote: requests-00011-8d312a5d.gz\n",
+      "652.\tLocal: Yes\tRemote: requests-00012-8ec0533a.gz\n",
+      "653.\tLocal: Yes\tRemote: requests-00013-3374a150.gz\n",
+      "654.\tLocal: Yes\tRemote: requests-00014-84b04f2f.gz\n",
+      "655.\tLocal: Yes\tRemote: requests-00015-b05ee1b8.gz\n",
+      "656.\tLocal: Yes\tRemote: requests-00016-bc241fe5.gz\n",
+      "657.\tLocal: Yes\tRemote: requests-00017-b7fa1cc6.gz\n",
+      "658.\tLocal: Yes\tRemote: requests-00018-a8929833.gz\n",
+      "659.\tLocal: Yes\tRemote: requests-00019-5d629451.gz\n",
+      "660.\tLocal: Yes\tRemote: requests-00020-f7491785.gz\n",
+      "661.\tLocal: Yes\tRemote: requests-00021-bf8fa8a0.gz\n",
+      "662.\tLocal: Yes\tRemote: requests-00022-d6480340.gz\n",
+      "663.\tLocal: Yes\tRemote: requests-00023-5495d187.gz\n",
+      "664.\tLocal: Yes\tRemote: requests-00000-9785ee73.gz\n",
+      "665.\tLocal: Yes\tRemote: requests-00001-76455c39.gz\n",
+      "666.\tLocal: Yes\tRemote: requests-00000-3a7a59c8.gz\n",
+      "667.\tLocal: Yes\tRemote: requests-00001-7458ced5.gz\n",
+      "668.\tLocal: Yes\tRemote: requests-00002-6b83854c.gz\n",
+      "669.\tLocal: Yes\tRemote: requests-00003-92b33d77.gz\n",
+      "670.\tLocal: Yes\tRemote: requests-00004-89e67bf4.gz\n",
+      "671.\tLocal: Yes\tRemote: requests-00005-59b58ef1.gz\n",
+      "672.\tLocal: Yes\tRemote: requests-00006-a1561c47.gz\n",
+      "673.\tLocal: Yes\tRemote: requests-00007-470c3455.gz\n",
+      "674.\tLocal: Yes\tRemote: requests-00008-5b1a601c.gz\n",
+      "675.\tLocal: Yes\tRemote: requests-00009-03d9770d.gz\n",
+      "676.\tLocal: Yes\tRemote: requests-00010-311f0cd7.gz\n",
+      "677.\tLocal: Yes\tRemote: requests-00011-7cf37ae2.gz\n",
+      "678.\tLocal: Yes\tRemote: requests-00012-2d6a5f2c.gz\n",
+      "679.\tLocal: Yes\tRemote: requests-00013-a731d3c9.gz\n",
+      "680.\tLocal: Yes\tRemote: requests-00014-bff33d09.gz\n",
+      "681.\tLocal: Yes\tRemote: requests-00015-b9a688ce.gz\n",
+      "682.\tLocal: Yes\tRemote: requests-00016-aed3aaaf.gz\n",
+      "683.\tLocal: Yes\tRemote: requests-00017-7aa1b9b3.gz\n",
+      "684.\tLocal: Yes\tRemote: requests-00018-3d4df635.gz\n",
+      "685.\tLocal: Yes\tRemote: requests-00019-ae441731.gz\n",
+      "686.\tLocal: Yes\tRemote: requests-00020-6c3399cd.gz\n",
+      "687.\tLocal: Yes\tRemote: requests-00021-63513136.gz\n",
+      "688.\tLocal: Yes\tRemote: requests-00022-71cf85a2.gz\n",
+      "689.\tLocal: Yes\tRemote: requests-00023-88808b55.gz\n",
+      "690.\tLocal: Yes\tRemote: requests-00024-15404624.gz\n",
+      "691.\tLocal: Yes\tRemote: requests-00025-691ecefa.gz\n",
+      "692.\tLocal: Yes\tRemote: requests-00026-707a7312.gz\n",
+      "693.\tLocal: Yes\tRemote: requests-00027-8cea72c1.gz\n",
+      "694.\tLocal: Yes\tRemote: requests-00028-7c7f0cda.gz\n",
+      "695.\tLocal: Yes\tRemote: requests-00029-79fd9d81.gz\n",
+      "696.\tLocal: Yes\tRemote: requests-00030-c4cf2b5a.gz\n",
+      "697.\tLocal: Yes\tRemote: requests-00031-b1ba2bd1.gz\n",
+      "698.\tLocal: Yes\tRemote: requests-00032-ed336ed5.gz\n",
+      "699.\tLocal: Yes\tRemote: requests-00033-ecd34daa.gz\n",
+      "700.\tLocal: Yes\tRemote: requests-00034-895d72f7.gz\n",
+      "701.\tLocal: Yes\tRemote: requests-00035-82c0fd5c.gz\n",
+      "702.\tLocal: Yes\tRemote: requests-00036-6790a604.gz\n",
+      "703.\tLocal: Yes\tRemote: requests-00037-e3aa4a67.gz\n",
+      "704.\tLocal: Yes\tRemote: requests-00038-188b8886.gz\n",
+      "705.\tLocal: Yes\tRemote: requests-00039-593b8710.gz\n",
+      "706.\tLocal: Yes\tRemote: requests-00040-c7465c07.gz\n",
+      "707.\tLocal: Yes\tRemote: requests-00041-f74061d8.gz\n",
+      "708.\tLocal: Yes\tRemote: requests-00042-13113bf7.gz\n",
+      "709.\tLocal: Yes\tRemote: requests-00043-9ad40cca.gz\n",
+      "710.\tLocal: Yes\tRemote: requests-00044-a6bca9cf.gz\n",
+      "711.\tLocal: Yes\tRemote: requests-00000-3fa519c1.gz\n",
+      "712.\tLocal: Yes\tRemote: requests-00001-95a5d509.gz\n",
+      "713.\tLocal: Yes\tRemote: requests-00002-3a5ecdef.gz\n",
+      "714.\tLocal: Yes\tRemote: requests-00003-540fa4f5.gz\n",
+      "715.\tLocal: Yes\tRemote: requests-00004-a98c5708.gz\n",
+      "716.\tLocal: Yes\tRemote: requests-00005-91970eb8.gz\n",
+      "717.\tLocal: Yes\tRemote: requests-00006-d6a9bc91.gz\n",
+      "718.\tLocal: Yes\tRemote: requests-00007-bfa57934.gz\n",
+      "719.\tLocal: Yes\tRemote: requests-00008-dcb8009a.gz\n",
+      "720.\tLocal: Yes\tRemote: requests-00009-9e174e95.gz\n",
+      "721.\tLocal: Yes\tRemote: requests-00010-debe13df.gz\n",
+      "722.\tLocal: Yes\tRemote: requests-00011-4bc0dad3.gz\n",
+      "723.\tLocal: Yes\tRemote: requests-00012-872d3fef.gz\n",
+      "724.\tLocal: Yes\tRemote: requests-00013-bede9710.gz\n",
+      "725.\tLocal: Yes\tRemote: requests-00014-a41f8e28.gz\n",
+      "726.\tLocal: Yes\tRemote: requests-00015-f3562487.gz\n",
+      "727.\tLocal: Yes\tRemote: requests-00016-818bf8b7.gz\n",
+      "728.\tLocal: Yes\tRemote: requests-00017-eaa6d3ac.gz\n",
+      "729.\tLocal: Yes\tRemote: requests-00018-0c878302.gz\n",
+      "730.\tLocal: Yes\tRemote: requests-00019-4ae89200.gz\n",
+      "731.\tLocal: Yes\tRemote: requests-00020-eb153d64.gz\n",
+      "732.\tLocal: Yes\tRemote: requests-00021-727518e2.gz\n",
+      "733.\tLocal: Yes\tRemote: requests-00022-2fabb3a4.gz\n",
+      "734.\tLocal: Yes\tRemote: requests-00000-dcd3c678.gz\n",
+      "735.\tLocal: Yes\tRemote: requests-00001-e31700f6.gz\n",
+      "736.\tLocal: Yes\tRemote: requests-00002-fba98df5.gz\n",
+      "737.\tLocal: Yes\tRemote: requests-00003-a5831d67.gz\n",
+      "738.\tLocal: Yes\tRemote: requests-00004-aa995fd0.gz\n",
+      "739.\tLocal: Yes\tRemote: requests-00005-a94fd0e3.gz\n",
+      "740.\tLocal: Yes\tRemote: requests-00006-dc170a5f.gz\n",
+      "741.\tLocal: Yes\tRemote: requests-00007-7215f3a8.gz\n",
+      "742.\tLocal: Yes\tRemote: requests-00008-48d9c0e8.gz\n",
+      "743.\tLocal: Yes\tRemote: requests-00009-5cbb4b36.gz\n",
+      "744.\tLocal: Yes\tRemote: requests-00010-e0f1e5dc.gz\n",
+      "745.\tLocal: Yes\tRemote: requests-00011-c6527c14.gz\n",
+      "746.\tLocal: Yes\tRemote: requests-00012-7e0f569b.gz\n",
+      "747.\tLocal: Yes\tRemote: requests-00013-bdc27fe7.gz\n",
+      "748.\tLocal: Yes\tRemote: requests-00014-12198eac.gz\n",
+      "749.\tLocal: Yes\tRemote: requests-00015-9359ec8b.gz\n",
+      "750.\tLocal: Yes\tRemote: requests-00016-054dee47.gz\n",
+      "751.\tLocal: Yes\tRemote: requests-00017-836557a9.gz\n",
+      "752.\tLocal: Yes\tRemote: requests-00018-ca84db78.gz\n",
+      "753.\tLocal: Yes\tRemote: requests-00019-11ae1e91.gz\n",
+      "754.\tLocal: Yes\tRemote: requests-00020-35e522a9.gz\n",
+      "755.\tLocal: Yes\tRemote: requests-00021-b22ff766.gz\n",
+      "756.\tLocal: Yes\tRemote: requests-00022-140631c3.gz\n",
+      "757.\tLocal: Yes\tRemote: requests-00023-50f14731.gz\n",
+      "758.\tLocal: Yes\tRemote: requests-00024-d5b15b90.gz\n",
+      "759.\tLocal: Yes\tRemote: requests-00025-becb8ba4.gz\n",
+      "760.\tLocal: Yes\tRemote: requests-00026-1f921bb8.gz\n",
+      "761.\tLocal: Yes\tRemote: requests-00027-15fbb7c9.gz\n",
+      "762.\tLocal: Yes\tRemote: requests-00028-7e3b3a89.gz\n",
+      "763.\tLocal: Yes\tRemote: requests-00029-e600bde8.gz\n",
+      "764.\tLocal: Yes\tRemote: requests-00030-f646e789.gz\n",
+      "765.\tLocal: Yes\tRemote: requests-00031-16f554e9.gz\n",
+      "766.\tLocal: Yes\tRemote: requests-00032-87469e3e.gz\n",
+      "767.\tLocal: Yes\tRemote: requests-00033-00311154.gz\n",
+      "768.\tLocal: Yes\tRemote: requests-00034-050448dd.gz\n",
+      "769.\tLocal: Yes\tRemote: requests-00035-1724ddba.gz\n",
+      "770.\tLocal: Yes\tRemote: requests-00036-431f836b.gz\n",
+      "771.\tLocal: Yes\tRemote: requests-00037-9ae9c15b.gz\n",
+      "772.\tLocal: Yes\tRemote: requests-00038-8ea7b9f7.gz\n",
+      "773.\tLocal: Yes\tRemote: requests-00039-e46a3b24.gz\n",
+      "774.\tLocal: Yes\tRemote: requests-00040-58a76ab0.gz\n",
+      "775.\tLocal: Yes\tRemote: requests-00041-715a9946.gz\n",
+      "776.\tLocal: Yes\tRemote: requests-00042-b8e7dcf2.gz\n",
+      "777.\tLocal: Yes\tRemote: requests-00043-5ae3563d.gz\n",
+      "778.\tLocal: Yes\tRemote: requests-00044-64dc885c.gz\n",
+      "779.\tLocal: Yes\tRemote: requests-00045-3463724a.gz\n",
+      "780.\tLocal: Yes\tRemote: requests-00046-87a9bcea.gz\n",
+      "781.\tLocal: Yes\tRemote: requests-00047-60b3f4af.gz\n",
+      "782.\tLocal: Yes\tRemote: requests-00000-5441c124.gz\n",
+      "783.\tLocal: Yes\tRemote: requests-00001-8a5f80d8.gz\n",
+      "784.\tLocal: Yes\tRemote: requests-00002-dac2996b.gz\n",
+      "785.\tLocal: Yes\tRemote: requests-00003-5ec2c754.gz\n",
+      "786.\tLocal: Yes\tRemote: requests-00004-4b813bd8.gz\n",
+      "787.\tLocal: Yes\tRemote: requests-00005-c43b76da.gz\n",
+      "788.\tLocal: Yes\tRemote: requests-00006-ff52a6f1.gz\n",
+      "789.\tLocal: Yes\tRemote: requests-00007-eeb92a6d.gz\n",
+      "790.\tLocal: Yes\tRemote: requests-00008-f19f0f43.gz\n",
+      "791.\tLocal: Yes\tRemote: requests-00009-eb5cc08b.gz\n",
+      "792.\tLocal: Yes\tRemote: requests-00010-bf54b116.gz\n",
+      "793.\tLocal: Yes\tRemote: requests-00011-018a8ba1.gz\n",
+      "794.\tLocal: Yes\tRemote: requests-00012-17f1ffea.gz\n",
+      "795.\tLocal: Yes\tRemote: requests-00013-32c19f49.gz\n",
+      "796.\tLocal: Yes\tRemote: requests-00014-05c3b284.gz\n",
+      "797.\tLocal: Yes\tRemote: requests-00015-4eaa22cb.gz\n",
+      "798.\tLocal: Yes\tRemote: requests-00016-66e10c12.gz\n",
+      "799.\tLocal: Yes\tRemote: requests-00017-f18feb5a.gz\n",
+      "800.\tLocal: Yes\tRemote: requests-00018-fae23802.gz\n",
+      "801.\tLocal: Yes\tRemote: requests-00019-27e7e8f3.gz\n",
+      "802.\tLocal: Yes\tRemote: requests-00020-d62ff0ea.gz\n",
+      "803.\tLocal: Yes\tRemote: requests-00021-a76d91a8.gz\n",
+      "804.\tLocal: Yes\tRemote: requests-00022-85264a36.gz\n",
+      "805.\tLocal: Yes\tRemote: requests-00023-b4fd78dd.gz\n",
+      "806.\tLocal: Yes\tRemote: requests-00024-5536bc82.gz\n",
+      "807.\tLocal: Yes\tRemote: requests-00025-0644272d.gz\n",
+      "808.\tLocal: Yes\tRemote: requests-00026-f3fcb02f.gz\n",
+      "809.\tLocal: Yes\tRemote: requests-00027-c335328e.gz\n",
+      "810.\tLocal: Yes\tRemote: requests-00028-e8d0398d.gz\n",
+      "811.\tLocal: Yes\tRemote: requests-00029-bcabed01.gz\n",
+      "812.\tLocal: Yes\tRemote: requests-00030-1e30038b.gz\n",
+      "813.\tLocal: Yes\tRemote: requests-00031-bf02040a.gz\n",
+      "814.\tLocal: Yes\tRemote: requests-00032-ae754a9a.gz\n",
+      "815.\tLocal: Yes\tRemote: requests-00033-5d4141c0.gz\n",
+      "816.\tLocal: Yes\tRemote: requests-00034-7e3e4636.gz\n",
+      "817.\tLocal: Yes\tRemote: requests-00035-eba820c1.gz\n",
+      "818.\tLocal: Yes\tRemote: requests-00036-fcc6b1c3.gz\n",
+      "819.\tLocal: Yes\tRemote: requests-00037-f71ede40.gz\n",
+      "820.\tLocal: Yes\tRemote: requests-00038-cee8f83a.gz\n",
+      "821.\tLocal: Yes\tRemote: requests-00039-70a8d769.gz\n",
+      "822.\tLocal: Yes\tRemote: requests-00040-e5b29609.gz\n",
+      "823.\tLocal: Yes\tRemote: requests-00041-80e988dd.gz\n",
+      "824.\tLocal: Yes\tRemote: requests-00042-17b1220f.gz\n",
+      "825.\tLocal: Yes\tRemote: requests-00043-5bf1045f.gz\n",
+      "826.\tLocal: Yes\tRemote: requests-00044-3131ad78.gz\n",
+      "827.\tLocal: Yes\tRemote: requests-00045-0d7ef588.gz\n",
+      "828.\tLocal: Yes\tRemote: requests-00046-291a14e2.gz\n",
+      "829.\tLocal: Yes\tRemote: requests-00047-ad8566b3.gz\n",
+      "830.\tLocal: Yes\tRemote: requests-00048-d8c60a69.gz\n",
+      "831.\tLocal: Yes\tRemote: requests-00049-e58b65e2.gz\n",
+      "832.\tLocal: Yes\tRemote: requests-00050-b2dd93fd.gz\n",
+      "833.\tLocal: Yes\tRemote: requests-00051-eebb6f85.gz\n",
+      "834.\tLocal: Yes\tRemote: requests-00052-c64c6670.gz\n",
+      "835.\tLocal: Yes\tRemote: requests-00053-2a2dfadc.gz\n",
+      "836.\tLocal: Yes\tRemote: requests-00000-a2f9c912.gz\n",
+      "837.\tLocal: Yes\tRemote: requests-00001-6f1eed1b.gz\n",
+      "838.\tLocal: Yes\tRemote: requests-00002-1f79ad84.gz\n",
+      "839.\tLocal: Yes\tRemote: requests-00003-ad10342f.gz\n",
+      "840.\tLocal: Yes\tRemote: requests-00004-191dbc18.gz\n",
+      "841.\tLocal: Yes\tRemote: requests-00005-ab8d5187.gz\n",
+      "842.\tLocal: Yes\tRemote: requests-00006-ccc5a2d9.gz\n",
+      "843.\tLocal: Yes\tRemote: requests-00007-5441f02e.gz\n",
+      "844.\tLocal: Yes\tRemote: requests-00008-004a51f7.gz\n",
+      "845.\tLocal: Yes\tRemote: requests-00009-4d6de45e.gz\n",
+      "846.\tLocal: Yes\tRemote: requests-00010-190b4388.gz\n",
+      "847.\tLocal: Yes\tRemote: requests-00011-c690fae9.gz\n",
+      "848.\tLocal: Yes\tRemote: requests-00012-7aee369a.gz\n",
+      "849.\tLocal: Yes\tRemote: requests-00013-6e1e5d92.gz\n",
+      "850.\tLocal: Yes\tRemote: requests-00014-ab26525e.gz\n",
+      "851.\tLocal: Yes\tRemote: requests-00015-9e91df69.gz\n",
+      "852.\tLocal: Yes\tRemote: requests-00016-d9929986.gz\n",
+      "853.\tLocal: Yes\tRemote: requests-00017-19311f93.gz\n",
+      "854.\tLocal: Yes\tRemote: requests-00018-60f77b6c.gz\n",
+      "855.\tLocal: Yes\tRemote: requests-00019-8156b3f0.gz\n",
+      "856.\tLocal: Yes\tRemote: requests-00020-03777a26.gz\n",
+      "857.\tLocal: Yes\tRemote: requests-00021-336a2706.gz\n",
+      "858.\tLocal: Yes\tRemote: requests-00022-a18d7ec0.gz\n",
+      "859.\tLocal: Yes\tRemote: requests-00023-7f876543.gz\n",
+      "860.\tLocal: Yes\tRemote: requests-00024-4128fde5.gz\n",
+      "861.\tLocal: Yes\tRemote: requests-00025-265d8bbe.gz\n",
+      "862.\tLocal: Yes\tRemote: requests-00026-acdd8590.gz\n",
+      "863.\tLocal: Yes\tRemote: requests-00027-a72aafea.gz\n",
+      "864.\tLocal: Yes\tRemote: requests-00028-a8ebece1.gz\n",
+      "865.\tLocal: Yes\tRemote: requests-00029-b7d131b7.gz\n",
+      "866.\tLocal: Yes\tRemote: requests-00030-a7264a2c.gz\n",
+      "867.\tLocal: Yes\tRemote: requests-00031-5af36370.gz\n",
+      "868.\tLocal: Yes\tRemote: requests-00032-70a506b4.gz\n",
+      "869.\tLocal: Yes\tRemote: requests-00033-60343aa7.gz\n",
+      "870.\tLocal: Yes\tRemote: requests-00034-114e3ac0.gz\n",
+      "871.\tLocal: Yes\tRemote: requests-00035-99473bc2.gz\n",
+      "872.\tLocal: Yes\tRemote: requests-00036-416a432b.gz\n",
+      "873.\tLocal: Yes\tRemote: requests-00037-d74295d1.gz\n",
+      "874.\tLocal: Yes\tRemote: requests-00038-d7a10544.gz\n",
+      "875.\tLocal: Yes\tRemote: requests-00039-1ab73be5.gz\n",
+      "876.\tLocal: Yes\tRemote: requests-00040-5c597063.gz\n",
+      "877.\tLocal: Yes\tRemote: requests-00041-1aaa5bf5.gz\n",
+      "878.\tLocal: Yes\tRemote: requests-00042-5e29298a.gz\n",
+      "879.\tLocal: Yes\tRemote: requests-00043-0c790d3a.gz\n",
+      "880.\tLocal: Yes\tRemote: requests-00044-88531363.gz\n",
+      "881.\tLocal: Yes\tRemote: requests-00045-f36904c2.gz\n",
+      "882.\tLocal: Yes\tRemote: requests-00000-92a6aa93.gz\n",
+      "883.\tLocal: Yes\tRemote: requests-00001-c70a46a3.gz\n",
+      "884.\tLocal: Yes\tRemote: requests-00002-b37eb68a.gz\n",
+      "885.\tLocal: Yes\tRemote: requests-00003-5caa7cb3.gz\n",
+      "886.\tLocal: Yes\tRemote: requests-00004-000e8a63.gz\n",
+      "887.\tLocal: Yes\tRemote: requests-00005-2945ceca.gz\n",
+      "888.\tLocal: Yes\tRemote: requests-00006-90c3fbe9.gz\n",
+      "889.\tLocal: Yes\tRemote: requests-00007-debf8c51.gz\n",
+      "890.\tLocal: Yes\tRemote: requests-00008-e8bca1f7.gz\n",
+      "891.\tLocal: Yes\tRemote: requests-00009-45d475a6.gz\n",
+      "892.\tLocal: Yes\tRemote: requests-00010-d2ae85f2.gz\n",
+      "893.\tLocal: Yes\tRemote: requests-00011-c4022186.gz\n",
+      "894.\tLocal: Yes\tRemote: requests-00012-c076f9fd.gz\n",
+      "895.\tLocal: Yes\tRemote: requests-00013-e5626271.gz\n",
+      "896.\tLocal: Yes\tRemote: requests-00000-a5664cb6.gz\n",
+      "897.\tLocal: Yes\tRemote: requests-00001-888cb7e6.gz\n",
+      "898.\tLocal: Yes\tRemote: requests-00002-e80369a5.gz\n",
+      "899.\tLocal: Yes\tRemote: requests-00003-a490dd3e.gz\n",
+      "900.\tLocal: Yes\tRemote: requests-00004-935be648.gz\n",
+      "901.\tLocal: Yes\tRemote: requests-00005-0cac80c7.gz\n",
+      "902.\tLocal: Yes\tRemote: requests-00006-75ed9eca.gz\n",
+      "903.\tLocal: Yes\tRemote: requests-00007-3a5baf43.gz\n",
+      "904.\tLocal: Yes\tRemote: requests-00008-50d382b5.gz\n",
+      "905.\tLocal: Yes\tRemote: requests-00009-2481413c.gz\n",
+      "906.\tLocal: Yes\tRemote: requests-00010-10894c30.gz\n",
+      "907.\tLocal: Yes\tRemote: requests-00011-0df780d8.gz\n",
+      "908.\tLocal: Yes\tRemote: requests-00012-4bf629c0.gz\n",
+      "909.\tLocal: Yes\tRemote: requests-00013-d3a21989.gz\n",
+      "910.\tLocal: Yes\tRemote: requests-00014-16d35e0e.gz\n",
+      "911.\tLocal: Yes\tRemote: requests-00015-55e8afcf.gz\n",
+      "912.\tLocal: Yes\tRemote: requests-00016-46cd208c.gz\n",
+      "913.\tLocal: Yes\tRemote: requests-00017-b5d0c61c.gz\n",
+      "914.\tLocal: Yes\tRemote: requests-00018-804145f4.gz\n",
+      "915.\tLocal: Yes\tRemote: requests-00019-b1eabdf5.gz\n",
+      "916.\tLocal: Yes\tRemote: requests-00020-4d7112bc.gz\n",
+      "917.\tLocal: Yes\tRemote: requests-00021-32f3f7ec.gz\n",
+      "918.\tLocal: Yes\tRemote: requests-00022-b537a3e3.gz\n",
+      "919.\tLocal: Yes\tRemote: requests-00023-3619ced9.gz\n",
+      "920.\tLocal: Yes\tRemote: requests-00024-257db6c6.gz\n",
+      "921.\tLocal: Yes\tRemote: requests-00025-3657fe60.gz\n",
+      "922.\tLocal: Yes\tRemote: requests-00026-fcb03d92.gz\n",
+      "923.\tLocal: Yes\tRemote: requests-00000-978d0411.gz\n",
+      "924.\tLocal: Yes\tRemote: requests-00001-6dcba114.gz\n",
+      "925.\tLocal: Yes\tRemote: requests-00002-c9cdf0fe.gz\n",
+      "926.\tLocal: Yes\tRemote: requests-00003-5d9df411.gz\n",
+      "927.\tLocal: Yes\tRemote: requests-00004-a998ffde.gz\n",
+      "928.\tLocal: Yes\tRemote: requests-00005-531146ee.gz\n",
+      "929.\tLocal: Yes\tRemote: requests-00006-2c11b022.gz\n",
+      "930.\tLocal: Yes\tRemote: requests-00007-f4464060.gz\n",
+      "931.\tLocal: Yes\tRemote: requests-00008-b44ccb22.gz\n",
+      "932.\tLocal: Yes\tRemote: requests-00009-79291b24.gz\n",
+      "933.\tLocal: Yes\tRemote: requests-00010-c2280483.gz\n",
+      "934.\tLocal: Yes\tRemote: requests-00011-2dbd3748.gz\n",
+      "935.\tLocal: Yes\tRemote: requests-00012-2a447008.gz\n",
+      "936.\tLocal: Yes\tRemote: requests-00013-a3c81a0a.gz\n",
+      "937.\tLocal: Yes\tRemote: requests-00014-e3fb827d.gz\n",
+      "938.\tLocal: Yes\tRemote: requests-00015-d1a2ceb7.gz\n",
+      "939.\tLocal: Yes\tRemote: requests-00016-e1c65984.gz\n",
+      "940.\tLocal: Yes\tRemote: requests-00017-7c9f0d8c.gz\n",
+      "941.\tLocal: Yes\tRemote: requests-00018-2b03bfa3.gz\n",
+      "942.\tLocal: Yes\tRemote: requests-00019-908ef798.gz\n",
+      "943.\tLocal: Yes\tRemote: requests-00020-91163cc0.gz\n",
+      "944.\tLocal: Yes\tRemote: requests-00021-dba82799.gz\n",
+      "945.\tLocal: Yes\tRemote: requests-00022-71bbcd9c.gz\n",
+      "946.\tLocal: Yes\tRemote: requests-00000-105ffeb3.gz\n",
+      "947.\tLocal: Yes\tRemote: requests-00001-ed87f0e7.gz\n",
+      "948.\tLocal: Yes\tRemote: requests-00002-355bffdb.gz\n",
+      "949.\tLocal: Yes\tRemote: requests-00003-3d8ed783.gz\n",
+      "950.\tLocal: Yes\tRemote: requests-00004-dacf1471.gz\n",
+      "951.\tLocal: Yes\tRemote: requests-00005-748eb122.gz\n",
+      "952.\tLocal: Yes\tRemote: requests-00006-0fb84b59.gz\n",
+      "953.\tLocal: Yes\tRemote: requests-00007-d4d1f8fc.gz\n",
+      "954.\tLocal: Yes\tRemote: requests-00008-97f8fe17.gz\n",
+      "955.\tLocal: Yes\tRemote: requests-00009-248193c5.gz\n",
+      "956.\tLocal: Yes\tRemote: requests-00010-af7b65b8.gz\n",
+      "957.\tLocal: Yes\tRemote: requests-00011-10d176dd.gz\n",
+      "958.\tLocal: Yes\tRemote: requests-00000-6cd45300.gz\n",
+      "959.\tLocal: Yes\tRemote: requests-00001-7ceccb8b.gz\n",
+      "960.\tLocal: Yes\tRemote: requests-00002-e7cf0967.gz\n",
+      "961.\tLocal: Yes\tRemote: requests-00003-d0da7bb2.gz\n",
+      "962.\tLocal: Yes\tRemote: requests-00004-ab2e918b.gz\n",
+      "963.\tLocal: Yes\tRemote: requests-00005-a07d87e4.gz\n",
+      "964.\tLocal: Yes\tRemote: requests-00006-3ba8aa73.gz\n",
+      "965.\tLocal: Yes\tRemote: requests-00007-4d9a5a20.gz\n",
+      "966.\tLocal: Yes\tRemote: requests-00000-90b2ff27.gz\n",
+      "967.\tLocal: Yes\tRemote: requests-00001-e943311c.gz\n",
+      "968.\tLocal: Yes\tRemote: requests-00002-5c89301c.gz\n",
+      "969.\tLocal: Yes\tRemote: requests-00003-1cbb5bb1.gz\n",
+      "970.\tLocal: Yes\tRemote: requests-00004-d58e0cdc.gz\n",
+      "971.\tLocal: Yes\tRemote: requests-00005-9dae3f8f.gz\n",
+      "972.\tLocal: Yes\tRemote: requests-00006-f5934591.gz\n",
+      "973.\tLocal: Yes\tRemote: requests-00007-28a73752.gz\n",
+      "974.\tLocal: Yes\tRemote: requests-00008-c2f7167e.gz\n",
+      "975.\tLocal: Yes\tRemote: requests-00009-5a247b24.gz\n",
+      "976.\tLocal: Yes\tRemote: requests-00010-a6415ee4.gz\n",
+      "977.\tLocal: Yes\tRemote: requests-00011-1b37fb03.gz\n",
+      "978.\tLocal: Yes\tRemote: requests-00012-00246a7f.gz\n",
+      "979.\tLocal: Yes\tRemote: requests-00013-bc2000ab.gz\n",
+      "980.\tLocal: Yes\tRemote: requests-00014-4b691333.gz\n",
+      "981.\tLocal: Yes\tRemote: requests-00015-117c0efc.gz\n",
+      "982.\tLocal: Yes\tRemote: requests-00016-4a3ad743.gz\n",
+      "983.\tLocal: Yes\tRemote: requests-00017-f10634da.gz\n",
+      "984.\tLocal: Yes\tRemote: requests-00018-84ed1ff4.gz\n",
+      "985.\tLocal: Yes\tRemote: requests-00019-89e3e359.gz\n",
+      "986.\tLocal: Yes\tRemote: requests-00020-a82fbd6b.gz\n",
+      "987.\tLocal: Yes\tRemote: requests-00000-6c41e5d7.gz\n",
+      "988.\tLocal: Yes\tRemote: requests-00001-47300976.gz\n",
+      "989.\tLocal: Yes\tRemote: requests-00002-3cb5ea84.gz\n",
+      "990.\tLocal: Yes\tRemote: requests-00003-eb24052e.gz\n",
+      "991.\tLocal: Yes\tRemote: requests-00000-271e2656.gz\n",
+      "992.\tLocal: Yes\tRemote: requests-00000-76f5725a.gz\n",
+      "993.\tLocal: Yes\tRemote: requests-00001-f2b584b9.gz\n",
+      "994.\tLocal: Yes\tRemote: requests-00002-6fd8ad64.gz\n",
+      "995.\tLocal: Yes\tRemote: requests-00003-a2fa862b.gz\n",
+      "996.\tLocal: Yes\tRemote: requests-00004-e05a601f.gz\n",
+      "997.\tLocal: Yes\tRemote: requests-00005-ac8aa6bb.gz\n",
+      "998.\tLocal: Yes\tRemote: requests-00006-4d2a6cad.gz\n",
+      "999.\tLocal: Yes\tRemote: requests-00007-b47f9385.gz\n",
+      "1000.\tLocal: Yes\tRemote: requests-00008-53805932.gz\n",
+      "1001.\tLocal: Yes\tRemote: requests-00009-0a5d54cc.gz\n",
+      "1002.\tLocal: Yes\tRemote: requests-00010-d6e1448c.gz\n",
+      "1003.\tLocal: Yes\tRemote: requests-00011-35c27777.gz\n",
+      "1004.\tLocal: Yes\tRemote: requests-00012-6ca1fb0d.gz\n",
+      "1005.\tLocal: Yes\tRemote: requests-00013-feb557d5.gz\n",
+      "1006.\tLocal: Yes\tRemote: requests-00014-6eb48f8d.gz\n",
+      "1007.\tLocal: Yes\tRemote: requests-00015-48e1abb1.gz\n",
+      "1008.\tLocal: Yes\tRemote: requests-00016-f53a3c02.gz\n",
+      "1009.\tLocal: Yes\tRemote: requests-00017-79e32127.gz\n",
+      "1010.\tLocal: Yes\tRemote: requests-00018-0f0c2767.gz\n",
+      "1011.\tLocal: Yes\tRemote: requests-00019-8cfb10bc.gz\n",
+      "1012.\tLocal: Yes\tRemote: requests-00020-f5a9be70.gz\n",
+      "1013.\tLocal: Yes\tRemote: requests-00021-83b89b46.gz\n",
+      "1014.\tLocal: Yes\tRemote: requests-00022-426552af.gz\n",
+      "1015.\tLocal: Yes\tRemote: requests-00023-b17f6322.gz\n",
+      "1016.\tLocal: Yes\tRemote: requests-00024-58b51dc5.gz\n",
+      "1017.\tLocal: Yes\tRemote: requests-00025-5d335627.gz\n",
+      "1018.\tLocal: Yes\tRemote: requests-00026-77118107.gz\n",
+      "1019.\tLocal: Yes\tRemote: requests-00027-357d3778.gz\n",
+      "1020.\tLocal: Yes\tRemote: requests-00028-17dc2c68.gz\n",
+      "1021.\tLocal: Yes\tRemote: requests-00029-712a2b24.gz\n",
+      "1022.\tLocal: Yes\tRemote: requests-00030-2ee392f8.gz\n",
+      "1023.\tLocal: Yes\tRemote: requests-00031-c866e2b0.gz\n",
+      "1024.\tLocal: Yes\tRemote: requests-00032-40052cf1.gz\n",
+      "1025.\tLocal: Yes\tRemote: requests-00033-2573bd3c.gz\n",
+      "1026.\tLocal: Yes\tRemote: requests-00034-b1027542.gz\n",
+      "1027.\tLocal: Yes\tRemote: requests-00035-0073fef7.gz\n",
+      "1028.\tLocal: Yes\tRemote: requests-00036-baa566e4.gz\n",
+      "1029.\tLocal: Yes\tRemote: requests-00037-c8cafc38.gz\n",
+      "1030.\tLocal: Yes\tRemote: requests-00038-eff56a6e.gz\n",
+      "1031.\tLocal: Yes\tRemote: requests-00039-109b8959.gz\n",
+      "1032.\tLocal: Yes\tRemote: requests-00040-7105d0ab.gz\n",
+      "1033.\tLocal: Yes\tRemote: requests-00041-2520c0b9.gz\n",
+      "1034.\tLocal: Yes\tRemote: requests-00042-7ce2083f.gz\n",
+      "1035.\tLocal: Yes\tRemote: requests-00043-64d22031.gz\n",
+      "1036.\tLocal: Yes\tRemote: requests-00044-620f66dc.gz\n",
+      "1037.\tLocal: Yes\tRemote: requests-00045-03276816.gz\n",
+      "1038.\tLocal: Yes\tRemote: requests-00046-6c598b85.gz\n",
+      "1039.\tLocal: Yes\tRemote: requests-00047-d5b816fb.gz\n",
+      "1040.\tLocal: Yes\tRemote: requests-00048-607d3e0e.gz\n",
+      "1041.\tLocal: Yes\tRemote: requests-00049-168267f0.gz\n",
+      "1042.\tLocal: Yes\tRemote: requests-00050-64505433.gz\n",
+      "1043.\tLocal: Yes\tRemote: requests-00051-59beecf3.gz\n",
+      "1044.\tLocal: Yes\tRemote: requests-00052-58cc82ca.gz\n",
+      "1045.\tLocal: Yes\tRemote: requests-00053-185ade60.gz\n",
+      "1046.\tLocal: Yes\tRemote: requests-00054-67ab6646.gz\n",
+      "1047.\tLocal: Yes\tRemote: requests-00055-2d4c170a.gz\n",
+      "1048.\tLocal: Yes\tRemote: requests-00056-72ca324d.gz\n",
+      "1049.\tLocal: Yes\tRemote: requests-00057-4ae90eee.gz\n",
+      "1050.\tLocal: Yes\tRemote: requests-00058-7054141f.gz\n",
+      "1051.\tLocal: Yes\tRemote: requests-00059-f0c4e410.gz\n",
+      "1052.\tLocal: Yes\tRemote: requests-00060-dec88f30.gz\n",
+      "1053.\tLocal: Yes\tRemote: requests-00000-588aba81.gz\n",
+      "1054.\tLocal: Yes\tRemote: requests-00001-c3349f2c.gz\n",
+      "1055.\tLocal: Yes\tRemote: requests-00002-3ad5812e.gz\n",
+      "1056.\tLocal: Yes\tRemote: requests-00003-77f15ff0.gz\n",
+      "1057.\tLocal: Yes\tRemote: requests-00004-18897624.gz\n",
+      "1058.\tLocal: Yes\tRemote: requests-00005-8e3f4fd9.gz\n",
+      "1059.\tLocal: Yes\tRemote: requests-00006-aed24973.gz\n",
+      "1060.\tLocal: Yes\tRemote: requests-00007-6ee6747e.gz\n",
+      "1061.\tLocal: Yes\tRemote: requests-00008-faab0013.gz\n",
+      "1062.\tLocal: Yes\tRemote: requests-00009-62be6d49.gz\n",
+      "1063.\tLocal: Yes\tRemote: requests-00010-9b7d2444.gz\n",
+      "1064.\tLocal: Yes\tRemote: requests-00011-4b25962c.gz\n",
+      "1065.\tLocal: Yes\tRemote: requests-00012-9e2892ba.gz\n",
+      "1066.\tLocal: Yes\tRemote: requests-00013-c6f5e5a7.gz\n",
+      "1067.\tLocal: Yes\tRemote: requests-00014-e073af74.gz\n",
+      "1068.\tLocal: Yes\tRemote: requests-00015-7627a2cb.gz\n",
+      "1069.\tLocal: Yes\tRemote: requests-00016-7ca2b854.gz\n",
+      "1070.\tLocal: Yes\tRemote: requests-00017-f26aa116.gz\n",
+      "1071.\tLocal: Yes\tRemote: requests-00018-b734dbf3.gz\n",
+      "1072.\tLocal: Yes\tRemote: requests-00019-a387cee2.gz\n",
+      "1073.\tLocal: Yes\tRemote: requests-00020-485ed0d0.gz\n",
+      "1074.\tLocal: Yes\tRemote: requests-00021-8eee0dd0.gz\n",
+      "1075.\tLocal: Yes\tRemote: requests-00022-21366f25.gz\n",
+      "1076.\tLocal: Yes\tRemote: requests-00023-a606abf6.gz\n",
+      "1077.\tLocal: Yes\tRemote: requests-00024-9e23da6e.gz\n",
+      "1078.\tLocal: Yes\tRemote: requests-00025-ecb40c9c.gz\n",
+      "1079.\tLocal: Yes\tRemote: requests-00026-7c523aa8.gz\n",
+      "1080.\tLocal: Yes\tRemote: requests-00027-692fc50a.gz\n",
+      "1081.\tLocal: Yes\tRemote: requests-00028-6334bac5.gz\n",
+      "1082.\tLocal: Yes\tRemote: requests-00029-4eee5913.gz\n",
+      "1083.\tLocal: Yes\tRemote: requests-00030-9c948ab7.gz\n",
+      "1084.\tLocal: Yes\tRemote: requests-00031-d87a11f3.gz\n",
+      "1085.\tLocal: Yes\tRemote: requests-00032-35b77e24.gz\n",
+      "1086.\tLocal: Yes\tRemote: requests-00033-31b89f22.gz\n",
+      "1087.\tLocal: Yes\tRemote: requests-00034-6c8ed8f5.gz\n",
+      "1088.\tLocal: Yes\tRemote: requests-00035-80f3644c.gz\n",
+      "1089.\tLocal: Yes\tRemote: requests-00036-1906c4ef.gz\n",
+      "1090.\tLocal: Yes\tRemote: requests-00037-1a8e4557.gz\n",
+      "1091.\tLocal: Yes\tRemote: requests-00038-66688cef.gz\n",
+      "1092.\tLocal: Yes\tRemote: requests-00039-96dfb57b.gz\n",
+      "1093.\tLocal: Yes\tRemote: requests-00040-528aa84f.gz\n",
+      "1094.\tLocal: Yes\tRemote: requests-00041-5aef345b.gz\n",
+      "1095.\tLocal: Yes\tRemote: requests-00042-d1b5e150.gz\n",
+      "1096.\tLocal: Yes\tRemote: requests-00043-df5d9fe9.gz\n",
+      "1097.\tLocal: Yes\tRemote: requests-00044-5c0a7888.gz\n",
+      "1098.\tLocal: Yes\tRemote: requests-00045-38c3bc8b.gz\n",
+      "1099.\tLocal: Yes\tRemote: requests-00046-5e74b0ae.gz\n",
+      "1100.\tLocal: Yes\tRemote: requests-00047-d3b18f19.gz\n",
+      "1101.\tLocal: Yes\tRemote: requests-00048-05389499.gz\n",
+      "1102.\tLocal: Yes\tRemote: requests-00049-4f869c36.gz\n",
+      "1103.\tLocal: Yes\tRemote: requests-00050-c94deaa5.gz\n",
+      "1104.\tLocal: Yes\tRemote: requests-00051-2cea7ab8.gz\n",
+      "1105.\tLocal: Yes\tRemote: requests-00052-b75a2c94.gz\n",
+      "1106.\tLocal: Yes\tRemote: requests-00053-c4fb3fb9.gz\n",
+      "1107.\tLocal: Yes\tRemote: requests-00054-7d07acdd.gz\n",
+      "1108.\tLocal: Yes\tRemote: requests-00055-5c897387.gz\n",
+      "1109.\tLocal: Yes\tRemote: requests-00056-7303cdeb.gz\n",
+      "1110.\tLocal: Yes\tRemote: requests-00057-0a10ea07.gz\n",
+      "1111.\tLocal: Yes\tRemote: requests-00058-69f5b583.gz\n",
+      "1112.\tLocal: Yes\tRemote: requests-00059-44ba4769.gz\n",
+      "1113.\tLocal: Yes\tRemote: requests-00060-2be04f28.gz\n",
+      "1114.\tLocal: Yes\tRemote: requests-00061-816eea8e.gz\n",
+      "1115.\tLocal: Yes\tRemote: requests-00062-416f9c70.gz\n",
+      "1116.\tLocal: Yes\tRemote: requests-00063-d1958b59.gz\n",
+      "1117.\tLocal: Yes\tRemote: requests-00064-623dad42.gz\n",
+      "1118.\tLocal: Yes\tRemote: requests-00065-a8a5cefb.gz\n",
+      "1119.\tLocal: Yes\tRemote: requests-00066-6adedb41.gz\n",
+      "1120.\tLocal: Yes\tRemote: requests-00000-79e2f7bd.gz\n",
+      "1121.\tLocal: Yes\tRemote: requests-00001-9f2c4b0c.gz\n",
+      "1122.\tLocal: Yes\tRemote: requests-00002-d867bb0b.gz\n",
+      "1123.\tLocal: Yes\tRemote: requests-00003-7b7211ea.gz\n",
+      "1124.\tLocal: Yes\tRemote: requests-00004-9cff1ae5.gz\n",
+      "1125.\tLocal: Yes\tRemote: requests-00005-db33dc5b.gz\n",
+      "1126.\tLocal: Yes\tRemote: requests-00000-5b719ff3.gz\n",
+      "1127.\tLocal: Yes\tRemote: requests-00001-222ce57a.gz\n",
+      "1128.\tLocal: Yes\tRemote: requests-00002-6604040b.gz\n",
+      "1129.\tLocal: Yes\tRemote: requests-00003-3d7d8c92.gz\n",
+      "1130.\tLocal: Yes\tRemote: requests-00004-e0de59f5.gz\n",
+      "1131.\tLocal: Yes\tRemote: requests-00005-e1f715b8.gz\n",
+      "1132.\tLocal: Yes\tRemote: requests-00006-cd083677.gz\n",
+      "1133.\tLocal: Yes\tRemote: requests-00007-b26247d8.gz\n",
+      "1134.\tLocal: Yes\tRemote: requests-00008-4ab2cfea.gz\n",
+      "1135.\tLocal: Yes\tRemote: requests-00009-1c68f434.gz\n",
+      "1136.\tLocal: Yes\tRemote: requests-00010-2fe5b019.gz\n",
+      "1137.\tLocal: Yes\tRemote: requests-00011-8b446bbc.gz\n",
+      "1138.\tLocal: Yes\tRemote: requests-00012-b9af75a3.gz\n",
+      "1139.\tLocal: Yes\tRemote: requests-00013-0fdee62d.gz\n",
+      "1140.\tLocal: Yes\tRemote: requests-00014-b1aee5b5.gz\n",
+      "1141.\tLocal: Yes\tRemote: requests-00015-10191d19.gz\n",
+      "1142.\tLocal: Yes\tRemote: requests-00016-7f0559a2.gz\n",
+      "1143.\tLocal: Yes\tRemote: requests-00017-f2b2c3d2.gz\n",
+      "1144.\tLocal: Yes\tRemote: requests-00018-b0ac89ed.gz\n",
+      "1145.\tLocal: Yes\tRemote: requests-00019-19884ffc.gz\n",
+      "1146.\tLocal: Yes\tRemote: requests-00020-73a4a8ea.gz\n",
+      "1147.\tLocal: Yes\tRemote: requests-00021-add49214.gz\n",
+      "1148.\tLocal: Yes\tRemote: requests-00022-3c4e75b4.gz\n",
+      "1149.\tLocal: Yes\tRemote: requests-00023-222d778b.gz\n",
+      "1150.\tLocal: Yes\tRemote: requests-00024-7ce63e27.gz\n",
+      "1151.\tLocal: Yes\tRemote: requests-00025-9cecc260.gz\n",
+      "1152.\tLocal: Yes\tRemote: requests-00026-ce6511ed.gz\n",
+      "1153.\tLocal: Yes\tRemote: requests-00027-642592ae.gz\n",
+      "1154.\tLocal: Yes\tRemote: requests-00028-ed183e3a.gz\n",
+      "1155.\tLocal: Yes\tRemote: requests-00029-ee980a2a.gz\n",
+      "1156.\tLocal: Yes\tRemote: requests-00030-56f386d0.gz\n",
+      "1157.\tLocal: Yes\tRemote: requests-00031-afc834d1.gz\n",
+      "1158.\tLocal: Yes\tRemote: requests-00032-6041f25a.gz\n",
+      "1159.\tLocal: Yes\tRemote: requests-00033-1d592e63.gz\n",
+      "1160.\tLocal: Yes\tRemote: requests-00034-b33d943b.gz\n",
+      "1161.\tLocal: Yes\tRemote: requests-00035-594c9b55.gz\n",
+      "1162.\tLocal: Yes\tRemote: requests-00036-f0890f80.gz\n",
+      "1163.\tLocal: Yes\tRemote: requests-00037-dd6fd529.gz\n",
+      "1164.\tLocal: Yes\tRemote: requests-00038-7f39f23c.gz\n",
+      "1165.\tLocal: Yes\tRemote: requests-00039-deeaa6d8.gz\n",
+      "1166.\tLocal: Yes\tRemote: requests-00040-c86d781e.gz\n",
+      "1167.\tLocal: Yes\tRemote: requests-00041-27bbc0d4.gz\n",
+      "1168.\tLocal: Yes\tRemote: requests-00042-4bcb7a4e.gz\n",
+      "1169.\tLocal: Yes\tRemote: requests-00043-f110296d.gz\n",
+      "1170.\tLocal: Yes\tRemote: requests-00044-9f89a8b6.gz\n",
+      "1171.\tLocal: Yes\tRemote: requests-00045-4a9898f6.gz\n",
+      "1172.\tLocal: Yes\tRemote: requests-00046-1b4ebfe3.gz\n",
+      "1173.\tLocal: Yes\tRemote: requests-00047-76c2db32.gz\n",
+      "I will attempt to download 119 files.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Press enter to begin, or q to quit \n",
+      "Downloading: account_dim-00000-f5038498.gz\n",
+      "Success\n",
+      "Downloading: assignment_dim-00000-c36a7689.gz\n",
+      "Success\n",
+      "Downloading: assignment_fact-00000-2e0dd051.gz\n",
+      "Success\n",
+      "Downloading: assignment_group_dim-00000-27db5aaf.gz\n",
+      "Success\n",
+      "Downloading: assignment_group_fact-00000-f1b5fe92.gz\n",
+      "Success\n",
+      "Downloading: assignment_group_rule_dim-00000-20680b49.gz\n",
+      "Success\n",
+      "Downloading: assignment_group_score_dim-00000-7ae8f32b.gz\n",
+      "Success\n",
+      "Downloading: assignment_group_score_fact-00000-9074ea0f.gz\n",
+      "Success\n",
+      "Downloading: assignment_override_dim-00000-47d7db7e.gz\n",
+      "Success\n",
+      "Downloading: assignment_override_fact-00000-1915e098.gz\n",
+      "Success\n",
+      "Downloading: assignment_override_user_dim-00000-77eb4caa.gz\n",
+      "Success\n",
+      "Downloading: assignment_override_user_fact-00000-7159d514.gz\n",
+      "Success\n",
+      "Downloading: assignment_override_user_rollup_fact-00000-5f6dd787.gz\n",
+      "Success\n",
+      "Downloading: assignment_rule_dim-00000-7225a11b.gz\n",
+      "Success\n",
+      "Downloading: communication_channel_dim-00000-eb7d1170.gz\n",
+      "Success\n",
+      "Downloading: communication_channel_fact-00000-958a1852.gz\n",
+      "Success\n",
+      "Downloading: conference_dim-00000-ef5be0e5.gz\n",
+      "Success\n",
+      "Downloading: conference_fact-00000-6d5c2eb2.gz\n",
+      "Success\n",
+      "Downloading: conference_participant_dim-00000-0719c5b0.gz\n",
+      "Success\n",
+      "Downloading: conference_participant_fact-00000-ecb9c07d.gz\n",
+      "Success\n",
+      "Downloading: conversation_dim-00000-56c32424.gz\n",
+      "Success\n",
+      "Downloading: conversation_message_dim-00000-da96a9e4.gz\n",
+      "Success\n",
+      "Downloading: conversation_message_participant_fact-00000-02075b76.gz\n",
+      "Success\n",
+      "Downloading: conversation_message_participant_fact-00001-3990abd5.gz\n",
+      "Success\n",
+      "Downloading: course_dim-00000-5b48adc7.gz\n",
+      "Success\n",
+      "Downloading: course_score_dim-00000-8eaf6477.gz\n",
+      "Success\n",
+      "Downloading: course_score_fact-00000-62e7aecc.gz\n",
+      "Success\n",
+      "Downloading: course_section_dim-00000-63a92a38.gz\n",
+      "Success\n",
+      "Downloading: course_ui_canvas_navigation_dim-00000-b419a2ad.gz\n",
+      "Success\n",
+      "Downloading: course_ui_navigation_item_dim-00000-098a3d0c.gz\n",
+      "Success\n",
+      "Downloading: course_ui_navigation_item_fact-00000-96eaa771.gz\n",
+      "Success\n",
+      "Downloading: discussion_entry_dim-00000-4b41808c.gz\n",
+      "Success\n",
+      "Downloading: discussion_entry_fact-00000-2f037f74.gz\n",
+      "Success\n",
+      "Downloading: discussion_topic_dim-00000-c6c192d6.gz\n",
+      "Success\n",
+      "Downloading: discussion_topic_fact-00000-55cfebee.gz\n",
+      "Success\n",
+      "Downloading: enrollment_dim-00000-37136d59.gz\n",
+      "Success\n",
+      "Downloading: enrollment_fact-00000-552e14f4.gz\n",
+      "Success\n",
+      "Downloading: enrollment_rollup_dim-00000-5405ffd0.gz\n",
+      "Success\n",
+      "Downloading: enrollment_term_dim-00000-bad49cc0.gz\n",
+      "Success\n",
+      "Downloading: external_tool_activation_dim-00000-903f1257.gz\n",
+      "Success\n",
+      "Downloading: external_tool_activation_fact-00000-7b6f11ad.gz\n",
+      "Success\n",
+      "Downloading: file_dim-00000-dda644c6.gz\n",
+      "Success\n",
+      "Downloading: file_fact-00000-1702d3ba.gz\n",
+      "Success\n",
+      "Downloading: group_dim-00000-6481df05.gz\n",
+      "Success\n",
+      "Downloading: group_fact-00000-d0c78601.gz\n",
+      "Success\n",
+      "Downloading: group_membership_dim-00000-10516d99.gz\n",
+      "Success\n",
+      "Downloading: group_membership_fact-00000-dd72bd7e.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_dim-00000-b66f806b.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_fact-00000-d835f148.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_group_association_fact-00000-a1c27ba1.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_group_dim-00000-f60e46ea.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_group_fact-00000-5b138b0a.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_question_result_dim-00000-48c8d782.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_question_result_fact-00000-c5cbb003.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_result_dim-00000-31c0018d.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_result_fact-00000-b043439d.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_rubric_criterion_dim-00000-ec2d4638.gz\n",
+      "Success\n",
+      "Downloading: learning_outcome_rubric_criterion_fact-00000-0746a294.gz\n",
+      "Success\n",
+      "Downloading: module_completion_requirement_dim-00000-690c6eb0.gz\n",
+      "Success\n",
+      "Downloading: module_completion_requirement_fact-00000-79cd62f9.gz\n",
+      "Success\n",
+      "Downloading: module_dim-00000-29e24c02.gz\n",
+      "Success\n",
+      "Downloading: module_fact-00000-e832627e.gz\n",
+      "Success\n",
+      "Downloading: module_item_dim-00000-4d586235.gz\n",
+      "Success\n",
+      "Downloading: module_item_fact-00000-63d1a901.gz\n",
+      "Success\n",
+      "Downloading: module_prerequisite_dim-00000-a0757c3c.gz\n",
+      "Success\n",
+      "Downloading: module_prerequisite_fact-00000-5aedc7a7.gz\n",
+      "Success\n",
+      "Downloading: module_progression_completion_requirement_dim-00000-74c4a53f.gz\n",
+      "Success\n",
+      "Downloading: module_progression_completion_requirement_fact-00000-b5f22af6.gz\n",
+      "Success\n",
+      "Downloading: module_progression_dim-00000-9fea8745.gz\n",
+      "Success\n",
+      "Downloading: module_progression_dim-00001-aaeab561.gz\n",
+      "Success\n",
+      "Downloading: module_progression_fact-00000-10b5fc11.gz\n",
+      "Success\n",
+      "Downloading: module_progression_fact-00001-04026d35.gz\n",
+      "Success\n",
+      "Downloading: pseudonym_dim-00000-da451682.gz\n",
+      "Success\n",
+      "Downloading: pseudonym_fact-00000-68410591.gz\n",
+      "Success\n",
+      "Downloading: quiz_dim-00000-e92dac6c.gz\n",
+      "Success\n",
+      "Downloading: quiz_fact-00000-c173e87e.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_answer_dim-00000-804f40ab.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_answer_dim-00001-68b50f49.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_answer_dim-00002-bfffd265.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_answer_fact-00000-c82d06a6.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_answer_fact-00001-2b0d9381.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_answer_fact-00002-0bbee5f7.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_dim-00000-533bac25.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_fact-00000-ebb35794.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_group_dim-00000-f6fffb18.gz\n",
+      "Success\n",
+      "Downloading: quiz_question_group_fact-00000-e00c4ae7.gz\n",
+      "Success\n",
+      "Downloading: quiz_submission_dim-00000-48e0508f.gz\n",
+      "Success\n",
+      "Downloading: quiz_submission_fact-00000-9124331b.gz\n",
+      "Success\n",
+      "Downloading: quiz_submission_historical_dim-00000-d426c6e7.gz\n",
+      "Success\n",
+      "Downloading: quiz_submission_historical_fact-00000-b0df0425.gz\n",
+      "Success\n",
+      "Downloading: requests-00000-8e6af56f.gz\n",
+      "Success\n",
+      "Downloading: requests-00001-790a397b.gz\n",
+      "Success\n",
+      "Downloading: role_dim-00000-63fdee5b.gz\n",
+      "Success\n",
+      "Downloading: submission_comment_dim-00000-aaf1fa9a.gz\n",
+      "Success\n",
+      "Downloading: submission_comment_fact-00000-fceab0e2.gz\n",
+      "Success\n",
+      "Downloading: submission_dim-00000-201559a3.gz\n",
+      "Success\n",
+      "Downloading: submission_dim-00001-b85b854b.gz\n",
+      "Success\n",
+      "Downloading: submission_dim-00002-7370540c.gz\n",
+      "Success\n",
+      "Downloading: submission_dim-00003-d8a31bf9.gz\n",
+      "Success\n",
+      "Downloading: submission_dim-00004-e1862304.gz\n",
+      "Success\n",
+      "Downloading: submission_fact-00000-2744be21.gz\n",
+      "Success\n",
+      "Downloading: submission_fact-00001-84354ca8.gz\n",
+      "Success\n",
+      "Downloading: submission_fact-00002-a9c00364.gz\n",
+      "Success\n",
+      "Downloading: submission_fact-00003-082011f6.gz\n",
+      "Success\n",
+      "Downloading: submission_fact-00004-5ff7a0d4.gz\n",
+      "Success\n",
+      "Downloading: submission_file_fact-00000-bc8d0ad1.gz\n",
+      "Success\n",
+      "Downloading: user_dim-00000-6eee8f8c.gz\n",
+      "Success\n",
+      "Downloading: wiki_dim-00000-07eef6e0.gz\n",
+      "Success\n",
+      "Downloading: wiki_fact-00000-252b3fbb.gz\n",
+      "Success\n",
+      "Downloading: wiki_page_dim-00000-45b46d1b.gz\n",
+      "Success\n",
+      "Downloading: wiki_page_fact-00000-4e658212.gz\n",
+      "Success\n",
+      "Downloading: requests-00000-60d4d55e.gz\n",
+      "Success\n",
+      "Downloading: requests-00001-b9f02622.gz\n",
+      "Success\n",
+      "Downloading: requests-00000-a1e875d5.gz\n",
+      "Success\n",
+      "Downloading: requests-00001-0c8a9e5e.gz\n",
+      "Success\n",
+      "Downloading: requests-00000-e1d4452b.gz\n",
+      "Success\n",
+      "Downloading: requests-00001-a8ff7754.gz\n",
+      "Success\n",
+      "Downloading: requests-00000-953a589b.gz\n",
+      "Success\n",
+      "Downloading: requests-00001-9e152530.gz\n",
+      "Success\n",
+      "Out of 119 files, 119 succeeded and 0 failed.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pipelines\n",
+    "pipelines.sync_non_interactive()\n"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "dir(pipelines)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import pipelines\n",
+    "print(dir(pipelines))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import plotly.graph_objects as go\n",
+    "\n",
+    "import networkx as nx\n",
+    "\n",
+    "G = nx.random_geometric_graph(50, 0.125)\n",
+    "\n",
+    "\n",
+    "edge_x = []\n",
+    "edge_y = []\n",
+    "for edge in G.edges():\n",
+    "    x0, y0 = G.nodes[edge[0]]['pos']\n",
+    "    x1, y1 = G.nodes[edge[1]]['pos']\n",
+    "    edge_x.append(x0)\n",
+    "    edge_x.append(x1)\n",
+    "    edge_x.append(None)\n",
+    "    edge_y.append(y0)\n",
+    "    edge_y.append(y1)\n",
+    "    edge_y.append(None)\n",
+    "\n",
+    "edge_trace = go.Scatter(\n",
+    "    x=edge_x, y=edge_y,\n",
+    "    line=dict(width=0.5, color='#888'),\n",
+    "    hoverinfo='none',\n",
+    "    mode='lines')\n",
+    "\n",
+    "node_x = []\n",
+    "node_y = []\n",
+    "for node in G.nodes():\n",
+    "    x, y = G.nodes[node]['pos']\n",
+    "    node_x.append(x)\n",
+    "    node_y.append(y)\n",
+    "\n",
+    "node_trace = go.Scatter(\n",
+    "    x=node_x, y=node_y,\n",
+    "    mode='markers',\n",
+    "    hoverinfo='text',\n",
+    "    marker=dict(\n",
+    "        showscale=True,\n",
+    "        # colorscale options\n",
+    "        #'Greys' | 'YlGnBu' | 'Greens' | 'YlOrRd' | 'Bluered' | 'RdBu' |\n",
+    "        #'Reds' | 'Blues' | 'Picnic' | 'Rainbow' | 'Portland' | 'Jet' |\n",
+    "        #'Hot' | 'Blackbody' | 'Earth' | 'Electric' | 'Viridis' |\n",
+    "        colorscale='YlGnBu',\n",
+    "        reversescale=True,\n",
+    "        color=[],\n",
+    "        size=10,\n",
+    "        colorbar=dict(\n",
+    "            thickness=15,\n",
+    "            title='Node Connections',\n",
+    "            xanchor='left',\n",
+    "            titleside='right'\n",
+    "        ),\n",
+    "        line_width=2))\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "fig = go.Figure(data=[edge_trace, node_trace],\n",
+    "             layout=go.Layout(\n",
+    "                title='<br>Network graph made with Python',\n",
+    "                titlefont_size=16,\n",
+    "                showlegend=False,\n",
+    "                hovermode='closest',\n",
+    "                margin=dict(b=20,l=5,r=5,t=40),\n",
+    "                annotations=[ dict(\n",
+    "                    text=\"Python code: <a href='https://plotly.com/ipython-notebooks/network-graphs/'> https://plotly.com/ipython-notebooks/network-graphs/</a>\",\n",
+    "                    showarrow=False,\n",
+    "                    xref=\"paper\", yref=\"paper\",\n",
+    "                    x=0.005, y=-0.002 ) ],\n",
+    "                xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),\n",
+    "                yaxis=dict(showgrid=False, zeroline=False, showticklabels=False))\n",
+    "                )\n",
+    "fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Gav and DB env",
+   "language": "python",
+   "name": "dbs"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/outcomes.py b/outcomes.py
new file mode 100644
index 0000000..82bf883
--- /dev/null
+++ b/outcomes.py
@@ -0,0 +1,1340 @@
+
+
+import requests, json, codecs, csv, re, sys
+from collections import defaultdict
+
+from pipelines import fetch, url, header
+from courses import getCoursesInTerm, getTerms
+
+from concurrent.futures import ThreadPoolExecutor
+
+
+f = codecs.open('cache/slo/log.txt','w','utf-8')
+
+VERBOSE = 1
+
+SLO_CURRENT_SOURCE = 'cache/slo/2018_slo.csv'          # term 21
+#SLO_CURRENT_SOURCE = 'cache/slo/2020_slo.csv'
+
+
+## Outcomes import format looks like this
+## https://canvas.instructure.com/doc/api/file.outcomes_csv.html
+
+# vendor_guid,object_type,title,description,display_name,calculation_method,calculation_int,workflow_state,parent_guids,ratings,,,,,,,
+# a,group,Parent group,parent group description,G-1,,,active,,,,,,,,,
+# b,group,Child group,child group description,G-1.1,,,active,a,,,,,,,,
+# c,outcome,Learning Standard,outcome description,LS-100,decaying_average,40,active,a b,3,Excellent,2,Better,1,Good,,
+
+def outcome_overview(term=21):
+    d = input("Pick a department. Enter the short code for it: ")
+    d = d.upper()
+    
+    account_level,dept_details = outcome_groups()
+    local_source = slo_source_by_dept()
+    this_term,sections_by_course, section_to_cid = outcomes_attached_to_courses(term,d)
+    f.write("\n\n* * * * * * * *\nACCOUNT LEVEL GROUPS & OUTCOMES\n")
+    f.write(json.dumps(account_level,indent=4))
+    f.write("\n\n* * * * * * * *\LOCAL OUTCOMES ON FILE\n")
+    f.write(json.dumps(local_source,indent=4))
+    f.write("\n\n* * * * * * * *\OUTCOMES ATTACHED TO COURSES IN THIS TERM\n")
+    f.write(json.dumps(this_term,indent=4))
+    f_t_s, s_t_f = x_ref_dept_names()
+    act = []
+    acct_folder_present = "Present"
+    if d in account_level: act = account_level[s_t_f[d]].keys() #
+    else: acct_folder_present = "Missing"
+    
+    all_sections = {}
+    act_col = {}
+    local_col = {}
+    course_col = {}
+    
+    report = [['Course','Account Lvl','Local Txt','Course Shell'],['------','------','------','------',]]
+    report.append( ['Department folder', acct_folder_present, '','' ] )
+    big_union = list(set().union( *[ this_term[d].keys(), act, local_source[d].keys()] ))
+    big_union.sort()
+    for C in big_union:
+        row = [ C, ]
+        if d in account_level and C in account_level[d]:
+            act_col[C] = str(len(account_level[d][C])-1)+" outcomes"
+            row.append(str(len(account_level[d][C])-1)+" outcomes")
+        else:
+            act_col[C] = "NO"
+            row.append("No")
+        if d in local_source and C in local_source[d]:
+            local_col[C] = str(len(local_source[d][C]))+" outcomes"
+            row.append(str(len(local_source[d][C]))+" outcomes")
+        else:
+            local_col[C] = "NO"
+            row.append('No')
+        row.append('')  # handle sections below
+        report.append(row)
+        
+        if C in sections_by_course.keys():
+            for S in sections_by_course[C]:
+                report.append( ['  '+S,'','',str(len(sections_by_course[C][S]))+" outcomes"] )
+                all_sections[S] = sections_by_course[C][S]
+    
+    print("Semester is: Spring 2017")
+    for R in report:
+        print("{: <20} {: >20} {: >20} {: >20}".format(*R))
+    
+    if acct_folder_present == 'Missing':
+        a = input("Department's outcome group is missing. Add it? (y/n) ")
+        if a == 'y':
+            create_dept_group(d)
+            account_level,dept_details = outcome_groups()
+    action = ''
+    while action != 'q':
+        f.flush()
+        print("\n------\nOptions:")
+        print("   a {classcode}    -  Create account level outcomes, using local outcomes")
+        print("   c {section num}  -  Connect the acct level outcomes to a particular course")
+        print("   d                -  Enter Account level outcomes for the whole department")
+        print("   t                -  Connect outcomes to all sections in this term")
+        print("   p                -  Pick new department")
+        print("   q                -  quit")
+        
+        action = input('> ')
+        a = re.search('^(\D)\s?(.*)$',action)
+        if a:
+            if a.group(1) == 'a':
+                classcode = a.group(2)
+                print("Creating outcomes for: " + classcode)
+                if classcode in account_level[d]:                       # whether or not there's a dept folder already...
+                    print(account_level[d][classcode]['outcome_group'])
+                    create_acct_lvl_outcomes(local_source[d][classcode],dept_details[d],'',str(account_level[d][classcode]['outcome_group']['id']))
+                else:
+                    create_acct_lvl_outcomes(local_source[d][classcode],dept_details[d],classcode)
+            elif a.group(1) == 'c':
+                section = a.group(2)
+                print(sections_by_course)
+                this_course = ''
+                # need to find the course that this section corresponds to
+                for C in sections_by_course.keys():
+                    if section in sections_by_course[C].keys():
+                        this_course = C
+                f.write("\n--------> SECTIONS BY COURSE -------->\n\n" + json.dumps(sections_by_course,indent=2))
+                f.write("\n--------> ALL SECTIONS -------->\n\n" + json.dumps(all_sections,indent=2))
+                f.write("\n--------> OC GROUPS THIS DEPT -------->\n\n" + json.dumps(account_level[d],indent=2))
+                if this_course:
+                    print("Connecting outcomes for course: " + this_course + ", to section: " + section)
+                try:
+                    connect_acct_oc_to_course(section_to_cid[section], str(account_level[d][this_course]['outcome_group']['id']))
+                except KeyError as e:
+                    print("Couldn't do this because there don't appear to be account level outcomes for " + str(d))
+            elif a.group(1) == 'd':     # put in acct level OCs for all classes in dept's local
+                for classcode in local_source[d]:
+                    print("Creating outcomes for: " + classcode)
+                    if classcode in account_level[d]:                       # whether or not there's a dept folder already...
+                        print(account_level[d][classcode]['outcome_group'])
+                        create_acct_lvl_outcomes(local_source[d][classcode],dept_details[d],'',str(account_level[d][classcode]['outcome_group']['id']))
+                    else:
+                        create_acct_lvl_outcomes(local_source[d][classcode],dept_details[d],classcode)
+            elif a.group(1) == 't':
+                #for C in local_source[d]:
+                i = 0
+                for C in sections_by_course.keys():
+                    if i > 10: break
+                    print('Finding sections for ' + C + ':')
+                    
+                    for S in sections_by_course[C]:
+                        print("Connecting outcomes for course: " + C + ", to section: " + S)
+                        if len(sections_by_course[C][S]):
+                            print("** looks like it already has some")
+                            continue
+                        try:
+                            connect_acct_oc_to_course(section_to_cid[S], str(account_level[d][C]['outcome_group']['id']))
+                            i += 1
+                        except KeyError as e:
+                            print("Couldn't do this because there don't appear to be account level outcomes for " + str(C))
+            elif a.group(1) == 'p':
+                action = 'q'
+                outcome_overview(term)
+                        
+                    
+
+            
+    """
+    
+    if d in this_term:
+        print("\n\nIn this terms courses:\n"+json.dumps(this_term[d].keys(),indent=4))
+    if d in account_level:
+        print("Account level:\n"+json.dumps(account_level[d].keys(),indent=4))
+    else:    
+        print("Missing department at account level. ")
+    if d in local_source:
+        print("\n\nLocal:\n"+json.dumps(local_source[d].keys(),indent=4))
+    """
+    
+def create_acct_lvl_outcomes(src,dept,makefolder='',folder=0):
+    print("these... ")
+    print(json.dumps(dept,indent=2))
+    print(json.dumps(src,indent=2))
+    
+    parent_group = str(dept['id'])
+    
+    if makefolder:
+        print("I need to make a folder for this course: " + makefolder + " with parent id=" + parent_group)
+        new_folder = create_course_group(makefolder,parent_group)
+        parent_group = str(new_folder['id'])
+    else:
+        parent_group = folder
+    
+    for this_outcome in src:
+        #this_outcome = src[2]
+        short_name = this_outcome[1] + ". " + " ".join(this_outcome[2].split(" ")[0:4]) + "..."
+        
+        parameters = { "title": short_name,
+               "display_name": short_name,
+               "description": this_outcome[2],
+               "mastery_points": 3,
+               }
+        t = url + '/api/v1/accounts/1/outcome_groups/'+parent_group+'/outcomes'
+        print(t)
+        #print(parameters)
+        print(json.dumps(parameters,indent=2))
+        r = requests.post(t,data=parameters, headers=header)
+        print(r.text)
+
+def connect_acct_oc_to_course(course_id,oc_group_id):
+    global results, results_dict
+    results_dict = {}
+    print("these... ")
+    print(json.dumps(course_id,indent=2))
+    print(json.dumps(oc_group_id,indent=2))
+    
+    # get course's outcome group id
+    results_dict = fetch(url + '/api/v1/courses/' + str(course_id) + '/root_outcome_group')
+    
+    print('Course outcome group:')
+    print(results_dict)
+    
+    og_id = str(results_dict['id'])
+        
+    # get all the account level outcomes for this course
+    these_outcomes = fetch(url + '/api/v1/accounts/1/outcome_groups/' + oc_group_id + '/outcomes?outcome_style=full')
+    print('Outcomes in account level group for course:')
+    print(these_outcomes)
+    
+    for o in these_outcomes:
+        o_id = str(o['outcome']['id'])
+        
+        t = url + '/api/v1/courses/' + str(course_id) + '/outcome_groups/' + og_id + '/outcomes/' + o_id
+        printt
+        r = requests.put(t, headers=header)
+        print(r.text )
+    
+def outcome_groups_dump():
+    print("Loading account level outcomes..." )
+    top_level = fetch(url + '/api/v1/accounts/1/outcome_groups', VERBOSE)
+    codecs.open('cache/slo/outcome_groups.json','w','utf-8').write( json.dumps(top_level,indent=2))
+
+
+def outcome_groups_backup():
+    top_level = fetch(url + '/api/v1/accounts/1/outcome_groups')
+    course_groups = defaultdict( list )
+    dept_groups = defaultdict( list )
+    
+    # Arranging the "groups" into a structure
+    for O in top_level:
+        if isinstance(O, dict):
+            parent = 0
+            if 'parent_outcome_group' in O: 
+                parent = O['parent_outcome_group']['id']
+                if O['parent_outcome_group']['id'] == 1:
+                    dept_groups[O['id']].append(O)
+                else:
+                    course_groups[O['parent_outcome_group']['id']].append(O)
+    
+    # Add actual outcomes to the structure
+    group_links = fetch(url + '/api/v1/accounts/1/outcome_group_links')
+
+    for G in group_links:
+        if 'outcome' in G:
+            # print(str(G['outcome']['id']) + "\t" + str(G['outcome_group']['id']) + "\t" + str(G['outcome']['title']) + "\t")
+            parent = G['outcome_group']['id']
+            course_groups[parent].append(G)
+    
+    # Traverse the tree and print it
+    for D in dept_groups.keys():
+        print("Department: " + dept_groups[D][0]['title'])
+        dept_id = dept_groups[D][0]['id']
+        for C in course_groups[dept_id]:
+            print("  Course: " + C['title'])
+            course_id = C['id']
+            for O in course_groups[course_id]:
+                print("    " + str(O['outcome']['id']) + "\t" + O['outcome']['title'])
+    return dept_groups
+
+def create_course_group(short,parent):
+    t = url + '/api/v1/accounts/1/outcome_groups/'+parent+'/subgroups'
+    
+    new_group = {'title': short }
+    data = json.dumps(new_group)
+    
+    print(t)
+    r = requests.post(t,data=new_group, headers=header)
+    result = json.loads(r.text)
+    print(r.text)
+    return result
+    
+def create_dept_group(short):
+    full_to_short_names, short_to_full = x_ref_dept_names()
+    print("Creating Dept-Level outcome group for " + short_to_full[short] + " (" + short + ")")
+    #a = input("Press return to continue or c to cancel.")
+    #if a == 'c': return
+    
+    t = url + '/api/v1/accounts/1/outcome_groups/1/subgroups'
+    
+    new_group = {'title': short_to_full[short] }
+    data = json.dumps(new_group)
+    
+    print(t)
+    r = requests.post(t,data=new_group, headers=header)
+    print(r.text)
+    
+def outcomes_attached_to_courses(term=65,limitdept=''):
+    # For each class in a term, check to see if it has outcomes and/or
+    # an outcome group attached to it. 
+    courses = getCoursesInTerm(term,show=0,active=0)
+    bycode = read_slo_source()
+    by_dept = defaultdict(dict)
+    sections_by_course = defaultdict(dict)
+    
+    section_to_courseid = {}
+    
+    print("Loading course-attached outcomes for this semester...")
+    
+    for R in courses:
+        results = []
+        #print(R)
+        id = R['id']
+        name = R['name']
+        b = ''
+        
+        
+        ### TODO Handle this:     CSIS/DM85 WEB DESIGN 40823/24
+        #  MCTV17A/18/THEA17
+        #  MUS4B/5ABCD
+        #  APE
+        # 
+        
+        # the dept/num combo is 'codeguess'
+        codeguess = name.split(' ')[0]
+        if '/' in codeguess:
+            a,b = fix_joined_class(codeguess)
+            codeguess = a
+        # get the dept name alone...
+        a = re.search('^(\D+)(\d+)(.*)$',codeguess)
+        if a:
+            dept = a.group(1)
+        else:
+            print(" ! Problem getting dept from: %s\n ! Original: %s" % (codeguess,name))
+            dept = 'unknown'
+        
+        # section number?
+        a = re.search('\s(\d+)$',name)
+        section = ''
+        if a:
+            section = a.group(1)
+        if limitdept:
+            if not dept == limitdept:
+                continue
+                
+        section_to_courseid[section] = str(id)
+        # look for outcomes in this term's course
+        results = fetch(url + '/api/v1/courses/' + str(id) + '/outcome_group_links')
+        if len(results): 
+            #print("  found online SLOs"    # summarize_course_online_slo(results))
+            print(results)
+            details = [ fetch(url + '/api/v1/outcomes/' + str(oo['outcome']['id']), VERBOSE) for oo in results ]
+            by_dept[dept][codeguess] = [results,details]
+            sections_by_course[codeguess][section] = [results,details]
+        else: 
+            by_dept[dept][codeguess] = []
+            sections_by_course[codeguess][section] = []
+            #print("  no online SLOs."  )
+    codecs.open('cache/slo/slo_attached_by_dept.json','w','utf-8').write( json.dumps(by_dept,indent=2))
+    codecs.open('cache/slo/slo_attached_by_course.json','w','utf-8').write( json.dumps(sections_by_course,indent=2))
+    codecs.open('cache/slo/slo_attached_sect_to_courseid.json','w','utf-8').write( json.dumps(section_to_courseid,indent=2))
+    return by_dept, sections_by_course, section_to_courseid
+    
+def summarize_course_online_slo(outcome_list):
+    print("Found online SLOs: ")
+    for o in outcome_list:
+        ass = ''
+        title = ''
+        dname = ''
+        if o['assessed'] == 'true': ass = "\t(has assessments) "
+        if 'display_name' in o['outcome']: dname = "\t " + str(o['outcome']['display_name'])
+        if 'title' in o['outcome']: title = "\t " + str(o['outcome']['title'])
+        print("  " + str(o['outcome']['id']) + ass + dname + title)
+        details = fetch_outcome_details(o['outcome']['id'])
+        print("  " + details['description'])
+        
+def fetch_outcome_details(id):
+    return fetch(url + '/api/v1/outcomes/' + str(id), VERBOSE)
+
+# Report on the actual evaluation data?        
+def outcome_report1():
+    output = open('cache/slo/report.txt','w')
+    # first, get all classes in a term, then filter to published classes
+    #t = url + "/api/v1/accounts/1/courses?published=true&enrollment_term_id=18"
+    #while t: t = fetch(t)
+    results = [ {'id':1697,'course_code':'anth5 10407'},{'id':1825,'course_code':'anth1 10398'},{'id':2565,'course_code':'csis8 10705'}]
+    for c in results:
+        oc_t = url + '/api/v1/courses/' + str(c['id']) + '/outcome_results'
+        while oc_t: oc_t = fetch_dict(oc_t) # TODO
+        if len(results_dict['outcome_results']):
+            print(c['id'], "\t", c['course_code'])
+            output.write( "\t".join([str(c['id']), c['course_code'], "\n"]))
+            num_entries = len(results_dict['outcome_results'])
+            total_score = 0
+            by_student = {}
+            for R in results_dict['outcome_results']:
+                usr = R['links']['user']
+                print(usr)
+                print(R)
+                if usr in by_student: by_student[usr] += 1
+                else: by_student[usr] = 1
+                total_score += R['score']
+            num_students = len(by_student.keys())
+            average_score = total_score / (5.0 * num_entries)
+            output.write("Total Students: " + str(num_students))
+            output.write("\nTotal Entries: " + str(num_entries))
+            output.write("\nAverage Score: " + str(average_score) + "\n\n")
+
+
+
+# For the given course, get all outcome measurements, and display scores and stats.
+def outcome_report2():
+    print("Getting course level outcomes.")
+    output = open('cache/slo/report.txt','w')
+    res = [ {'id':1697,'course_code':'anth5 10407'},{'id':1825,'course_code':'anth1 10398'},{'id':2565,'course_code':'csis8 10705'}]
+    for c in res:
+        results = fetch(url + "/api/v1/courses/" + str(c['id']) + "/outcome_groups", VERBOSE)
+        for outcome in results:
+            f.write("Outcome groups\n")
+            f.write(json.dumps(outcome,indent=4))
+            if 'subgroups_url' in outcome:
+                f.write("\nfound subgroup. getting it: " + outcome['subgroups_url'])
+                t = url+outcome['subgroups_url']
+                rr = fetch(t, VERBOSE)
+                f.write("\nThis: \n" + json.dumps(rr,indent=4))
+            
+            
+            crs_oc_grps = fetch(url + '/api/v1/courses/' + str(c['id']) + '/outcome_results', VERBOSE)
+            
+            if len(crs_oc_grps['outcome_results']):
+                f.write( str(len(crs_oc_grps['outcome_results'])) + " results here.\n")
+                print(c['id'], "\t", c['course_code'])
+                output.write( "\t".join([str(c['id']), c['course_code'], "\n"]))
+                num_entries = len(crs_oc_grps['outcome_results'])
+                total_score = 0
+                by_student = {}
+                for R in crs_oc_grps['outcome_results']:
+                    usr = R['links']['user']
+                    print(usr)
+                    print(R)
+                    f.write( "\t".join([str(c['id']), c['course_code'], "\n"]))
+                    f.write(json.dumps(R,indent=4))
+                    if usr in by_student: by_student[usr] += 1
+                    else: by_student[usr] = 1
+                    total_score += R['score']
+                num_students = len(by_student.keys())
+                average_score = total_score / (5.0 * num_entries)
+                output.write("Total Students: " + str(num_students))
+                output.write("\nTotal Entries: " + str(num_entries))
+                output.write("\nAverage Score: " + str(average_score) + "\n\n")
+
+def fix_joined_class(str):
+    parts = str.split('/')
+    class_num = re.search(r'(\D+)(\d+\D?)', parts[1])
+    if class_num:
+        dept = class_num.group(1)
+        num = class_num.group(2)
+        return parts[0]+num, parts[1]
+    else:
+        class_num = re.search(r'(\D+)(\d+)\/(\d+\D?)',str)
+        if class_num:
+            dept = class_num.group(1)
+            num1 = class_num.group(2)
+            num2 = class_num.group(3)
+            return dept+num1, dept+num2
+        else:
+            class_num = re.search(r'(\D+)\/(\D+)\/(\D+)(\d+)',str)
+            if class_num:
+                dept1 = class_num.group(1)
+                dept2 = class_num.group(2)
+                num = class_num.group(4)
+                return dept1+num, dept2+num
+            else:
+                class_num = re.search(r'(\D+)(\d+)(\D)\/(\D)',str)
+                if class_num:
+                    dept = class_num.group(1)
+                    num = class_num.group(2)
+                    ltr1 = class_num.group(3)
+                    ltr2 = class_num.group(4)
+                    return dept+num+ltr1, dept+num+ltr2
+    print("can't guess courses on: " + str)
+    return "",""
+    
+def split_slo_name(str):
+    n_parts = str.split(r' - ')
+    code = n_parts[0]
+    title = n_parts[1]
+    m = re.search(r'^([^\d]+)(\d+)$',code)
+    dept = ''
+    if m:
+        dept = m.groups()[0].rstrip()
+    return dept, title, code
+        
+def outcome_report3():
+    output = open('cache/slo/report.txt','w')
+    # with an uppercase dept abbreviation, look up the outcome group
+    #a = input("Department code (ex: AMT): ")
+    code_to_name = {}
+    name_to_code = {}
+    dnames = open('cache/slo/dept_names.csv','r').readlines()
+    for DN in dnames:
+        DN = DN.rstrip()
+        pts = DN.split(',')
+        code_to_name[pts[0]] = pts[1]
+        name_to_code[pts[1]] = pts[0]
+    print(json.dumps(name_to_code,indent=4))
+    
+    # get the main canvas slo subgroups
+    t = url + '/api/v1/accounts/1/outcome_groups'
+    while(t): t = fetch(t)
+    top_level = results
+    for TL in top_level:
+        if 'parent_outcome_group' in TL and TL['parent_outcome_group']['id']==1:
+            if TL['title'] in name_to_code:
+                print("Matched: " + TL['title'] + " " + name_to_code[TL['title']])
+            #else:
+            #    print("Didn't match: " + json.dumps(TL,indent=4))
+        else:
+            print("Not top level: " + TL['title'])
+    sample = ['DM61 - 3D Animation,3,Student will analyze character movements and synthesize necessary joints and kinematics for realistic animation.,"project, performance"']
+    rd = csv.reader(sample)
+    for row in rd:
+        (dept,title,code) = split_slo_name(row[0])
+        full_dept = "NOT FOUND"
+        if dept in code_to_name: full_dept = code_to_name[dept]
+        print("dept: " + dept)
+        print("dept long: " + full_dept)
+        print("title: " + title)
+        print("code: " + code)
+        print("number in course: " + row[1])
+        print("text of slo: " + row[2])
+        print("assessment: " + row[3])
+        
+def read_slo_source():
+    f = open(SLO_CURRENT_SOURCE,'r')
+    fr = csv.reader(f)
+    i=0
+    bycode = defaultdict(list)
+    for row in fr:
+        if i:
+            (d, t, c) = split_slo_name(row[0])
+            c = c.replace(" ","")
+            #print(d + "\t" + c + "\t" + row[1])
+            bycode[c].append(row)
+        i += 1
+    #print(json.dumps(bycode,indent=2))
+    return bycode
+
+def slo_source_by_dept():
+    bycode = read_slo_source()
+    bydept = defaultdict(dict)
+    for code in bycode.keys():
+        a = re.search('(\D+)(\d+)',code)
+        if a:
+            dept = a.group(1)
+            num  = a.group(2)
+            bydept[dept][code] = bycode[code]
+        else:
+            print("Couldn't interpret: " + code)
+    return bydept
+    
+
+def printj(j):
+    print( json.dumps(j, indent=2) )
+    
+def writej(o,j):    
+    o.write(json.dumps(j,indent=2))
+    o.write('\n')
+    o.flush()
+    
+# Get root outcome group
+def root_og():
+    f = url + '/api/v1/global/root_outcome_group' 
+    g = fetch(f)
+    printj(g)
+    return g
+
+
+def recur_og():
+    output = codecs.open('cache/outcomes_log.txt','w','utf-8')
+    #all = []
+    #r = root_og()
+    recur_main(output)
+    
+
+def recur_main(out,g_url=""):
+    if not g_url: 
+        g_url = url + '/api/v1/global/root_outcome_group'
+    print('fetching: %s' % g_url)
+    g = fetch(g_url,1)
+    printj(g)
+    writej(out,g)
+    
+    if "subgroups_url" in g:
+        print('Subgroups: ' + g['subgroups_url'])
+        for S in fetch(url+g["subgroups_url"]):
+            recur_main(S)
+    if "outcomes_url" in g:
+        print('Outcomes: ' + g['outcomes_url'])
+        for S in fetch(url+g["outcomes_url"]): 
+            recur_main(S)
+    out.write('\n')
+    print()
+
+
+def recur2(out,og={}):
+    if not og: 
+        return
+    if "subgroups_url" in og:
+        print('Subgroups: ' + og['subgroups_url'])
+        for S in fetch(url+og["subgroups_url"],1):
+            printj(S)
+            writej(out,S)
+            recur2(out,S)
+    if "outcomes_url" in og:
+        print('Outcomes: ' + og['outcomes_url'])
+        for S in fetch(url+og["outcomes_url"],1):
+            printj(S)
+            writej(out,S)
+            recur2(out,S)
+    out.write('\n')
+    print()
+
+
+
+def all_og():
+    output = codecs.open('cache/outcomes_log.txt','w','utf-8')
+    f = url + '/api/v1/accounts/1/outcome_groups'
+    g = fetch(f,1)
+    for OG in g:
+        printj(g)
+        writej(output,g)
+        recur2(output,OG)
+ 
+
+NUM_THREADS = 10
+
+
+
+def course_slo_getter(q):
+    i = q["i"]
+    folder = q["folder"]
+    imgsrc = q["url"]
+    total = q["total"]
+    (head,tail) = os.path.split(imgsrc)
+    
+    if os.path.exists( os.path.join(folder,tail) ):
+        print("  + Image %i was already downloaded." % i)
+        return
+    
+    print("  Image %i / %i, folder %s, getting %s" % (i,total,folder,imgsrc))
+    r = requests.get(imgsrc,stream=True)
+    if r.status_code == 200:
+        with open(os.path.join(folder,tail),'wb') as f:
+            r.raw.decode_content = True
+            shutil.copyfileobj(r.raw, f)
+            print("  + Done with image %i." % i)
+            time.sleep(0.75)
+    else:
+        print("  - Failed with image %i." % i)
+
+
+results = []
+
+def threaded_getter():
+    global results
+    threads = []
+    qqueue = []
+    i = 0
+
+    for img in soup.select('a[href] img'):
+        link = img.find_parent('a', href=True)
+        qqueue.append( {"i":i,"folder":folder,"url":fix_url(url,link['href']) } )
+        i += 1
+    
+    print("There are %i images to fetch." % len(qqueue))
+    
+    pool = ThreadPoolExecutor(max_workers=NUM_THREADS)
+    
+    for q in qqueue:
+        q["total"] = len(qqueue)
+        results.append( pool.submit(course_slo_getter, q) )
+
+
+
+# Creating outcomes with scale
+"""
+curl 'https://<canvas>/api/v1/accounts/1/outcome_groups/1/outcomes.json' \
+     -X POST \
+     --data-binary '{
+           "title": "Outcome Title",
+           "display_name": "Title for reporting",
+           "description": "Outcome description",
+           "vendor_guid": "customid9000",
+           "mastery_points": 3,
+           "ratings": [
+             { "description": "Exceeds Expectations", "points": 5 },
+             { "description": "Meets Expectations", "points": 3 },
+             { "description": "Does Not Meet Expectations", "points": 0 }
+           ]
+         }' \
+     -H "Content-Type: application/json" \
+     -H "Authorization: Bearer <token>"
+"""
+
+
+def demo_o_fetch():
+    print(fetch_outcome_details('269'))
+    print(fetch_outcome_details('270'))
+
+
+
+
+def outcome_groups_2021():
+    csvfile = codecs.open('cache/slo/ilearn_compact.csv','w','utf-8')
+    csvwriter = csv.writer(csvfile)
+    csvwriter.writerow('id parentid title type desc'.split(' '))
+    
+    print("Loading account level outcomes..." )
+        
+    # id, parentid, title, type, description
+    
+    #top_level = json.loads(codecs.open('cache/slo/outcome_groups.json','r','utf-8').read())
+    top_level = fetch(url + '/api/v1/accounts/1/outcome_groups', VERBOSE)
+    by_dept = defaultdict(dict)
+    dept_details = {}
+    
+    all_outcomes = {}
+    #course_groups = defaultdict( list )
+    #dept_groups = defaultdict( list )
+    
+    group_id_to_dept = {}
+    group_id_to_course = {}
+    
+    full_to_short_names, short_to_full_names = x_ref_dept_names()
+    
+    # Arranging the "groups" into a structure
+    for O in top_level:
+        if isinstance(O, dict):
+            if 'parent_outcome_group' in O: 
+                if O['parent_outcome_group']['id'] == 1:
+                    group_id_to_dept[str(O['id'])] = full_to_short_names[ O['title'] ]
+                    by_dept[ full_to_short_names[O['title']]] = {}
+                    dept_details[ full_to_short_names[O['title']] ] = O
+                else:    
+                    group_id_to_course[ str(O['id']) ] = O['title']
+    # repeat after dept names are gathered.
+    for O in top_level:
+        if isinstance(O, dict):
+            if 'parent_outcome_group' in O: 
+                if O['parent_outcome_group']['id'] != 1:
+                    parent_id = str(O['parent_outcome_group']['id'])
+                    by_dept[ group_id_to_dept[ parent_id ] ][O['title']] = {'outcome_group':O}
+    #print(json.dumps(by_dept, indent=4))
+    #print(json.dumps(group_id_to_dept, indent=4) )
+    #print(json.dumps(group_id_to_course, indent=4) )
+    # Add actual outcomes to the structure
+    
+    results = fetch('/api/v1/accounts/1/outcome_group_links', VERBOSE)
+    
+    for G in results:
+        if 'outcome' in G:
+            # find the dept and parent ...
+            # this depends on every outcome (group) having a different name
+            #print(G)
+            details = fetch(url + '/api/v1/outcomes/' + str(G['outcome']['id']), VERBOSE) 
+            print(details)
+            try:
+                parent = group_id_to_course[ str(G['outcome_group']['id']) ]
+                details['parent'] = parent
+            except:
+                print("can't find a course for this outcome group: ", G['outcome_group']['id'])
+                details['parent'] = 'unknown'
+                continue
+            all_outcomes[ details['id'] ] = details
+            dept = ''
+            crs = ''
+            for D in by_dept.keys():
+                for C in by_dept[D].keys():
+                    if C == parent:
+                        dept = D
+                        crs = C
+            #print(G['outcome']['title'] )
+            #print("Parent: " + parent + "\n\n")
+            by_dept[dept][crs][G['outcome']['display_name']] = G
+    f.write("\n\n+++++++++++ DEPT DETAILS\n\\n" + json.dumps(dept_details, indent=4)+"\n\n")
+    f.write("\n\n+++++++++++ SLOS BY DEPT\n\\n" + json.dumps(by_dept, indent=4)+"\n\n")
+    codecs.open('cache/slo/all_canvas_outcomes.json','w','utf-8').write( json.dumps(by_dept,indent=2))
+    codecs.open('cache/slo/canvas_outcomes_list.json','w','utf-8').write( json.dumps(all_outcomes,indent=2))
+    return by_dept, dept_details
+    
+
+
+def x_ref_dept_names():
+    full_to_short_names = {}
+    short_to_full = {}
+    for L in open('cache/slo/dept_names.csv','r').read().split('\n'):
+        parts = L.split(',')
+        full_to_short_names[parts[1]] = parts[0]
+        short_to_full[parts[0]] = parts[1]
+    return full_to_short_names, short_to_full
+
+
+
+
+## 2023 Updated Work
+    
+call_num = 1
+groups_queue = {}
+outcomes_queue = {}
+groups_raw_log = codecs.open('cache/slo/rawlog.txt','w','utf-8')
+
+def all_outcome_results_in_term(termid=''):
+    terms = [171,172,174,176,178]
+    for t in terms:
+        print("\n\nTERM: ", str(t))
+        all_outcome_results_in_term_sub(str(t))
+
+
+
+def all_outcome_results_in_term_sub(termid=''):
+    if not termid:
+        termid = str(getTerms(printme=1, ask=1))
+    courses = getCoursesInTerm(term=termid,get_fresh=1,show=1,active=0)
+    log = codecs.open('cache/slo/assessed_slos_term_%s.txt' % str(termid),'w','utf-8')
+    items = {}
+    
+    for C in courses:
+        groups_raw_log.write(json.dumps(C,indent=2))
+        groups_raw_log.write("\n\n")
+        print(C['id'],C['name'])
+        res = fetch( url + '/api/v1/courses/%s/outcome_results' % str(C['id']))
+        items[C['id']] = res
+        groups_raw_log.write(json.dumps(res,indent=2))
+        groups_raw_log.write("\n\n")
+        groups_raw_log.flush()
+    log.write(json.dumps(items,indent=2))
+
+
+def all_linked_outcomes_in_term(termid=''):
+    terms = [172,174,176,178]
+    for t in terms:
+        all_linked_outcomes_in_term_sub(str(t))
+        
+        
+def all_linked_outcomes_in_term_sub(termid=''):
+    if not termid:
+        termid = str(getTerms(printme=1, ask=1))
+    courses = getCoursesInTerm(term=termid,get_fresh=0,show=1,active=0)
+    items = {}
+
+    csvfile = codecs.open('cache/slo/linked_slos_term_%s_compact.csv' % str(termid),'w','utf-8')
+    csvwriter = csv.writer(csvfile)
+    csvwriter.writerow('courseid coursename ogid oid vendorguid points mastery assessed desc'.split(' '))
+
+    
+    for C in courses:
+        log_obj = {'course':C,'og':[],'outcomes':[],'subgroups':[]}
+        items[C['id']] = log_obj
+        
+        groups_raw_log.write(json.dumps(C,indent=2))
+        groups_raw_log.write("\n\n")
+        print(C['id'],C['name'])
+        results_dict = fetch(url + '/api/v1/courses/%s/root_outcome_group' % str(C['id']))
+        log_obj['og'].append(results_dict)
+        groups_raw_log.write(json.dumps(results_dict,indent=2))
+        groups_raw_log.write("\n\n")
+        
+        # these_outcomes = fetch(url + '/api/v1/accounts/1/outcome_groups/' + oc_group_id + '/outcomes?outcome_style=full')
+        # these_outcomes = fetch(url + '/api/v1/accounts/1/outcome_groups/' + oc_group_id + '/outcomes?outcome_style=full')
+
+        u1 = url + '/api/v1/courses/%s/outcome_groups/%s/outcomes' % (str(C['id']), str(results_dict['id']))
+        groups_raw_log.write("\n" + u1 + "\n")
+        outcomes_list = fetch( u1 )
+        groups_raw_log.write(json.dumps(outcomes_list,indent=2))
+        groups_raw_log.write("\n\n")
+        
+        if 'errors' in outcomes_list:
+            continue
+        
+        if len(outcomes_list):
+            for oo in outcomes_list:
+                outcome = fetch( url + '/api/v1/outcomes/%s' % str(oo['outcome']['id']) )
+                log_obj['outcomes'].append(outcome)
+                csvwriter.writerow([C['id'], C['course_code'], results_dict['id'], outcome['id'], outcome['vendor_guid'], outcome['points_possible'], outcome['mastery_points'], outcome['assessed'], outcome['description']])
+                groups_raw_log.write(json.dumps(outcome,indent=2))
+                groups_raw_log.write("\n\n")
+                
+        
+        #"/api/v1/courses/12714/outcome_groups/6631/subgroups"
+        u2 = url + '/api/v1/courses/%s/outcome_groups/%s/subgroups' % (str(C['id']), str(results_dict['id']))
+        groups_raw_log.write("\n" + u2 + "\n")
+        g2 = fetch( u2 )
+        log_obj['subgroups'].append(g2)
+        groups_raw_log.write(json.dumps(g2,indent=2))
+        groups_raw_log.write("\n\n")
+        
+        for subgroup in g2:
+            print("      doing subgroup id %s" % str(subgroup['id']))
+            u3 = url + '/api/v1/courses/%s/outcome_groups/%s/outcomes' % (str(C['id']), str(subgroup['id']))
+            groups_raw_log.write("\n" + u3 + "\n")
+            outcomes_list = fetch( u3 )
+            log_obj['subgroups'].append(outcomes_list)
+            groups_raw_log.write(json.dumps(outcomes_list,indent=2))
+            groups_raw_log.write("\n\n")
+
+            
+            if 'errors' in outcomes_list:
+                continue
+            
+            if len(outcomes_list):
+                for oo in outcomes_list:
+                    outcome = fetch( url + '/api/v1/outcomes/%s' % str(oo['outcome']['id']) )
+                    log_obj['outcomes'].append(outcome)
+                    csvwriter.writerow([C['id'], C['course_code'], subgroup['id'], outcome['id'], outcome['vendor_guid'], outcome['points_possible'], outcome['mastery_points'], outcome['assessed'], outcome['description']])
+
+                    groups_raw_log.write(json.dumps(outcome,indent=2))
+                    groups_raw_log.write("\n\n")
+                    
+            
+        
+        csvfile.flush()
+        log = codecs.open('cache/slo/linked_slos_term_%s.txt' % str(termid),'w','utf-8')
+        log.write(json.dumps(items,indent=2))
+        log.close()
+        
+        
+        
+        
+        
+
+
+
+
+def assemblerow(g,parent=''):
+    # prep it for csv output
+    id = '-1'
+    pid = '-1'
+    ctype = ''
+    title = ''
+    dname = ''
+    desc = ''
+    guid = ''
+    if 'id' in g:
+        id = g['id']
+    if 'parent_outcome_group' in g:
+        pid = g['parent_outcome_group']
+    elif parent:
+        pid = parent
+        ctype = 'outcome'
+    if 'title' in g:
+        title = g['title']
+    if 'context_type' in g:
+        ctype = g['context_type']
+    if 'display_name' in g:
+        dname = g['display_name']
+    if 'description' in g:
+        desc = g['description']
+    if 'vendor_guid' in g:
+        guid = g['vendor_guid']
+    return [id, pid, guid, ctype, title, dname, desc]
+
+#g_url = url + '/api/v1/global/root_outcome_group'
+#g_url = url + '/api/v1/accounts/1/outcome_groups'
+
+
+def recur_full_fetch(out,g,parent=""):
+    global call_num
+    my_call_num = call_num 
+    call_num += 1
+    print("Start call # %i" % my_call_num)
+    
+    groups_raw_log.write(json.dumps(g,indent=2))
+    groups_raw_log.write("\n\n")
+    
+    row = assemblerow(g,parent)
+    print(row)
+    out.writerow(row)
+        
+    if "subgroups_url" in g:
+        print('Subgroups: ' + g['subgroups_url'])
+        oo = fetch(url+g["subgroups_url"])
+        for S in oo:
+            subgroup = fetch(url+S['url'])
+            print("  parent: ", row[0], "    sub   ",S)
+            recur_full_fetch(out,S,parent=row[0])
+    if "outcomes_url" in g:
+        print('Outcomes: ' + g['outcomes_url'])
+        oo = fetch(url+g["outcomes_url"])
+        groups_raw_log.write(json.dumps(oo,indent=2))
+        groups_raw_log.write("\n\n")
+
+        for S in oo: 
+            outcome = fetch(url+S['outcome']['url'])
+            print("    otc   ",S)
+            recur_full_fetch(out,outcome,parent=row[0])
+    print("Finished call # %i" % my_call_num)
+    print()
+
+
+# some curriqunet course versions don't have outcomes, and some versions
+# should be used even though they are historical. Given the course code (ACCT101)
+# return the appropriate cq course version.
+def find_cq_course_version(code):
+    ranks = csv.reader( codecs.open('cache/courses/all_courses_ranked.csv','r','utf-8') )
+    header = next(ranks)
+    historical_list = []
+    for row in ranks:
+        r = {header[i]: row[i] for i in range(len(header))}
+        if r['code'] == code:
+            if int(r['numoutcomes']) == 0: 
+                continue
+            if r['coursestatus'] == 'Historical':
+                historical_list.append(r)
+                continue
+            if r['coursestatus'] == 'Active':
+                return r['cqcourseid']
+    if len(historical_list):
+        return historical_list[0]['cqcourseid']
+    return 0
+
+def outcome_groups():
+    csvfile = codecs.open('cache/slo/ilearn_outcomes_and_groups.csv','w','utf-8')
+    csvwriter = csv.writer(csvfile)
+    csvwriter.writerow('id parentid guid type title displayname desc'.split(' '))
+    
+    print("Loading account level outcomes..." )
+    g_url = url + '/api/v1/accounts/1/outcome_groups/1'
+    print('fetching: %s' % g_url)
+    g = fetch(g_url,1)
+    recur_full_fetch(csvwriter,g)
+    
+def summary_string(s):
+    parts = s.split(" ")
+    return ' '.join(parts[:4]) + "..."
+
+
+"""
+def add_outcomes_course_id(canvas_id):
+
+        (dept,code,crn) = code_from_ilearn_name(C['name'])
+        if dept == 0:
+            non_matches.append(C['name'])
+            continue
+        cq_code = find_cq_course_version(code)
+        if cq_code:               # in cq_codes:
+"""
+
+
+
+def add_outcomes_course_code():
+    courses = "CMUN129 CSIS74 CSIS75 CSIS76 CSIS77 CSIS80 CSIS107 CSIS1 CSIS2 CSIS8".split(' ')
+    courses = "CSIS26 CSIS28 CSIS121 CSIS129 CSIS179 CSIS186".split(' ')
+    
+    for c in courses:
+        print("Adding outcomes to course: ", c)
+        add_outcomes_course_code_sub(c)
+
+def add_outcomes_course_code_sub(target_code='AJ184',term=178,fresh=0):
+
+    courses = getCoursesInTerm(term,get_fresh=fresh)
+    cq_course_code = 0
+    
+    for C in courses:
+        (dept,code,crn) = code_from_ilearn_name(C['name'])
+        if dept == 0:
+            continue
+        if code != target_code:
+            continue
+        cq_course_code = find_cq_course_version(code)
+        if cq_course_code:               
+            outcomes = []
+            ilearn_course_id = C['id']
+            print("Using cq course id: %s and ilearn course id: %s." % (str(cq_course_code),str(ilearn_course_id)))
+            
+            f = codecs.open('cache/courses/alloutcomes.csv','r','utf-8')
+            r = csv.reader(f)
+            header = next(r)
+            for row in r:
+                if row[1] == cq_course_code:
+                    row_dict = {header[i]: row[i] for i in range(len(header))}
+                    outcomes.append(row_dict)
+            print("   Got ", len(outcomes), " outcomes")
+            
+            quick_add_course_outcomes(ilearn_course_id, outcomes)
+
+        else:
+            print("Didn't find course %s in term %s.", (target_code,str(term)))
+            return 0
+
+
+
+    
+def add_csis_sp22():
+    search = find_cq_course_version('CSIS6')     #  ENGL1B')    #'CSIS6'
+    
+    print(search)
+    return
+    
+    outcomes = []
+    ilearn_course_id = '14853'
+    f = codecs.open('cache/courses/all_active_outcomes.csv','r','utf-8')
+    r = csv.reader(f)
+    header = next(r)
+    for row in r:
+        if row[0] == search:
+            print(row)
+            row_dict = {header[i]: row[i] for i in range(len(header))}
+            outcomes.append(row_dict)
+    print(row_dict)
+    
+    quick_add_course_outcomes(ilearn_course_id, outcomes)
+
+def quick_add_course_outcomes(ilearn_course_id, cq_outcome_id_list):
+    print("   Fetching course id %s..." % str(ilearn_course_id))
+    course_root_og = fetch(url + '/api/v1/courses/%s/root_outcome_group' % str(ilearn_course_id))
+    course_og_id = course_root_og['id']
+
+    for o in cq_outcome_id_list:
+        parameters = [ 
+            ("title", summary_string(o['outcome'])) ,
+            ("display_name", summary_string(o['outcome'])),
+            ("description", o['outcome']),
+            ("mastery_points", 2),
+            ("ratings[][description]", 'Exceeds Expectations'),
+            ("ratings[][points]", 3),
+            ("ratings[][description]", 'Meets Expectations'),
+            ("ratings[][points]", 2),
+            ("ratings[][description]", 'Partially Meets Expectations'),
+            ("ratings[][points]", 1),
+            ("ratings[][description]", 'Does Not Meet Expectations'),
+            ("ratings[][points]", 0),
+            ("vendor_guid", o['cqoutcomeid'])]
+
+        t = url + '/api/v1/courses/%s/outcome_groups/%s/outcomes' % (str(ilearn_course_id), str(course_og_id))
+        r = requests.post(t,data=parameters, headers=header)
+        result = json.loads(r.text)
+        new_outcome_id = result['outcome']['id']
+        print("   Added outcome: ", o['outcome'])
+    print()
+    
+        
+
+def stringpad(s,n):
+    if len(s) >= n:
+        return " " + s[:n-1]
+    pad = n - len(s)
+    return " "*pad + s
+    
+    
+def code_from_ilearn_name(n,verbose=0):
+    parts = n.split(" ")
+    code = parts[0]
+    crn = parts[-1]
+    dept = ''
+    num = ''
+    v = verbose
+
+    ### CUSTOM MATCHES ###
+    customs = [
+        ('HUM/MCTV AUDIO / CINEMA / MOTION PRODUCTION FA22', ("HUM","HUM25",crn)),
+        ('HUM25/MCTV6/MCTV26', ("HUM","HUM25",crn)),
+        ('MUS4B-5D', ("MUS","MUS4B",crn)),
+        ('MUS4', ("MUS","MUS4B",crn)),
+        ('MUS4/5', ("MUS","MUS4B",crn)),
+        ('KIN64', ("KIN","KIN64A",crn)),
+        ('KIN24', ("KIN","KIN24A",crn)),
+        ('ART/CD25A/B', ("ART","ART25A",crn)),
+        ('ENGR10A', ("ENGR","ENGR10",crn)),
+    ]
+    for c in customs:
+        if n == c[0] or code == c[0]: 
+            if v: print("ilearn: ", stringpad(n, 35), " dept: ", stringpad(c[1][0],6), '   num: ', stringpad('',7), '   code: ', stringpad(c[1][1],11),  "  crn: ", stringpad(crn,9),  "  R: C", end='')
+            return c[1]
+
+
+
+    a = re.search('^([A-Z]+)(\d+[A-Z]?)$', code)
+    if a:
+        dept = a.group(1)
+        num = a.group(2)
+        if v: print("ilearn: ", stringpad(n, 35), " dept: ", stringpad(dept,6), '   num: ', stringpad(num,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 1", end='')
+        return (dept,code,crn)
+    
+    # two depts, with nums on each
+    a = re.search('^([A-Z]+)(\d+[A-Z]*)\/([A-Z]+)(\d+[A-Z]*)$', code)
+    if a:
+        dept1 = a.group(1)
+        num1  = a.group(2)
+        dept2 = a.group(3)
+        num2  = a.group(4)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 2", end='')
+        return (dept1,code,crn)
+    
+    # two depts, same num, two letters  "ART/CD25AB
+    a = re.search('^([A-Z]+)\/([A-Z]+)(\d+)([A-Z])([A-Z])$', code)
+    if a:
+        dept1 = a.group(1)
+        dept2 = a.group(2)
+        num1  = a.group(3)+a.group(4)
+        num2  = a.group(3)+a.group(5)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 8", end='')
+        return (dept1,code,crn)
+    
+    # two depts, same num
+    a = re.search('^([A-Z]+)\/([A-Z]+)(\d+[A-Z]*)$', code)
+    if a:
+        dept1 = a.group(1)
+        dept2 = a.group(2)
+        num1  = a.group(3)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 3", end='')
+        return (dept1,code,crn)
+    
+    # three depts, same num
+    a = re.search('^([A-Z]+)\/([A-Z]+)\/([A-Z]+)(\d+[A-Z]*)$', code)
+    if a:
+        dept1 = a.group(1)
+        dept2 = a.group(2)
+        dept3 = a.group(3)
+        num1  = a.group(4)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 4", end='')
+        return (dept1,code,crn)
+    
+    # one dept, two nums
+    a = re.search('^([A-Z]+)(\d+[A-Z]*)\/(\d+[A-Z]*)$', code)
+    if a:
+        dept1 = a.group(1)
+        num1  = a.group(2)
+        num2  = a.group(3)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 5", end='')
+        return (dept1,code,crn)
+    
+    # A/B/C
+    a = re.search('^([A-Z]+)(\d+[A-Z])([\/[A-Z]+)$', code)
+    if a:
+        dept1 = a.group(1)
+        num1  = a.group(2)
+        other = a.group(3)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 6", end='')
+        return (dept1,code,crn)
+    
+    # AB
+    a = re.search('^([A-Z]+)(\d+)([A-Z])([A-Z])$', code)
+    if a:
+        dept1 = a.group(1)
+        num1  = a.group(2)+a.group(3)
+        num2 = a.group(2)+a.group(4)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 7", end='')
+        return (dept1,code,crn)
+    
+    # KIN71A-C
+    a = re.search('^([A-Z]+)(\d+)([A-Z])\-([A-Z])$', code)
+    if a:
+        dept1 = a.group(1)
+        num1  = a.group(2)+a.group(3)
+        num2 = a.group(2)+a.group(4)
+        code = dept1+num1
+        if v: print("ilearn: ", stringpad(n, 35), "*dept: ", stringpad(dept1,6), '   num: ', stringpad(num1,7), '   code: ', stringpad(code,11),  "  crn: ", stringpad(crn,9),  "  R: 7", end='')
+        return (dept1,code,crn)
+    
+    
+    return (0,0,0)
+
+
+def parse_ilearn_course_names_ALLSEMESTERS():
+    log = codecs.open('cache/log_ilearn_course_names_parsing.txt','w','utf-8')
+    
+    for t in [25,26,60,61,62,63,64,65,168,171,172,173,174,175,176,177,178]:
+        parse_ilearn_course_names_sub(str(t),1,log)
+        
+
+
+def parse_ilearn_course_names(term='178',fresh=1,log=0):
+    non_matches = []
+
+    courses = getCoursesInTerm(term,get_fresh=fresh)
+    for C in courses:
+        (dept,code,crn) = code_from_ilearn_name(C['name'])
+        if dept == 0:
+            non_matches.append(C['name'])
+            continue
+        cq_code = find_cq_course_version(code)
+        if cq_code:               # in cq_codes:
+            print("   cq course id: ", cq_code)
+        else:
+            print("   NO CQ MATCH")
+            non_matches.append( [code,crn] )
+    
+    print("Non matches:")
+    if log: log.write("\n\n------------\nTERM: " + str(term) + "\n")
+    for n in non_matches:
+        print(n)
+        if log: log.write(str(n)+"\n")
+    print("can't figure out shortname for ", len(non_matches), " courses...")
+
+
+
+    
+if __name__ == "__main__":  
+    print ('')
+    options = { 1: ['all outcome results in a semester', all_outcome_results_in_term],
+                2: ['all linked outcomes/courses in a semester', all_linked_outcomes_in_term],
+                3: ['Main outcome show & modify for a semester', outcome_overview],
+                4: ['The outcome groups and links in iLearn',     outcome_groups],
+                5: ['Outcome report #2 sample', outcome_report2],
+                6: ['fetch root outcome group', root_og],
+                7: ['recurisively fetch outcomes', recur_og],
+                8: ['get all outcome groups', all_og],
+                9: ['demo get outcomes', demo_o_fetch],
+               10: ['demo post outcomes to course', add_outcomes_course_code],
+               11: ['match ilearn courses to cq courses', parse_ilearn_course_names],
+                
+                }
+    
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+        
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
+
+
+
+
+
diff --git a/outcomes2022.py b/outcomes2022.py
new file mode 100644
index 0000000..931c39b
--- /dev/null
+++ b/outcomes2022.py
@@ -0,0 +1,130 @@
+# Outcomes 2023
+
+# Tasks:
+#
+# - List all courses (in semester) in iLearn:
+#     + SLOs associated with the course
+#     + Whether they are current or inactive
+#     + Whether they are attached to an assessment
+#     + Whether and by how many students, they have been assessed
+#
+# - Fetch most current SLOs from Curricunet
+#     + Assemble multiple versions of a (CQ) course and determine which semesters they apply to
+#     + Whether they are present in the relevant classes in iLearn
+#     + Insert SLO into course if not present
+#     + Mark as inactive (change name) if necessary
+
+
+# - Issue:
+#     + Course naming / sections joined...
+
+import concurrent.futures
+import pandas as pd
+from pipelines import fetch, url, header
+from courses import getCoursesInTerm
+import codecs, json
+from path_dict import PathDict
+
+
+NUM_THREADS = 8
+get_fresh = 0
+sem_courses = getCoursesInTerm(176,get_fresh)
+
+# shorter list for test?
+#sem_courses = sem_courses[:50]
+
+print("Got %i courses in current semester." % len(sem_courses))
+
+
+def course_slo_getter(q):
+    (name,id) = q
+    info = {'ilearnname':name,'ilearnid':id}
+    print("  + Thread getting %s %s" % (str(name),str(id)))
+    u1 = url + "/api/v1/courses/%s/outcome_groups" % str(id)
+    og_for_course = fetch(u1)
+    if len(og_for_course):
+        for og in og_for_course:
+            if "outcomes_url" in og:
+                outcomes = fetch(url + og["outcomes_url"])
+                og['outcomes'] = outcomes 
+                og['full_outcomes'] = {}
+                for oo in outcomes:
+                    print(" -> " + url + oo['outcome']['url'])
+                    this_outcome = fetch( url + oo['outcome']['url'] )
+                    og['full_outcomes'][this_outcome['id']] = this_outcome
+    og_for_course.insert(0,info)
+    print("  - Thread %s DONE" % str(id))
+    return og_for_course
+
+
+output = []
+with concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as pool:
+    results = []
+    for C in sem_courses:
+        print("Adding ", C['name'], C['id'], " to queue")
+        results.append( pool.submit(course_slo_getter, [C['name'], C['id']] ) )
+        
+    print("--  Done")
+    print("results array has %i items" % len(results))
+    
+    for r in concurrent.futures.as_completed(results):
+        output.append(r.result())
+
+raw_log = codecs.open('cache/outcome_raw_log.txt','w','utf-8')
+raw_log.write( json.dumps(output,indent=2) )
+
+
+def ilearn_shell_slo_to_csv(shell_slos):
+    L = ['canvasid','name','crn','has_outcomes',]
+    for i in range(1,11):
+        L.append("o%i_id" % i)
+        L.append("o%i_vendor_guid" % i)
+        L.append("o%i_desc" % i)
+        L.append("o%i_assd" % i)
+    df = pd.DataFrame(columns=L)
+    for S in shell_slos:
+        short = S[0]
+        this_crs = {'canvasid':short['ilearnid'], 'name':short['ilearnname'], 'has_outcomes':0, }
+        if len(S)>1:
+            full = S[1]
+            this_crs['has_outcomes'] = 1
+            
+            i = 1
+            
+            for o in full['outcomes']:
+                try:
+                    this_id = int(o['outcome']['id'])
+                    this_crs['o%i_id' % i] = o['outcome']['id']
+                except Exception as e:
+                    this_crs['o%i_id' % i] = '!'
+                try:
+                    this_crs['o%i_desc' % i] = full['full_outcomes'][this_id]['description']
+                except Exception as e:
+                    this_crs['o%i_desc' % i] = '!'
+                try:
+                    assessed = 0
+                    if full['full_outcomes'][this_id]['assessed'] == 'True':
+                        assessed = 1
+                    this_crs['o%i_assd' % i] = assessed
+                except Exception as e:
+                    this_crs['o%i_assd' % i] = '!'
+                try:
+                    this_crs['o%i_vendor_guid' % i] = full['full_outcomes'][this_id]['vendor_guid']
+                except Exception as e:
+                    this_crs['o%i_vendor_guid' % i] = '!'
+                
+                i += 1
+                
+        df2 = pd.DataFrame(this_crs, columns = df.columns, index=[0])
+        df = pd.concat( [df, df2], ignore_index = True )
+        
+    df.to_csv('cache/outcome.csv')
+    print(df)
+    
+        
+        
+        
+ilearn_shell_slo_to_csv(output)
+
+        
+        
\ No newline at end of file
diff --git a/patterns_8020.py b/patterns_8020.py
new file mode 100644
index 0000000..b541648
--- /dev/null
+++ b/patterns_8020.py
@@ -0,0 +1,27 @@
+from pampy import _
+
+pat8020 = []
+
+
+""" (programs) entityType  entityTitle   status   proposalType    sectionName      lastUpdated      lastUpdatedBy    
+    fieldName     displayName     lookUpDisplay    fieldValue     instanceSortOrder
+    lookUpDataset   (array of dicts, each has keys: name, value, and corresponding values.)
+    
+    subsections or fields (arrays) - ignore for now just takem in order
+    
+    (courses) same as above?  
+    
+    html values: markdown convert?
+    
+"""
+
+
+pat8020.append(   {"fieldName": _} )
+pat8020.append(   {"displayName": _} )
+pat8020.append(   {"entityType": _} )
+pat8020.append(   {"entityTitle": _} )
+pat8020.append(   {"lookUpDisplay": _} )
+pat8020.append(   {"fieldValue": _} )
+
+pat8020.append( { "attributes": { "fieldName": "Award Type" }, 
+                                    "lookUpDisplay": _ } ) 
\ No newline at end of file
diff --git a/patterns_topdown.py b/patterns_topdown.py
new file mode 100644
index 0000000..20adc79
--- /dev/null
+++ b/patterns_topdown.py
@@ -0,0 +1,560 @@
+from pampy import _
+
+pat = []
+
+
+# lookup field
+p0 = { "attributes": {
+        "fieldName": _, 
+        "fieldId": _, 
+        "isLookUpField": True, 
+        "displayName": _
+    }, 
+    "lookUpDisplay": _, 
+    "dataTypeDetails": {
+        "type": "lookup"
+    }, 
+    "fieldValue": _ }
+    
+def pp0(a,b,c,d,e):
+    r = ("lookup field", {'fieldname':a,'id':b,'displayname':c,'lookupdisplay':d,'value':e})
+    #print(r)
+    return r
+    
+    
+# num field
+p1 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": False, 
+                 "displayName": _, 
+                 }, 
+                "dataTypeDetails": 
+                 {"scale": _, 
+                 "type": "numeric", 
+                 "precision": _, 
+                 }, 
+                "fieldValue": _, 
+                }
+
+def pp1(a,b,c,d,e,f):
+    #r = "Generic Num Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,f)
+    r = ("num field", {'fieldname':a,'id':b,'displayname':c,'value':f})
+    #print(r)
+    return r
+
+# string field
+p2 = {"attributes": 
+         {"fieldName": _, 
+         "fieldId": _, 
+         "isLookUpField": False, 
+         "displayName": _, 
+         }, 
+        "dataTypeDetails": 
+         {"type": "string", 
+         "maxLength": _, 
+         }, 
+        "fieldValue": _, 
+        }                                       
+                                        
+def pp2(a,b,c,d,e):
+    #r = "String Label: %s (id %s) Value: %s" % (a,b,e)
+    r = ("string field", {'fieldname':a,'id':b,'displayname':c,'value':e})
+    #print(r)
+    return r
+
+
+# flag field
+p3 = {"attributes": 
+           {"fieldName": _, 
+           "fieldId": _, 
+           "isLookUpField": False, 
+           "displayName": _, 
+           }, 
+          "dataTypeDetails": 
+           {"type": "flag", 
+           }, 
+          "fieldValue": _, 
+          }
+
+def pp3(a,b,c,d):
+    #r = "Generic Flag Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,d)
+    r = ("flag field", {'fieldname':a,'id':b,'displayname':c,'value':d})
+    #print(r)
+    return r
+
+# attributes
+
+p4 = {"attributes": _, 
+  "subsections": _, 
+  "fields": _
+  }
+
+def pp4(a,b,c):
+    r = ("attributes", {'attributes':a, 'subsections':b, 'fields':c})
+    #print(r)
+    return r
+
+
+# section
+
+p5 = {"sectionOrInstance": "section", 
+   "sectionName": _, 
+   "sectionSortOrder": _, 
+   "oneToManySection": _, 
+   }
+
+def pp5(a,b,c):
+    r = ("section", {'name':a, 'sortorder':b, 'onetomanysection':c})
+    #print(r)
+    return r
+
+
+# sectionInstance
+
+p6 = {"instanceId": _, 
+     "sectionOrInstance": "sectionInstance", 
+     "instanceSortOrder": _, 
+     "sectionName": _, 
+     }
+
+def pp6(a,b,c):
+    r = ("sectioninstance", {'id':a, 'sortorder':b, 'name':c })
+    #print(r)
+    return r
+
+
+pat.append( p0 )
+pat.append( pp0 )
+
+pat.append( p1 )
+pat.append( pp1 )
+
+pat.append( p2 )
+pat.append( pp2 )
+
+pat.append( p3 )
+pat.append( pp3 )
+
+pat.append( p4 )
+pat.append( pp4 )
+
+pat.append( p5 )
+pat.append( pp5 )
+
+pat.append( p6 )
+pat.append( pp6 )
+
+
+"""
+curic_patterns.append( {
+            "attributes": {
+                "fieldName": "Division", 
+                "fieldId": 65000, 
+                "isLookUpField": True, 
+                "displayName": "Division"
+            }, 
+            "lookUpDisplay": _, 
+            "dataTypeDetails": {
+                "type": "lookup"
+            }, 
+            "fieldValue": _
+        } )
+
+def div1(a,b): 
+    r = "Division: %s, id: %s" % (a,b)
+    print(r)
+    return(r)
+
+curic_patterns.append(div1)
+
+curic_patterns.append( {
+            "attributes": {
+                "fieldName": "Department", 
+                "fieldId": 65001, 
+                "isLookUpField": True, 
+                "displayName": "Department"
+            }, 
+            "lookUpDisplay": _, 
+            "dataTypeDetails": {
+                "type": "lookup"
+            }, 
+            "fieldValue": _
+        })
+def d2(a,b): 
+    r = "Department: %s, id: %s" % (a,b)
+    print(r)
+    return r
+
+curic_patterns.append(d2)
+
+
+
+curic_patterns.append({
+    "attributes": {
+        "fieldName": "Award Type", 
+        "fieldId": 60221, 
+        "isLookUpField": True, 
+        "displayName": "Award Type"
+    }, 
+    "lookUpDisplay": _, 
+    "dataTypeDetails": {
+        "type": "lookup"
+    }, 
+    "fieldValue": _
+})
+def d3(a,b): 
+    r = "Award: %s, id: %s" % (a,b)
+    print(r)
+    return r
+
+curic_patterns.append(d3)
+
+
+p1 = {
+    "attributes": {
+        "fieldName": "Description", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Description"
+    }, 
+    "dataTypeDetails": {
+        "type": "string"
+    }, 
+    "fieldValue": _
+}
+
+def pp1(a,b): 
+    r = "Description (id:%s) %s" % (a,b)
+    #print(r[:40])
+    return r
+
+curic_patterns.append(p1)
+curic_patterns.append(pp1)
+
+
+p2 = {"attributes": {
+        "fieldName": "Program Title", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Program Title"
+    }, 
+    "dataTypeDetails": {
+        "type": "string", 
+        "maxLength": 250
+    }, 
+    "fieldValue":_
+}
+
+def pp2(a,b):
+    r = "Program (id:%s) %s" % (a,b)
+    #print(r)
+    return r
+
+curic_patterns.append(p2)
+curic_patterns.append(pp2)
+    
+
+
+
+p3 = { "attributes": {
+        "fieldName": "Course", 
+        "fieldId": _, 
+        "isLookUpField": True, 
+        "displayName": "Course"
+    }, 
+    "lookUpDataset": [
+        [
+            {
+                "name": "Max", 
+                "value": _
+            }, 
+            {
+                "name": "IsVariable", 
+                "value": _
+            }, 
+            {
+                "name": "Min", 
+                "value": _
+            }, 
+            {
+                "name": "Text", 
+                "value": _
+            }
+        ]
+    ], 
+    "dataTypeDetails": {
+        "type": "lookup"
+    }, 
+    "lookUpDisplay": _, 
+    "fieldValue": _
+}
+
+def pp3(a,b,c,d,e,f,g):
+    r = "Course (%s / %s) %s (%s), var? %s  %s - %s" % (a,g, f, e, c, b, d)
+    #print(r)
+    return r
+
+curic_patterns.append(p3)
+curic_patterns.append(pp3)
+    
+
+p4 = {
+    "attributes": {
+        "sectionOrInstance": "section", 
+        "sectionName": "Unit Range", 
+        "sectionSortOrder": 2, 
+        "oneToManySection": False
+    }, 
+    "subsections": [], 
+    "fields": [
+        {
+            "attributes": {
+                "fieldName": "Units Low", 
+                "fieldId": 59608, 
+                "isLookUpField": False, 
+                "displayName": "Units Low"
+            }, 
+            "dataTypeDetails": {
+                "scale": 2, 
+                "type": "numeric", 
+                "precision": 6
+            }, 
+            "fieldValue": _
+        }, 
+        {
+            "attributes": {
+                "fieldName": "Units High", 
+                "fieldId": 59609, 
+                "isLookUpField": False, 
+                "displayName": "Units High"
+            }, 
+            "dataTypeDetails": {
+                "scale": 2, 
+                "type": "numeric", 
+                "precision": 6
+            }, 
+            "fieldValue": _
+        }
+    ]
+}
+
+def pp4(a,b):
+    r = "Unit Range: %s - %s" % (a,b)
+    return r
+
+curic_patterns.append(p4)
+curic_patterns.append(pp4)
+    
+p5 =  {
+            "attributes": {
+                "fieldName": "Discipline", 
+                "fieldId": _, 
+                "isLookUpField": True, 
+                "displayName": "Discipline"
+            }, 
+            "lookUpDisplay": _, 
+            "dataTypeDetails": {
+                "type": "lookup"
+            }, 
+            "fieldValue": _
+        }
+def pp5(a,b,c):
+    r = "Discipline (%s) %s / %s" % (a,b,c)
+    #print(r)
+    return r
+
+curic_patterns.append(p5)
+curic_patterns.append(pp5)
+
+
+
+p6 = { "attributes": {
+        "fieldName": "Course Block Definition", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Course Block Definition"
+    }, 
+    "dataTypeDetails": {
+        "type": "string"
+    }, 
+    "fieldValue": _
+}
+
+def pp6(a,b):
+    r = "Block (%s) %s" % (a,b)
+    #print(r)
+    return r
+    
+    
+p7 = {
+    "attributes": {
+        "fieldName": "Block Header", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Block Header"
+    }, 
+    "dataTypeDetails": {
+        "type": "string", 
+        "maxLength": 4000
+    }, 
+    "fieldValue": _
+}
+
+def pp7(a,b):
+    r = "Block Header (%s) %s" % (b,a)
+    #print(r)
+    return r
+    
+    
+p8 = {
+    "attributes": {
+        "fieldName": "Block Footer", 
+        "fieldId": _, 
+        "isLookUpField": False, 
+        "displayName": "Block Footer"
+    }, 
+    "dataTypeDetails": {
+        "type": "string", 
+        "maxLength": 4000
+    }, 
+    "fieldValue": _
+}
+
+def pp8(a,b):
+    r = "Block Footer (%s) %s" % (b,a)
+    #print(r)
+    return r
+    
+curic_patterns.append(p6)
+curic_patterns.append(pp6)
+curic_patterns.append(p7)
+curic_patterns.append(pp7)
+curic_patterns.append(p8)
+curic_patterns.append(pp8)
+
+
+
+                                        
+                                        
+                            
+                            
+                            
+######################      
+######################   Trying to remove more junk      
+######################                  
+
+
+curic_patterns.append(j1)
+curic_patterns.append(jj1)
+
+
+    
+j3 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": True, 
+               "displayName": _, 
+               }, 
+              "lookUpDisplay": _, 
+              "dataTypeDetails": 
+               {"type": "lookup", 
+               }, 
+              "fieldValue": _, 
+                }
+
+def jj3(a,b,c,d,e):
+    r = "Generic lookup Field: Name: %s / %s, ID: %i, Displayname: %s, Value: %s " % (a,c,b,d,e)
+    #print(r)
+    return r
+
+
+curic_patterns.append(j2)
+curic_patterns.append(jj2)
+
+curic_patterns.append(j3)
+curic_patterns.append(jj3)
+
+
+    
+curic_patterns.append(j4)
+curic_patterns.append(jj4)
+
+
+
+j5 = {"attributes": 
+         {"fieldName": _, 
+         "fieldId": _, 
+         "isLookUpField": False, 
+         "displayName": _, 
+         }, 
+        "dataTypeDetails": 
+         {"scale": _, 
+         "type": "numeric", 
+         "precision": _, 
+         }, 
+        "fieldValue": _, 
+        }
+
+def jj5(a,b,c,d,e,f):
+    r = "Numeric Field, Name: %s / %s  Id: %s, Value: %s" % (a,c,b,f)
+
+
+    #print(r)
+    return r
+    
+curic_patterns.append(j5)
+curic_patterns.append(jj5)
+curic_patterns.append(j6)
+curic_patterns.append(jj6)
+    
+"""    
+    
+"""j2 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": False, 
+                 "displayName": _, 
+                 }, 
+                "dataTypeDetails": 
+                 {"scale": 2, 
+                 "type": "numeric", 
+                 "precision": 6, 
+                 }, 
+                "fieldValue": _, 
+                }
+
+def jj2(a,b,c,d):
+    r = "Generic Num Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,d)
+    print(r)
+    return r
+    
+j2 = {"attributes": 
+                 {"fieldName": _, 
+                 "fieldId": _, 
+                 "isLookUpField": False, 
+                 "displayName": _, 
+                 }, 
+                "dataTypeDetails": 
+                 {"scale": 2, 
+                 "type": "numeric", 
+                 "precision": 6, 
+                 }, 
+                "fieldValue": _, 
+                }
+
+def jj2(a,b,c,d):
+    r = "Generic Num Field: Name: %s, ID: %s, Displayname: %s, Value: %s" % (a,b,c,d)
+    print(r)
+    return r
+"""    
+    
+    
+
+
+
+
+
+
+
+
diff --git a/pipelines.py b/pipelines.py
new file mode 100644
index 0000000..fb05f00
--- /dev/null
+++ b/pipelines.py
@@ -0,0 +1,1958 @@
+# This Python file uses the following encoding: utf-8
+
+#from __future__ import print_function
+from time import strptime
+from bs4 import BeautifulSoup as bs
+from util import UnicodeDictReader
+from datetime import datetime as dt
+import pandas as pd
+import codecs, json, requests, re, csv, datetime, pysftp, os, jsondiff, os.path
+import sys, shutil, hmac, hashlib, base64, schedule, time, pathlib, datetime
+import pdb
+from collections import defaultdict
+from deepdiff import DeepDiff
+from secrets import apiKey, apiSecret, FTP_SITE, FTP_USER, FTP_PW, GOO, GOO_PIN, token, url, domain, account_id, header, g_id, g_secret
+from secrets import instructure_url, instructure_username, instructure_private_key
+
+
+
+"""
+Everything to do with fetching data, 
+ - From iLearn, via token
+ - current roster uploads from instructures sftp site
+ - raw logs and other from canvas data repo
+ - from ssb, use firefox to scrape the schedule
+ 
+ 
+And some subsequent processing:
+ - Raw roster files, into a more compact json format
+ - Raw logs into something more useful
+"""
+
+
+
+
+verbose = False
+
+users = {}
+users_by_id = {}
+
+# todo: all these constants for SSB  --  line 1008
+#
+# todo: https://stackoverflow.com/questions/42656247/how-can-i-use-canvas-data-rest-api-using-python
+
+schedfile = 'temp.csv'
+
+
+SEMESTER = 'Summer 2019'
+short_sem = 'su19'
+semester_begin = strptime('06/17', '%m/%d')
+filename = 'su19_sched.json'
+
+SEMESTER = 'Summer 2020'
+short_sem = 'su20'
+semester_begin = strptime('06/15', '%m/%d')
+filename = 'su20_sched.json'
+
+SEMESTER = 'Fall 2020'
+short_sem = 'fa20'
+semester_begin = strptime('08/24', '%m/%d')
+filename = 'fa20_sched.json'
+
+SEMESTER = 'Spring 2021'
+short_sem = 'sp21'
+semester_begin = strptime('02/01', '%m/%d')
+filename = 'sp21_sched.json'
+filename_html = 'sp21_sched.html'
+
+
+SEMESTER = 'Summer 2021 (View only)'
+short_sem = 'su21'
+semester_begin = strptime('06/14', '%m/%d')
+filename = 'su21_sched.json'
+filename_html = 'su21_sched.html'
+
+    
+
+
+# Current or upcoming semester is first.
+sems = ['su21', 'sp21', 'fa20', 'su20', 'sp20'] #, 'fa19']   #  'sp19']
+
+sys.setrecursionlimit( 100000 )
+
+local_data_folder = 'cache/canvas_data/'
+mylog = codecs.open(local_data_folder + 'temp_log.txt','w')
+
+
+
+
+
+
+
+
+
+gp = {}
+gp['ACCT'] = 'info'
+gp['AE'] = 'skill'
+gp['AH'] = 'well'
+gp['AJ'] = 'skill'
+gp['AMT'] = 'skill'
+gp['ANTH'] = 'soc'
+gp['APE'] = 'skill'
+gp['ART'] = 'art'
+gp['ASTR'] = 'stem'
+gp['ATH'] = 'well'
+gp['BIO'] = 'stem'
+gp['BOT'] = 'info'
+gp['BUS'] = 'info'
+gp['CD'] = 'skill'
+gp['CHEM'] = 'stem'
+gp['CMGT'] = 'skill'
+gp['CMUN'] = 'comm'
+gp['COS'] = 'skill'
+gp['CSIS'] = 'stem'
+gp['CWE'] = 'skill'
+gp['DM'] = 'info'
+gp['ECOL'] = 'stem'
+gp['ECON'] = 'info'
+gp['ENGL'] = 'soc'
+gp['ENGR'] = 'stem'
+gp['ENVS'] = 'stem'
+gp['ESL'] = 'comm'
+gp['ETHN'] = 'comm'
+gp['FRNH'] = 'comm'
+gp['GEOG'] = 'stem'
+gp['GEOL'] = 'stem'
+gp['GUID'] = 'soc'
+gp['HE'] = 'well'
+gp['HIST'] = 'soc'
+gp['HUM'] = 'soc'
+gp['HVAC'] = 'skill'
+gp['JFT'] = 'skill'
+gp['JLE'] = 'skill'
+gp['JOUR'] = 'comm'
+gp['JPN'] = 'comm'
+gp['KIN'] = 'well'
+gp['LIB'] = 'comm'
+gp['LIFE'] = 'well'
+gp['MATH'] = 'stem'
+gp['MCTV'] = 'art'
+gp['MUS'] = 'art'
+gp['PHIL'] = 'soc'
+gp['PHYS'] = 'stem'
+gp['POLS'] = 'soc'
+gp['PSCI'] = 'stem'
+gp['PSYC'] = 'soc'
+gp['RE'] = 'skill'
+gp['SJS'] = 'soc'
+gp['SOC'] = 'soc'
+gp['SPAN'] = 'comm'
+gp['THEA'] = 'art'
+gp['WELD'] = 'skill'
+gp['WTRM'] = 'skill'
+gp['MGMT'] = 'skill'
+gp['MKTG'] = 'skill'
+gp['HTM'] = 'skill'
+
+dean = {}
+dean['AH'] = 'et'
+dean['HE'] = 'et'
+dean['ATH'] = 'et'
+dean['KIN'] = 'et'
+dean['LIFE'] = 'et'
+dean['AE'] = 'ss'
+dean['APE'] = 'ss'
+dean['ACCT'] = 'ss'
+dean['AJ'] = 'ss'
+dean['AMT'] = 'ss'
+dean['HVAC'] = 'ss'
+dean['JFT'] = 'ss'
+dean['JLE'] = 'ss'
+dean['RE'] = 'ss'
+dean['WTRM'] = 'ss'
+dean['WELD'] = 'ss'
+dean['ANTH'] = 'nl'
+dean['ART'] = 'nl'
+dean['ASTR'] = 'jn'
+dean['BIO'] = 'jn'
+dean['BOT'] = 'ss'
+dean['BUS'] = 'ss'
+dean['CD'] = 'ss'
+dean['CHEM'] = 'jn'
+dean['CMGT'] = 'ss'
+dean['CMUN'] = 'nl'
+dean['COS'] = 'ss'
+dean['CSIS'] = 'ss'
+dean['CWE'] = 'ss'
+dean['DM'] = 'ss'
+dean['ECOL'] = 'jn'
+dean['ECON'] = 'nl'
+dean['ENGL'] = 'nl'
+dean['ENGR'] = 'jn'
+dean['ENVS'] = 'jn'
+dean['ESL'] = 'ss'
+dean['ETHN'] = 'nl'
+dean['FRNH'] = 'nl'
+dean['GEOG'] = 'jn'
+dean['GEOL'] = 'jn'
+dean['GUID'] = 'nl'
+dean['HIST'] = 'nl'
+dean['HUM'] = 'nl'
+dean['JOUR'] = 'nl'
+dean['JPN'] = 'nl'
+dean['LIB'] = 'kn'
+dean['MATH'] = 'jn'
+dean['MCTV'] = 'nl'
+dean['MGMT'] = 'ss'
+dean['MKTG'] = 'ss'
+dean['HTM'] = 'ss'
+dean['MUS'] = 'nl'
+dean['PHIL'] = 'nl'
+dean['PHYS'] = 'jn'
+dean['POLS'] = 'nl'
+dean['PSCI'] = 'jn'
+dean['PSYC'] = 'nl'
+dean['SJS'] = 'nl'
+dean['SOC'] = 'nl'
+dean['SPAN'] = 'nl'
+dean['THEA'] = 'nl'
+
+
+class FetchError(Exception):
+    pass
+
+
+DEBUG = 0
+
+def d(s,end=''): 
+    global DEBUG
+    if end and DEBUG: print(s,end=end)
+    elif DEBUG: print(s)
+
+################
+################  CANVAS API MAIN FETCHING FUNCTIONS
+################
+################
+################
+
+
+
+
+# Main canvas querying fxn    
+def fetch(target,verbose=0):
+    # if there are more results, recursivly call myself, adding on to the results.
+    results = 0
+    if target[0:4] != "http": target = url + target
+    if verbose: 
+        print("++ Fetching: " + target)
+    r2 = requests.get(target, headers = header)
+    #if verbose:
+    #print "++ Got: " + r2.text
+    try:
+        results = json.loads(r2.text)
+        count = len(results)
+    except:
+        print("-- Failed to parse: ", r2.text)
+    if verbose: 
+        print("Got %i results" % count)
+    if verbose > 1:
+        print(r2.headers)
+        
+        tempout = codecs.open('cache/fetchcache.txt','a','utf-8')
+        tempout.write(r2.text+"\n\n")
+        tempout.close()
+    
+    if ('link' in r2.headers and count > 0):
+        links = r2.headers['link'].split(',')
+        for L in links:
+            ll = L.split(';')
+            link = ll[0].replace("<","")
+            link = link.replace(">","")
+            if re.search(r'next', ll[1]):
+                if (verbose):  print("++ More link: " + link)
+                #link = re.sub(r'per_page=10$', 'per_page=100', link)    # link.replace('per_page=10','per_page=500')
+                #if (verbose):  print("++ More link: " + link)
+                
+                nest = fetch(link,verbose)
+                if isinstance(results,dict): results.update(nest)
+                else: results.extend(nest)
+    return results    
+    
+# Main canvas querying fxn - stream version - don't die on big requests
+def fetch_stream(target,verbose=0): 
+    # if there are more results, recursivly call myself, adding on to the results.
+    results = 0
+    while target:
+        if target[0:4] != "http": target = url + target
+        if verbose: 
+            print("++ Fetching: " + target)
+        r2 = requests.get(target, headers = header)
+        if r2.status_code == 502:
+            raise FetchError()
+        try:
+            results = json.loads(r2.text)
+            count = len(results)
+        except:
+            print("-- Failed to parse: ", r2.text)
+        if verbose: 
+            print("Got %i results" % count)
+        if verbose > 1:
+            print(r2.headers)
+            tempout = codecs.open('cache/fetchcache.txt','a','utf-8')
+            tempout.write(r2.text+"\n\n")
+            tempout.close()
+        
+        next_link_found = 0
+        if ('link' in r2.headers and count > 0):
+            links = r2.headers['link'].split(',')
+            for L in links:
+                ll = L.split(';')
+                link = ll[0].replace("<","")
+                link = link.replace(">","")
+                if re.search(r'next', ll[1]):
+                    target = link 
+                    next_link_found = 1
+                    break
+        if not next_link_found: target = 0
+        yield results
+
+    
+# for dicts with one key, collapse that one key out, cause
+# paging makes problems... example: enrollment_terms    
+def fetch_collapse(target,collapse='',verbose=0):
+    # if there are more results, recursivly call myself, adding on to the results.
+    results = 0
+    if target[0:4] != "http": target = url + target
+    if verbose: 
+        print("++ Fetching: " + target)
+    r2 = requests.get(target, headers = header)
+    #if verbose:
+    #print "++ Got: " + r2.text
+    try:
+        results = json.loads(r2.text)
+    except:
+        print("-- Failed to parse: ", r2.text)
+    if verbose: print(r2.headers)
+    
+    if collapse and collapse in results:
+        results = results[collapse]
+    
+    if ('link' in r2.headers):
+        links = r2.headers['link'].split(',')
+        for L in links:
+            ll = L.split(';')
+            link = ll[0].replace("<","")
+            link = link.replace(">","")
+            if re.search(r'next', ll[1]):
+                if (verbose):  print("++ More link: " + link)
+                nest = fetch_collapse(link, collapse, verbose)
+                if isinstance(results,dict): results.update(nest)
+                else: results.extend(nest)
+    return results    
+    
+
+
+################
+################  SCHEDULE PARSING HELPERS
+################
+################
+################
+
+# Teacher name format changed. Remove commas and switch first to last   
+def fix_t_name(str):
+    str = str.strip()
+    str = re.sub('\s+',' ',str)
+    parts = str.split(', ')
+    if len(parts)>1:
+        return parts[1].strip() + " " + parts[0].strip()
+    return str
+
+# Separate dept and code
+def split_class_dept(c):
+    return c.split(' ')[0]
+def split_class_code(c):
+    num = c.split(' ')[1]
+    parts = re.match('(\d+)([a-zA-Z]+)',num)
+    #ret = "Got %s, " % c
+    if parts:
+        r = int(parts.group(1))
+        #print(ret + "returning %i." % r)
+        return r
+    #print(ret + "returning %s." % num)
+    return int(num)
+def split_class_code_letter(c):
+    num = c.split(' ')[1]
+    parts = re.match('(\d+)([A-Za-z]+)',num)
+    if parts:
+        return parts.group(2)
+    return ''
+    
+# go from sp20 to 2020spring
+def shortToLongSem(s):
+    parts = re.search(r'(\w\w)(\d\d)', s)
+    yr = parts.group(2)
+    season = parts.group(1)
+    seasons = {'sp':'spring','su':'summer','fa':'fall','wi':'winter'}
+    return '20'+yr+seasons[season]
+    
+# Go to the semesters folder and read the schedule. Return dataframe    
+def getSemesterSchedule(short='sp21'):                # I used to be current_schedule
+    # todo: Some semesters have a different format.... partofday   type   site  xxx i just dL'd them again
+    
+    filename = 'cache/semesters/'+shortToLongSem(short)+'/' + short + '_sched.json'
+    print("opening %s" % filename)
+    #openfile = open(filename,'r')
+    #a = json.loads(openfile)
+    #return pd.DataFrame(a)
+    schedule = pd.read_json(filename)
+    schedule.teacher = schedule['teacher'].apply(fix_t_name)
+    #print schedule['teacher']
+    for index,r in schedule.iterrows():
+        tch = r['teacher']
+        parts = tch.split(' . ')
+        if len(parts)>1:
+            #print "Multiple teachers: (" + tch + ")"
+            schedule.loc[index,'teacher'] = parts[0]
+            #print "  Fixed original: ", schedule.loc[index]
+            
+            for t in parts[1:]: 
+                r['teacher'] = t
+                schedule.loc[-1] = r
+                #print "  New row appended: ", schedule.loc[-1]
+    schedule = schedule.assign(dept = schedule['code'].apply(split_class_dept))
+    schedule = schedule.assign(codenum = schedule['code'].apply(split_class_code))
+    schedule = schedule.assign(codeletter = schedule['code'].apply(split_class_code_letter))
+    #print(schedule)
+    schedule['sem'] = short
+    #print schedule.columns
+    return schedule
+
+
+
+online_courses = {}
+def prep_online_courses_df():
+    global online_courses
+    schedule = current_schedule()   # from banner
+    online_courses = schedule[lambda x: x.type=='online']
+    
+def course_is_online(crn):
+    global online_courses
+    #print "looking up: " + str(crn)
+    #print online_courses
+    course = online_courses[lambda x: x.crn==int(crn)]
+    return len(course)
+    
+def get_crn_from_name(name):
+    #print "name is: "
+    #print(name)
+    m = re.search( r'(\d\d\d\d\d)', name)
+    if m: return int(m.groups(1)[0])
+    else: return 0
+
+def get_enrlmts_for_user(user,enrollments):
+    #active enrollments
+    u_en = enrollments[ lambda x: (x['user_id'] == user) & (x['workflow']=='active') ]
+    return u_en[['type','course_id']]
+
+
+
+################
+################  CANVAS DATA
+################
+################
+################
+
+    
+# Get something from Canvas Data
+def do_request(path):  
+    #Set up the request pieces
+    method = 'GET'
+    host = 'api.inshosteddata.com'
+    apiTime = dt.utcnow().strftime('%a, %d %b %y %H:%M:%S GMT')
+    apiContentType = 'application/json'
+
+    msgList = []
+    msgList.append(method)
+    msgList.append(host)
+    msgList.append(apiContentType)
+    msgList.append('')
+    msgList.append(path)
+    msgList.append('')
+    msgList.append(apiTime)
+    msgList.append(apiSecret)
+
+    msgStr = bytes("".join("%s\n" % k for k in msgList).strip(),'utf-8')
+
+    sig = base64.b64encode(hmac.new(key=bytes(apiSecret,'utf-8'),msg=msgStr,digestmod=hashlib.sha256).digest())
+    sig = sig.decode('utf-8')
+
+    headers = {}
+    headers['Authorization'] = 'HMACAuth {}:{}'.format(apiKey,sig)
+    headers['Date'] = apiTime
+    headers['Content-type'] = apiContentType
+
+
+    #Submit the request/get a response
+    uri = "https://"+host+path
+    print (uri)
+    print (headers)
+    response = requests.request(method='GET', url=uri, headers=headers, stream=True)
+
+    #Check to make sure the request was ok
+    if(response.status_code != 200):
+        print(('Request response went bad. Got back a ', response.status_code, ' code, meaning the request was ', response.reason))
+    else:
+        #Use the downloaded data
+        jsonData = response.json()
+        #print(json.dumps(jsonData, indent=4))
+        return jsonData
+
+# Canvas data, download all new files
+def sync_non_interactive():
+    resp = do_request('/api/account/self/file/sync')    
+    mylog.write(json.dumps(resp, indent=4))
+    #mylog.close()
+    gotten = os.listdir(local_data_folder)
+    wanted = []
+    i = 0
+    for x in resp['files']:
+        filename = x['filename']
+        exi = "No "
+        if filename in gotten: exi = "Yes"
+        else: wanted.append(x)
+        
+        print(str(i) + '.\tLocal: %s\tRemote: %s' % ( exi, filename ))
+        i += 1
+    print("I will attempt to download %i files." % len(wanted))
+    
+    #answer = input("Press enter to begin, or q to quit ")
+    #if not answer == '': return
+    
+    good_count = 0
+    bad_count = 0
+    for W in wanted:
+        print("Downloading: " + W['filename'])
+        response = requests.request(method='GET', url=W['url'], stream=True)
+        if(response.status_code != 200):
+            print('Request response went bad. Got back a %s code, meaning the request was %s' % \
+                 (response.status_code, response.reason))
+            print('URL: ' + W['url'])
+            bad_count += 1
+            
+        else:
+            #Use the downloaded data
+            with open(local_data_folder + W['filename'], 'wb') as fd:
+                for chunk in response.iter_content(chunk_size=128):
+                    fd.write(chunk)
+            print("Success")
+            good_count += 1
+    print("Out of %i files, %i succeeded and %i failed." % (len(wanted),good_count,bad_count))
+        
+
+# list files in canvas_data (online) and choose one or some to download.
+def interactive():
+    resp = do_request('/api/account/self/file/sync')    
+    mylog.write(json.dumps(resp, indent=4))
+    #mylog.close()
+    i = 0
+    gotten = os.listdir(local_data_folder)
+    for x in resp['files']:
+        print(str(i) + '.\t' + x['filename'])
+        i += 1
+    which = input("Which files to get? (separate with commas, or say 'all') ")
+    if which=='all':
+        which_a = list(range(i-1))
+    else:
+        which_a = which.split(",")
+    for W in which_a:
+        this_i = int(W)
+        this_f = resp['files'][this_i]
+        filename = this_f['filename']
+        if filename in gotten: continue
+        print("Downloading: " + filename)
+        response = requests.request(method='GET', url=this_f['url'], stream=True)
+        if(response.status_code != 200):
+            print(('Request response went bad. Got back a ', response.status_code, ' code, meaning the request was ', response.reason))
+        else:
+            #Use the downloaded data
+            with open(local_data_folder + filename, 'wb') as fd:
+                for chunk in response.iter_content(chunk_size=128):
+                    fd.write(chunk)
+            print("Success")
+        """if filename.split('.')[-1] == 'gz':
+            try:
+                plain_filename = 'canvas_data/' + ".".join(filename.split('.')[:-1])
+                pf = open(plain_filename,'w')
+                with gzip.open('canvas_data/' + filename , 'rb') as f:
+                    pf.write(f.read())    
+            except Exception as e:
+                print "Failed to ungizp. Probably too big: " + str(e)"""
+
+
+
+
+
+
+
+
+######   SSB SCHEDULE
+######
+######
+######
+
+def todays_date_filename():       # helper    
+    n = datetime.now()
+    m = n.month
+    if m < 10: m = "0"+str(m)
+    d = n.day
+    if d < 10: d = "0" + str(d)
+    return "reg_" + short_sem + "_" + str(n.year) + str(m) + str(d)
+
+def nowAsStr():     # possible duplicate
+    #Get the current time, printed in the right format
+    currentTime = datetime.datetime.utcnow()
+    prettyTime = currentTime.strftime('%a, %d %b %Y %H:%M:%S GMT')
+    return prettyTime
+
+
+def row_has_data(r):      # helper    
+    if r.find_all('th'):
+        return False
+    if len(r.find_all('td')) > 2:
+        return True
+    if re.search('Note\:', r.get_text()):
+        return True
+    return False
+
+#dbg = open('cache/temp_scheddebug_' + 'sp20' + '.txt','w')
+
+
+def row_text(r):   # helper  
+    #global dbg
+    
+    d("Row Txt Fxn gets:  ")
+    arr = []
+    for t in r.find_all('td'):
+        if t.contents and len(t.contents) and t.contents[0].name == 'img':
+            arr.append("1")
+            d("img")
+        r_text = t.get_text()
+        arr.append(r_text)
+        if 'colspan' in t.attrs and t['colspan']=='2':
+            d('[colspan2]')
+            arr.append('')
+        d("\t"+r_text, end=" ")
+    d('')
+    
+    if len(arr)==1 and re.search('Note\:',arr[0]):
+        note_line = clean_funny( arr[0] )
+        note_line = re.sub(r'\n',' ', note_line)
+        note_line = re.sub(r'"','', note_line)
+        #note_line = re.sub(r',','\,', note_line)
+        return ',,,,,,,,,,,,,,,,,,"' + note_line + '"\n'
+    del arr[0]
+    arr[1] = clean_funny(arr[1])
+    arr[2] = clean_funny(arr[2])
+    if arr[1]: arr[1] = arr[1] + " " + arr[2]
+    del arr[2]
+    arr = [ re.sub(r'&nbsp;','',a) for a in arr]
+    arr = [ re.sub(',','. ',a) for a in arr]
+    arr = [ re.sub('\(P\)','',a) for a in arr]
+    arr = [ a.strip() for a in arr]
+    #del arr[-1]
+    r = ','.join(arr)+'\n'
+    r = re.sub('\n','',r)
+    r = re.sub('add to worksheet','',r)
+    d("Row Txt Fxn returns:  " + r + "\n\n")
+
+    return r + '\n'
+
+
+
+# Take banner's html and make a csv(?) file  
+def ssb_to_csv(src):
+    #out = codecs.open(schedfile,'w','utf-8')
+    output = 'crn,code,sec,cmp,cred,name,days,time,cap,act,rem,wl_cap,wl_act,wl_rem,teacher,date,loc,ztc,note\n'
+    b = bs(src, 'html.parser')
+    tab = b.find(class_="datadisplaytable")
+    if not tab:
+        print("hmm... didn't find a 'datadisplaytable' in this html: ")
+        #print(src)
+        return 0
+    rows = tab.find_all('tr')
+    drows = list(filter(row_has_data,rows))
+    for dd in drows:
+        t = row_text(dd)
+        output += t
+    return output
+    
+
+        
+def clean_funny(str):
+    if str and str.encode('utf8') == ' ': return ''
+    return str
+def clean_funny2(str):
+    if str and str == '\xa0': return ''
+    if str and str == ' ': return ''
+    return str
+    
+def clean_funny3(str):
+    return re.sub('\xa0','',str)
+
+
+
+### course is a list of 1-3 lists, each one being a line in the schedule's output. First one has section     
+def course_start(course):
+    #todo: use this to make a early/late/short field and store semester dates w/ other constants
+    
+    start = datetime(2019,1,28)
+    end   = datetime(2019,5,24)
+
+    # is it normal, early, late, winter?
+    li = course[0]
+    date = li[12]
+    
+    if date=='01/28-05/24':
+        return 'Normal'
+    if date=='TBA':
+        return 'TBA'
+    if date=='01/02-01/25':
+        return 'Winter'
+    if date=='01/02-01/24':
+        return 'Winter'
+    
+    ma = re.search(  r'(\d+)\/(\d+)\-(\d+)\/(\d+)', date)
+    if ma:
+        # TODO do these years matter?
+        mystart = datetime(2019, int(ma.group(1)), int(ma.group(2)))
+        if int(ma.group(1)) > 10: mystart = datetime(2018, int(ma.group(1)), int(ma.group(2)))
+        myend   = datetime(2019, int(ma.group(3)), int(ma.group(4)))
+        length = myend - mystart
+        weeks = length.days / 7
+        
+        if mystart != start:
+            if mystart < start:
+                #print 'Early Start ', str(weeks), " weeks ",
+                return 'Early start'
+            else:
+                #print 'Late Start ', str(weeks), " weeks ",
+                return 'Late start'
+        else:
+            if myend > end:
+                #print 'Long class ', str(weeks), " weeks ",
+                return 'Long term'
+            else:
+                #print 'Short term ', str(weeks), " weeks ",
+                return 'Short term'
+        #return ma.group(1) + '/' + ma.group(2) + " end: " + ma.group(3) + "/" + ma.group(4)
+    else:
+        return "Didn't match: " + date
+
+        
+def time_to_partofday(t):
+    #todo: account for multiple sites/rows
+    # 11:20 am-12:10 pm
+    mor     = strptime('12:00 PM', '%I:%M %p')
+    mid     = strptime( '2:00 PM', '%I:%M %p')
+    aft     = strptime( '6:00 PM', '%I:%M %p')
+    if t == 'TBA':
+        return 'TBA'
+    t = t.upper()    
+    parts = t.split('-')
+    try:
+        begin = strptime(parts[0], '%I:%M %p')
+        end = strptime(parts[1], '%I:%M %p')
+        if end > aft:
+            return "Evening"
+        if end > mid:
+            return "Afternoon"
+        if end > mor:
+            return "Midday"
+        return "Morning"
+        #return begin,end
+    except Exception as e:
+        #print 'problem parsing: ', t, "   ",
+        return ""
+
+# Deduce a 'site' field, based on room name and known offsite locations    
+def room_to_site(room,verbose=0):
+    #todo: account for multiple sites/rows
+    #todo: better way to store these offsite labels
+    othersites = 'AV,SBHS I-243,SBHS I-244,LOADCS,HOPEH,HOPEG,PLY,SAS,SBHS,LOHS,CHS,SBRAT,'.split(',')    
+    # is it gilroy, mh, hol, other, online or hybrid?
+    site = 'Gilroy'
+    #if len(course[0]) > 13:
+    #    room = course[0][13]
+    if room in othersites:
+        site = "Other"
+    if room == 'TBA':
+        site = 'TBA'
+    if room == 'AV':
+        site = 'San Martin Airport'
+    if re.search('MHG',room):
+        site = 'Morgan Hill'
+    if re.search('HOL',room):
+        site = 'Hollister'
+    if re.search('COY',room):
+        site = 'Coyote Valley'
+    if re.search('OFFSTE',room):
+        site = 'Other'
+    if re.search('ONLINE',room):
+        site = 'Online'
+    if verbose: print(room, '\t', end=' ') 
+    return site
+    
+
+from io import StringIO
+
+    
+# take text lines and condense them to one dict per section    
+def to_section_list(input_text,verbose=0):
+    this_course = ''
+    #todo: no output files
+    #jout = codecs.open(filename, 'w', 'utf-8')
+    #input = csv.DictReader(open(schedfile,'r'))
+    #input = UnicodeDictReader(input_text.splitlines())
+    all_courses = []
+    
+    
+    try:
+        f = StringIO(input_text)
+    except:
+        print("ERROR with this input_text:")
+        print(input_text)
+    reader = csv.reader(f, delimiter=',')
+    headers = next(reader)
+    for r in reader:
+        d = dict(list(zip(headers,r)))
+        #pdb.set_trace()
+        # clean funny unicode char in blank entries
+        r = {k: clean_funny2(v) for k,v in list(d.items()) }
+        if verbose: print("Cleaned: " + str(r))
+        
+        if 'time' in r:
+            if r['time']=='TBA': r['time'] = ''
+            if r['time']: r['partofday'] = time_to_partofday(r['time'])
+            
+        r['type'] = ''
+        
+        if 'loc' in r:
+            if r['loc'] == 'ONLINE': r['type'] = 'online'
+                                                         
+            if r['loc'] == 'ONLINE LIVE': r['type'] = 'online live'
+            if r['loc']: r['site'] = room_to_site(r['loc'],verbose)
+        
+        if 'code' in r:
+            if re.search(r'ONLINE\sLIVE',r['code']):
+                r['type'] = 'online live'
+            elif re.search(r'ONLINE',r['code']):
+                r['type'] = 'online'
+                
+        # does it have a section? it is the last course
+        if r['crn']:   # is a new course or a continuation?
+            if verbose: print("  it's a new section.")
+            if this_course:
+                if not this_course['extra']: this_course.pop('extra',None)
+                all_courses.append(this_course)
+            this_course = r
+            #print(r['name'])
+            this_course['extra'] = []
+        else:
+            # is a continuation line
+            if verbose: print("  additional meeting: " + str(r))
+            for k,v in list(r.items()):
+                if not v: r.pop(k,None)
+            # TODO: if extra line is different type?
+            #if this_course['type']=='online' and r['type'] != 'online': this_course['type'] = 'hybrid'
+            #elif this_course['type']!='online' and r['type'] == 'online': this_course['type'] = 'hybrid'
+            this_course['extra'].append(r)
+    return all_courses
+    
+
+# Schedule / course filling history
+# csv headers:  crn, code, teacher,  datetime, cap, act, wlcap, wlact
+# Log the history of enrollments per course during registration
+def log_section_filling(current_sched_list):
+    rows = 'timestamp crn code teacher cap act wl_cap wl_act'.split(' ')
+    rows_j = 'crn code teacher cap act wl_cap wl_act'.split(' ')
+    print(rows_j)
+    now = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M')
+    csv_fn = 'cache/reg_history_' + short_sem + '.csv'
+    with codecs.open(csv_fn,'a','utf-8') as f:
+        writer = csv.writer(f)
+        for S in current_sched_list:
+            #print(S)
+            items = [now,]
+            [ items.append( S[X] ) for X in rows_j ]
+            writer.writerow(items)
+            
+# Same as above, but compressed, act only
+def log_section_filling2(current_sched_list):
+
+
+
+    now = datetime.datetime.now().strftime('%Y-%m-%dT%H')
+    
+    todays_data = { int(S['crn']): S['act'] for S in current_sched_list }
+    #print(todays_data)
+    
+    todays_df = pd.DataFrame.from_dict(todays_data, orient='index', columns=[now])
+    todays_df = todays_df.rename_axis('crn')
+    #print(todays_df)
+    todays_df.to_csv('cache/reg_today_new.csv', index=True)
+    
+    try:
+        myframe = pd.read_csv('cache/reg_data_' + short_sem + '.csv')
+        print(myframe)
+    except:
+        fff = open('cache/reg_data_'+short_sem+'.csv','w')
+        fff.write('crn\n')
+        fff.close()
+        myframe = pd.read_csv('cache/reg_data_' + short_sem + '.csv')
+        #myframe = pd.DataFrame.from_dict(todays_data, orient='index', columns=[now])
+        #myframe = myframe.rename_axis('crn')
+        print("Creating new data file for this semester.")
+    
+    new_df = myframe.join( todays_df, on='crn', how='outer' )
+    new_df = new_df.rename_axis('crn')
+    print(new_df)
+    
+    reg_data_filename = 'reg_data_' + short_sem + '.csv'
+    new_df.to_csv('cache/' + reg_data_filename, index=False)
+    put_file('/web/phowell/schedule/', 'cache/', reg_data_filename, 0)                
+    print('ok')
+    
+    
+
+# Use Firefox and log in to ssb and get full schedule. Only works where selenium is installed
+def scrape_schedule():
+                                  
+    #url = "https://ssb.gavilan.edu/prod/twbkwbis.P_GenMenu?name=bmenu.P_StuMainMnu"
+    url = "https://ssb-prod.ec.gavilan.edu/PROD/twbkwbis.P_GenMenu?name=bmenu.P_MainMnu"
+                    
+                 
+    text = ''
+    
+    from selenium import webdriver
+    from selenium.webdriver.common.keys import Keys
+    from selenium.webdriver.support.ui import WebDriverWait, Select
+    from selenium.webdriver.common.by import By
+    from selenium.webdriver.support import expected_conditions as EC
+
+    try:
+        driver = webdriver.Firefox()
+        driver.get(url)
+        driver.find_element_by_id("UserID").clear()
+        driver.find_element_by_id("UserID").send_keys(GOO)
+        driver.find_element_by_name("PIN").send_keys(GOO_PIN)
+        driver.find_element_by_name("loginform").submit()
+        driver.implicitly_wait(5)
+        
+        print(driver.title)
+        
+        driver.find_element_by_link_text("Students").click()
+        driver.implicitly_wait(5)
+        print(driver.title)
+        
+        driver.find_element_by_link_text("Registration").click()
+        driver.implicitly_wait(5)
+        print(driver.title)
+        
+        driver.find_element_by_link_text("Search for Classes").click()
+        driver.implicitly_wait(15)
+        print(driver.title)
+        
+        dd = Select(driver.find_element_by_name("p_term"))
+        if (dd):
+            dd.select_by_visible_text(SEMESTER)    
+            driver.find_element_by_xpath("/html/body/div/div[4]/form").submit()
+            driver.implicitly_wait(15)
+            print(driver.title)
+
+        driver.find_element_by_xpath("/html/body/div/div[4]/form/input[18]").click()   
+        driver.implicitly_wait(10)
+        print(driver.title)
+        
+        driver.find_element_by_name("SUB_BTN").click()
+        driver.implicitly_wait(40)
+        time.sleep(15)
+        driver.implicitly_wait(40)
+        print(driver.title)
+        text = driver.page_source
+        driver.quit()
+        
+    except Exception as e:
+        print("Got an exception: ", e)
+    finally:
+        print("")
+        #driver.quit()
+    
+          
+                                                   
+                                      
+              
+    
+    codecs.open('cache/' + filename_html,'w', 'utf-8').write(text)
+    
+    
+    
+    #print(text)
+    as_list = ssb_to_csv(text)
+    #print(as_list)
+    as_dict = to_section_list(as_list)
+    jj = json.dumps(as_dict,indent=2)
+    
+    # TODO
+    try:
+        ps = codecs.open('cache/'+filename,'r','utf-8')
+        prev_sched = json.loads(ps.read())
+        ps.close()
+        
+        if 1:   # sometimes I want to re-run this without affecting the logs.
+            log_section_filling(as_dict)
+            log_section_filling2(as_dict)
+        
+        dd = DeepDiff(prev_sched, as_dict, ignore_order=True)
+        pretty_json = json.dumps(  json.loads( dd.to_json() ), indent=2 )
+        codecs.open('cache/%s_sched_diff.json' % short_sem,'w','utf-8').write(  pretty_json )    # dd.to_json() )
+        
+    except Exception as e:
+        print(e)
+        print("Can't do diff?")
+    
+    # Next, rename the prev sched_xxYY.json data file to have its date,
+    # make this new one, and then upload it to the website. 
+    # Maybe even count the entries and do a little sanity checking
+    #
+    # print("Last modified: %s" % time.ctime(os.path.getmtime("test.txt")))
+    # print("Created: %s" % time.ctime(os.path.getctime("test.txt")))
+    
+                                       
+    try:
+        last_mod = time.ctime(os.path.getmtime('cache/' + filename))
+        
+        import pathlib
+        prev_stat = pathlib.Path('cache/' + filename).stat()
+        mtime = dt.fromtimestamp(prev_stat.st_mtime)
+        print(mtime)
+    except:
+        print("Couldn't Diff.")
+    # fname = pathlib.Path('test.py')
+    # assert fname.exists(), f'No such file: {fname}'  # check that the file exists
+    # print(fname.stat())
+    #
+    # os.stat_result(st_mode=33206, st_ino=5066549581564298, st_dev=573948050, st_nlink=1, st_uid=0, st_gid=0, st_size=413, 
+    #                st_atime=1523480272, st_mtime=1539787740, st_ctime=1523480272)
+
+
+
+    codecs.open('cache/' + filename, 'w', 'utf-8').write(jj)
+    
+                                                                                                         
+    
+    put_file('/web/phowell/schedule/', 'cache/', filename, 0)                             #  /gavilan.edu/_files/php/
+    
+    return as_dict
+ 
+def dza_sched():
+    text = codecs.open('cache/sched_fa22_deanza.html','r','utf-8').read()
+    as_list = ssb_to_csv(text)
+    #print(as_list)
+    as_dict = to_section_list(as_list)
+    jj = json.dumps(as_dict,indent=2)
+    codecs.open('cache/fa22_sched_deanza.json','w','utf-8').write(jj)
+    
+# recreate schedule json files with most current online schedule format. 
+def recent_schedules():
+    #  # todo: sems is a global in this file. Is that the right thing to do?   
+    #all_scheds = [ os.listdir( 'cache/rosters/' + shortToLongSem(s)) for s in sems ]
+    #for i,s in enumerate(sems):
+    for s in ['sp21',]:
+        filename = 'cache/sched_' + s + '.html'
+        print("Filename is %s" % filename)
+        input = codecs.open( filename, 'r', 'utf-8').read()
+        output = ssb_to_csv(input)
+        
+        csv_fn = 'cache/temp_sched_' + s + '.csv'
+        if os.path.isfile(csv_fn):
+            os.remove(csv_fn)
+        
+        codecs.open(csv_fn,'w','utf-8').write(output)
+        
+        jsn = to_section_list(output)
+        jsn_fn = 'cache/semesters/'+shortToLongSem(s)+'/'+s+'_sched.json'
+        if os.path.isfile(jsn_fn):
+            os.remove(jsn_fn)
+        codecs.open(jsn_fn,'w').write(json.dumps(jsn))
+    print("I put the most recent schedule JSON files in ./cache/semesters/... folders.")
+
+
+
+
+
+################
+################  ROSTERS AND REGISTRATION
+################
+################
+################
+
+# todo: the pipeline is disorganized. Organize it to have 
+# a hope of taking all this to a higher level.
+#     
+
+# todo: where does this belong in the pipeline? compare with recent_schedules()
+
+
+
+# Take the generically named rosters uploads files and move them to a semester folder and give them a date.    
+def move_to_folder(sem,year,folder):
+    semester = year+sem
+    semester_path = 'cache/rosters/%s' % semester
+    now = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M')
+    print("+  Moving roster files to folder: %s" % semester_path)
+    if not os.path.isdir(semester_path):
+        print("+  Creating folder: %s" % semester_path)
+        os.makedirs(semester_path)
+    os.rename('cache/rosters/courses-%s.csv' % folder,     'cache/rosters/%s/courses.%s.csv' % (semester,now))
+    os.rename('cache/rosters/enrollments-%s.csv' % folder, 'cache/rosters/%s/enrollments.%s.csv' % (semester,now))
+    os.rename('cache/rosters/users-%s.csv' % folder,       'cache/rosters/%s/users.%s.csv' % (semester,now))
+    
+
+
+# Take raw upload (csv) files and make one big json out of them. 
+# This relates to enrollment files, not schedule. 
+def convert_roster_files(semester="",year="",folder=""):
+    if not semester:
+        semester = input("the semester? (ex: spring) ")
+        folder = input("Folder? (ex 2020-02-25-14-58-20) ")
+    uf = open('cache/rosters/users-'+folder+'.csv','r')
+    cf = open('cache/rosters/courses-'+folder+'.csv','r')
+    ef = open('cache/rosters/enrollments-'+folder+'.csv','r')
+    u = csv.DictReader(uf)
+    c = csv.DictReader(cf)
+    e = csv.DictReader(ef)
+    uu = [i for i in u]
+    cc = [i for i in c]
+    ee = [i for i in e]
+    uf.close()
+    cf.close()
+    ef.close()
+    myrosterfile = 'cache/rosters/roster_%s_%s.json' % (year, semester) 
+
+    if os.path.exists(myrosterfile):
+        print("  --  Moving previous combined roster json file. opening %s ..." % myrosterfile)
+        last_fileobj = open(myrosterfile,'r')
+        last_file = json.load(last_fileobj)
+                             
+        last_fileobj.close()
+
+        info = last_file[3]
+        last_date = info['date_filestring']
+        
+        print('  --  writing:  cache/rosters/%s%s/roster_%s.json ...' % (year,semester,last_date))
+        
+        try:
+            os.rename(myrosterfile, 'cache/rosters/%s%s/roster_%s.json ...' % (year,semester,last_date))
+            print('  --  ok')
+        except Exception as e:
+            print("  **  Failed because i couldn't move the previous roster file: %s" % myrosterfile)
+            print(e)
+            myrosterfile = "new_" + myrosterfile
+            pass
+            #os.remove('cache/old_rosters/roster_'+semester+'.'+last_date+'.json')
+            #os.rename(myrosterfile, 'cache/old_rosters/roster_'+semester+'.'+last_date+'.json')
+
+    newinfo = {'date_filestring': datetime.datetime.now().strftime('%Y-%m-%dT%H-%M'), }
+    try:
+        new_roster = codecs.open(myrosterfile,'w', 'utf-8')
+        new_roster.write( json.dumps( [uu,cc,ee,newinfo], indent=2 ))
+        new_roster.close()
+        print("  --  Wrote roster info to: %s." % myrosterfile)
+    except Exception as e:
+        print("  **  Failed because i couldn't move the previous roster file: %s" % myrosterfile)
+        print("  **  " + str(e))
+            
+                                
+                                              
+
+
+
+# From instructure sftp site
+def fetch_current_rosters():
+    dt_label = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    cnopts = pysftp.CnOpts()
+    cnopts.hostkeys = None
+    with pysftp.Connection(instructure_url,username=instructure_username, private_key=instructure_private_key,cnopts=cnopts) as sftp:
+        sftp.chdir('SIS')
+        files = sftp.listdir()
+        print("\n--> %s I see these files at instructure ftp site: " % dt_label )
+        [print("   %s" % f) for f in files]
+        i = 0
+        got_courses = 0
+        if len(files)>1:  # and 'users.csv' in files:
+            try:
+                if 'users.csv' in files:
+                    sftp.get('users.csv','cache/rosters/users-'+dt_label+'.csv')
+                    i += 1
+            except:
+                print(' * users.csv not present')
+            try:
+                if 'courses.csv' in files:
+                    sftp.get('courses.csv','cache/rosters/courses-'+dt_label+'.csv')
+                    i += 1
+                    got_courses = 1
+            except:
+                print(' * courses.csv not present')
+            try:
+               if 'enrollments.csv' in files:
+                    sftp.get('enrollments.csv','cache/rosters/enrollments-'+dt_label+'.csv')
+                    i += 1
+            except:
+                print(' * enrollments.csv not present')
+            print('   Saved %i data files in rosters folder.' % i)
+            
+            if got_courses:
+                courses = open('cache/rosters/courses-%s.csv' % dt_label,'r')
+                courses.readline()
+                a = courses.readline()
+                print(a)
+                courses.close()
+                parts = a.split(',')
+                year = parts[1][0:4]
+                ss = parts[1][4:6]
+                #print parts[1]
+                sem = {'30':'spring', '50':'summer', '70':'fall' }
+                this_sem = sem[ss]
+                print(" -> This semester is: %s, %s" % (this_sem,year))
+                
+                print(' -> %s building data file...' % dt_label)
+                convert_roster_files(this_sem,year,dt_label)
+                print(' -> moving files...')
+                move_to_folder(this_sem,year,dt_label)
+            else:
+                print(" * No courses file. Not moving files.")
+        else:
+            print("--> Don't see files.")
+    sftp.close()
+
+def fetch_current_rosters_auto():
+    
+    schedule.every().hour.at(":57").do(fetch_current_rosters)
+    
+    schedule.every().day.at("12:35").do(sync_non_interactive)
+    schedule.every().day.at("21:00").do(sync_non_interactive)
+    
+    
+    print("running every hour on the :57\n")
+    while True:
+        try:
+            schedule.run_pending()
+        except Exception as e:
+            import traceback
+            print(" ---- * * * Failed with: %s" % str(e))
+            ff = open('cache/pipeline.log.txt','a')
+            ff.write(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + "\n")
+            ff.write(traceback.format_exc()+"\n---------\n\n")
+            ff.close()
+            #schedule.CancelJob
+        time.sleep(15)
+
+
+
+    
+# read schedule file with an eye toward watching what's filling up
+def schedule_filling():
+    sem = 'spring2021'    # todo: hardcoded
+    days = []
+    for f in sorted(os.listdir('cache/rosters/'+sem+'/')):
+        if f.endswith('.html'):
+            match = re.search(r'sched_(\d\d\d\d)_(\d\d)_(\d+)\.html',f)
+            if match:
+                print(f)
+                y = match.group(1)
+                m = match.group(2)
+                d = match.group(3)
+                print("Schedule from %s %s %s." % (y,m,d))
+                csv_sched = ssb_to_csv(open('cache/rosters/'+sem+'/'+f,'r').read())
+                jsn = to_section_list(csv_sched)
+                #print(json.dumps(jsn,indent=2))
+                days.append(jsn)
+    day1 = days[-2]
+    day2 = days[-1]
+    df = jsondiff.diff(day1, day2)
+    gains = defaultdict( list )
+    
+    for D in df.keys():
+        if isinstance(D, int):
+            #print(day1[D]['code'] + '\t' + day1[D]['crn'] + ' Before: ' + day1[D]['act'] + ' After: ' + day2[D]['act'])
+            try:
+                gain = int(day2[D]['act']) - int(day1[D]['act'])
+                gains[gain].append(  day1[D]['code'] + ' ' + day1[D]['crn'] )
+            except:
+                print("No gain for " + str(D))
+            #print("\t" + str(df[D]))
+        else:
+            print(D)
+            print(df[D])
+    for key, value in sorted(gains.items(), key=lambda x: x[0]): 
+        print("{} : {}".format(key, value))
+    
+    #print(json.dumps(gains,indent=2))
+
+    
+
+
+
+
+
+################
+################  SENDING DATA AWAY
+################
+################
+################
+
+# Upload a json file to www
+def put_file(remotepath,localpath, localfile,prompt=1):
+    folder = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    cnopts = pysftp.CnOpts()
+    
+    cnopts.hostkeys = None
+
+    with pysftp.Connection(FTP_SITE,username=FTP_USER, password=FTP_PW,cnopts=cnopts) as sftp:
+        #todo: these paths
+        #files = sftp.listdir()
+        #print(folder + "\tI see these files on remote: ", files, "\n")
+        sftp.chdir(remotepath)
+        files = sftp.listdir()
+        print(folder + "\tI see these files on remote: ", files, "\n")
+        
+        localf = os.listdir(localpath)
+        
+        print("I see these local: ", localf)
+        
+        if prompt:
+            input('ready to upload')
+        sftp.put(localpath+localfile, localfile, preserve_mtime=True)
+        sftp.close()
+       
+        
+        """
+        # copy files and directories from local static, to remote static,
+        # preserving modification times on the files
+        for f in localf:
+            print("This local file: " + f + " ", end=' ')
+            if not f in files: 
+                sftp.put('video_srt/'+classfoldername+'/'+f, f, preserve_mtime=True)
+                print("Uploaded.")
+            else:
+                print("Skipped.")
+        """
+
+        """if len(files)==3 and 'users.csv' in files:
+            sftp.get('courses.csv','rosters/courses-'+folder+'.csv')
+            sftp.get('users.csv','rosters/users-'+folder+'.csv')
+            sftp.get('enrollments.csv','rosters/enrollments-'+folder+'.csv')
+            print folder + '\tSaved three data files in rosters folder.'
+            
+            courses = open('rosters/courses-'+folder+'.csv','r')
+            courses.readline()
+            a = courses.readline()
+            print a
+            courses.close()
+            parts = a.split(',')
+            year = parts[1][0:4]
+            ss = parts[1][4:6]
+            #print parts[1]
+            sem = {'30':'spring', '50':'summer', '70':'fall' }
+            this_sem = sem[ss]
+            #print this_sem, "", year
+            print folder + '\tbuilding data file...'
+            convert_roster_files(this_sem,year,folder)
+            print folder + '\tmoving files...'
+            move_to_folder(this_sem,year,folder)
+        else:
+            print folder + "\tDon't see all three files."""
+    
+    
+    
+################
+################  GOOGLE DOCS
+################
+################
+################
+
+def sec(t): return "<h3>"+t+"</h3>\n"
+def para(t): return "<p>"+t+"</p>\n"
+def ul(t): return "<ul>"+t+"</ul>\n"
+def li(t): return "<li>"+t+"</li>\n"
+
+def question(t,bracket=1): 
+    ret = ''
+    match = re.search( r'\[(.*)\]', t)
+    if match and bracket:
+        ret += "<a name='" + match.group(1) + "'></a>"
+        t = re.sub( r'\[.*\]','',t)
+    else:
+        parts = t.split(' ')
+        id = ''
+        for p in parts:
+            if re.search(r'[a-zA-Z]',p[0]): id += p[0]
+        ret += "<a name='%s'></a>" % id.lower()
+    return ret + '<div class="accordion" data-accordion=""><h4 class="acrd_cntl">' + t + '</h4>\n<div class="acrd_cntnt">'
+    
+def answer(t): 
+    return t + '</div></div>\n'
+
+def read_paragraph_element(element,type="NORMAL_TEXT"):
+    """Returns the text in the given ParagraphElement.
+
+        Args:
+            element: a ParagraphElement from a Google Doc.
+    """
+    text_run = element.get('textRun')
+    begin = ''
+    end = ''
+    if not text_run:
+        return ''
+    if 'textStyle' in text_run and 'link' in text_run['textStyle']:
+        begin = '<a href="' + text_run['textStyle']['link']['url'] + '">'
+        end = '</a>'
+    if 'textStyle' in text_run and 'bold' in text_run['textStyle'] and text_run['textStyle']['bold']==True and type=="NORMAL_TEXT":
+        begin = '<strong>' + begin
+        end = end + '</strong>'
+    
+    content = text_run.get('content')
+    content = re.sub(u'\u000b','<br />\n',content)
+    
+    return begin + content + end
+    
+
+def get_doc(docid, bracket=1, verbose=0):
+    import pickle
+    import os.path
+    from googleapiclient.discovery import build
+    from google_auth_oauthlib.flow import InstalledAppFlow
+    from google.auth.transport.requests import Request
+    
+    #ooout = open(fileout,'w')
+
+    # If modifying these scopes, delete the file token.pickle.
+    SCOPES = ['https://www.googleapis.com/auth/documents.readonly']
+    creds = None
+    # The file token.pickle stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    if os.path.exists('token.pickle'):
+        with open('token.pickle', 'rb') as token:
+            creds = pickle.load(token)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(
+                'credentials.json', SCOPES)
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open('token.pickle', 'wb') as token:
+            pickle.dump(creds, token)
+
+    service = build('docs', 'v1', credentials=creds)
+
+    # Retrieve the documents contents from the Docs service.
+    document = service.documents().get(documentId=docid).execute()
+    if verbose: print(document)
+        
+    tempout = codecs.open('cache/trash/gdoctemp.txt','w','utf-8')
+    tempout.write( json.dumps(document,indent=2) + "\n\n\n------------------------------------\n\n")
+    if verbose: print('The title of the document is: {}'.format(document.get('title'))) 
+    doc_content = document.get('body').get('content')
+    if verbose: print(doc_content)
+    
+    doc_objects = document.get('inlineObjects')
+    if verbose: print(doc_objects)
+    
+    doc_lists = document.get('lists')
+    
+    text = '<div class="acrd_grp" data-accordion-group="">'
+    last_type = ''
+    answer_text = ''
+    in_a_list = ''
+    
+    img_count = 1
+    img_lookup = {}
+    img_heights = {}
+    img_widths = {}
+    
+    if doc_objects:
+        for k,value in doc_objects.items():
+            tempout.write( "->" + k + "=" + json.dumps(value,indent=2) + "\n\n\n--\n\n")
+            if 'inlineObjectProperties' in value:
+                if 'embeddedObject' in value['inlineObjectProperties']:
+                    if 'imageProperties' in value['inlineObjectProperties']['embeddedObject']:
+                        if 'contentUri' in value['inlineObjectProperties']['embeddedObject']['imageProperties']:
+                            print(k)
+                            uu = value['inlineObjectProperties']['embeddedObject']['imageProperties']['contentUri']
+                            response = requests.get(uu, stream=True)
+                            name = 'image_' + str(img_count) + '.' + response.headers['content-type'].split('/')[1]
+                            img_count += 1
+                            
+                            img_lookup[k] = name
+                            
+                            with open('cache/doc_images/'+name, 'wb') as out_file:
+                                shutil.copyfileobj(response.raw, out_file)
+                            print(uu)
+                            print(response.headers)
+                            print(name)
+                            #input('x?')
+                            del response
+                    if 'size' in  value['inlineObjectProperties']['embeddedObject']: 
+                        img_heights[k] = int(value['inlineObjectProperties']['embeddedObject']['size']['height']['magnitude'])
+                        img_widths[k] = int(value['inlineObjectProperties']['embeddedObject']['size']['width']['magnitude'])
+        
+    tempout.write('- - - - - - - -\n\n')
+    #for value in doc_lists:
+    #    tempout.write( json.dumps(value,indent=2) + "\n\n\n--\n\n")
+    
+    tempout.write('- - - - - - - -\n\n')
+    list_stack = []
+    list_depth = 0
+    last_list_depth = 0
+    for value in doc_content:
+        tempout.write( json.dumps(value,indent=2) + "\n\n\n")
+        if verbose: print(json.dumps(value, sort_keys=True, indent=4))
+
+        # todo: x link, x bold, list, image.
+        tag_fxn = para
+        if 'paragraph' in value:
+            this_text = ''
+            
+            if 'bullet' in value['paragraph']:
+                # either we're (1)starting a new list, (2)in one, (3)starting a nested one, or (4)finished a nested one. 
+                
+                lid = value['paragraph']['bullet']['listId']
+                
+                if not list_stack:  # 1
+                    list_stack.append(lid)
+                else:
+                    if lid == list_stack[0]:   # 2
+                        pass
+                        
+                    else:
+                        if not lid in list_stack:   # 3
+                            list_stack.append(lid)
+                        else:                       # 4
+                            x = list_stack.pop()
+                            while x != lid: list_stack.pop()
+            elif len(list_stack) > 0:                           #  current para isn't a bullet but we still have a list open.             
+                list_stack = []
+            
+            list_depth = len(list_stack)
+            
+            deeper = list_depth - last_list_depth
+            
+            if deeper > 0:
+                answer_text += "<ul>" * deeper
+            elif deeper < 0:
+                deeper = -1 * deeper
+                answer_text += "</ul>" * deeper
+            
+            if len(list_stack):
+                tag_fxn = li
+                
+            elements = value.get('paragraph').get('elements')
+            
+            # inlineObjectElement": {
+            # "inlineObjectId": "kix.ssseeu8j9cfx",
+            
+            if 'paragraphStyle' in value.get('paragraph'):
+                style = value.get('paragraph').get('paragraphStyle')
+                #text += json.dumps(style, sort_keys=True, indent=4)
+                if 'namedStyleType' in style:
+                    type = style['namedStyleType']
+            
+            for elem in elements:
+                
+                # text content
+                this_text += read_paragraph_element(elem,type)
+                
+                # image content
+                if 'inlineObjectElement' in elem:
+                    vpi = elem['inlineObjectElement']
+                    if 'inlineObjectId' in vpi:
+                        ii = vpi['inlineObjectId']
+                        if ii in img_lookup:
+                            img = img_lookup[ii]
+                            h = img_heights[ii]
+                            w = img_widths[ii]
+                            this_text += '<img src="doc_images/%s" width="%i" height="%i" />' % (img,w,h)
+                
+                
+            
+            if last_type=='NORMAL_TEXT' and type!=last_type:
+                text += answer(answer_text)
+                answer_text = ''
+            
+            if type=='HEADING_2':
+                text += sec(this_text)
+                this_text = ''
+            elif type=='HEADING_3': 
+                text += question(this_text,bracket)
+                this_text = ''
+            else:
+                answer_text += tag_fxn(this_text)
+                this_text = ''
+            last_type = type
+            last_list_depth = list_depth
+                
+        elif 'table' in value:
+            # The text in table cells are in nested Structural Elements and tables may be
+            # nested.
+            text += "\nTABLE\n"
+            #table = value.get('table')
+            #for row in table.get('tableRows'):
+            #    cells = row.get('tableCells')
+            #    for cell in cells:
+            #        text += read_strucutural_elements(cell.get('content'))
+        #elif 'tableOfContents' in value:
+        #    # The text in the TOC is also in a Structural Element.
+        #    toc = value.get('tableOfContents')
+        #    text += read_strucutural_elements(toc.get('content'))
+        
+        #else:
+        #    print(json.dumps(value, sort_keys=True, indent=4))
+    
+    text += answer(answer_text)
+    #text += '</div>'
+    #print(text)
+    return text
+    
+######### TRY #2 ######    
+
+
+def read_paragraph_element_2(element,type="NORMAL_TEXT"):
+    text_run = element.get('textRun')
+    begin = ''
+    end = ''
+    if not text_run: return ''
+    if 'textStyle' in text_run and 'link' in text_run['textStyle']:
+        begin = '<a href="' + text_run['textStyle']['link']['url'] + '">'
+        end = '</a>'
+    if 'textStyle' in text_run and 'bold' in text_run['textStyle'] and text_run['textStyle']['bold']==True and type=="NORMAL_TEXT":
+        begin = '<strong>' + begin
+        end = end + '</strong>'
+    elif 'textStyle' in text_run and 'italic' in text_run['textStyle'] and text_run['textStyle']['italic']==True and type=="NORMAL_TEXT":
+        begin = '<em>' + begin
+        end = end + '</em>'
+    content = text_run.get('content')
+    content = re.sub(u'\u000b','<br />\n',content)
+    return begin + content + end
+
+# t is a string that begins with "Icons: " ... and contains comma(space) separated list
+def handle_icons(t):
+    text = t[7:].strip()
+    parts = text.split(", ")
+    return ('icons',parts)
+
+# t is a string that begins with "Tags: " ... and contains comma(space) separated list
+def handle_tags(t):
+    text = t[6:].strip()
+    parts = text.split(", ")
+    return ('tags',parts)
+
+def handle_question(t,bracket=1):
+    anchor = ''
+    match = re.search( r'\[(.*)\]', t)
+    if match and bracket:
+        anchor = match.group(1).lower()
+        t = re.sub( r'\[.*\]','',t)
+    else:
+        parts = t.split(' ')
+        for p in parts:
+            if re.search(r'[a-zA-Z]',p[0]): anchor += p[0].lower()
+    return ('question', t, anchor)
+
+def handle_answer(t):
+    return ('answer',t)
+
+def handle_sec(t): return ('section',t)
+def handle_para(t): return ('paragraph',t)
+def handle_ul(t): return ('unorderdedlist',t)
+def handle_li(t): return ('listitem',t)
+
+
+
+img_count = 1
+img_lookup = {}
+img_heights = {}
+img_widths = {}
+
+
+def fetch_doc_image(k,value):
+    global img_count, img_lookup, img_heights, img_widths
+    if 'inlineObjectProperties' in value:
+        if 'embeddedObject' in value['inlineObjectProperties']:
+            if 'imageProperties' in value['inlineObjectProperties']['embeddedObject']:
+                if 'contentUri' in value['inlineObjectProperties']['embeddedObject']['imageProperties']:
+                    print(k)
+                    uu = value['inlineObjectProperties']['embeddedObject']['imageProperties']['contentUri']
+                    response = requests.get(uu, stream=True)
+                    name = 'image_' + str(img_count) + '.' + response.headers['content-type'].split('/')[1]
+                    img_count += 1
+                    img_lookup[k] = name
+                    
+                    with open('cache/doc_images/'+name, 'wb') as out_file:
+                        shutil.copyfileobj(response.raw, out_file)
+                    print(uu)
+                    print(response.headers)
+                    print(name)
+                    del response
+            if 'size' in  value['inlineObjectProperties']['embeddedObject']: 
+                img_heights[k] = int(value['inlineObjectProperties']['embeddedObject']['size']['height']['magnitude'])
+                img_widths[k] = int(value['inlineObjectProperties']['embeddedObject']['size']['width']['magnitude'])
+        
+
+def get_doc_generic(docid, bracket=1, verbose=0):
+    import pickle
+    import os.path
+    from googleapiclient.discovery import build
+    from google_auth_oauthlib.flow import InstalledAppFlow
+    from google.auth.transport.requests import Request
+    global img_count, img_lookup, img_heights, img_widths
+
+# If modifying these scopes, delete the file token.pickle.
+    SCOPES = ['https://www.googleapis.com/auth/documents.readonly']
+    creds = None
+    # The file token.pickle stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    if os.path.exists('token.pickle'):
+        with open('token.pickle', 'rb') as token:
+            creds = pickle.load(token)
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(
+                'credentials.json', SCOPES)
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open('token.pickle', 'wb') as token:
+            pickle.dump(creds, token)
+
+    service = build('docs', 'v1', credentials=creds)
+
+    # Retrieve the documents contents from the Docs service.
+    document = service.documents().get(documentId=docid).execute()
+    
+    tempout = codecs.open('cache/trash/gdoctemp.txt','w','utf-8')
+    tempout.write( json.dumps(document,indent=2) \
+        + "\n\n\n------------------------------------\n\n")
+    if verbose: print('The title of the document is: {}'.format(document.get('title'))) 
+
+    doc_content = document.get('body').get('content')
+    doc_objects = document.get('inlineObjects')
+    doc_lists = document.get('lists')
+    
+    #text = ''
+    result = []
+    last_type = ''
+    #answer_text = ''
+    answer = []
+    in_a_list = ''
+    
+    # Get all the images
+    for k,value in doc_objects.items():
+        tempout.write( "->" + k + "=" + json.dumps(value,indent=2) + "\n\n\n--\n\n")
+        fetched = fetch_doc_image(k,value)
+
+    list_stack = []
+    list_depth = 0
+    last_list_depth = 0
+    for value in doc_content:
+        tempout.write( json.dumps(value,indent=2) + "\n\n\n")
+        if verbose: print(json.dumps(value, sort_keys=True, indent=4))
+
+        tag_fxn = handle_para
+        if 'paragraph' in value:
+            this_text = ''
+            
+            # First we deal with if we're in a list.
+            if 'bullet' in value['paragraph']:
+                # either we're (1)starting a new list, (2)in one (do nothing), 
+                #  (3)starting a nested one, or (4)finished a nested one. 
+                lid = value['paragraph']['bullet']['listId']
+                if not list_stack:  # 1
+                    list_stack.append(lid)
+                else:
+                    if not lid == list_stack[0]:
+                        if not lid in list_stack:   # 3
+                            list_stack.append(lid)
+                        else:                       # 4
+                            x = list_stack.pop()
+                            while x != lid: list_stack.pop()
+            elif len(list_stack) > 0:    
+                #  current para isn't a bullet but we still have a list open.             
+                list_stack = []
+                
+                
+            list_depth = len(list_stack)
+            deeper = list_depth - last_list_depth
+            if deeper > 0:
+                answer.append("<ul>" * deeper)
+            elif deeper < 0:
+                deeper = -1 * deeper
+                answer.append("</ul>" * deeper)
+            if len(list_stack):
+                tag_fxn = handle_li
+            
+            # NOW the tag_fxn is either 'para' or 'li'... let's get the styling info next,
+            elements = value.get('paragraph').get('elements')
+            if 'paragraphStyle' in value.get('paragraph'):
+                style = value.get('paragraph').get('paragraphStyle')
+                if 'namedStyleType' in style:
+                    type = style['namedStyleType']
+            
+            # and FINALLY, the actual contents.
+            for elem in elements:
+                # text content
+                this_text += read_paragraph_element_2(elem,type)
+                
+                # image content
+                if 'inlineObjectElement' in elem:
+                    vpi = elem['inlineObjectElement']
+                    if 'inlineObjectId' in vpi:
+                        ii = vpi['inlineObjectId']
+                        if ii in img_lookup:
+                            img = img_lookup[ii]
+                            h = img_heights[ii]
+                            w = img_widths[ii]
+                            this_text += '<img src="doc_images/%s" width="%i" height="%i" />' % (img,w,h)
+                
+                
+            # Now for something tricky. Call an appropriate handler, based on:
+            #  (a) what is the paragraph style type?
+            #  (b) is it different from the prev one?
+            
+            if last_type=='NORMAL_TEXT' and type!=last_type:
+                if this_text.strip():
+                    result.append(handle_answer(answer))
+                answer = []
+                #answer_text = ''
+            
+            if type=='HEADING_2' and this_text.strip():
+                result.append( handle_sec(this_text) )
+                this_text = ''
+            elif type=='HEADING_3' and this_text.strip(): 
+                result.append(handle_question(this_text,bracket))
+                this_text = ''
+            else:
+                if this_text.lower().startswith('tags:'):
+                    tag_fxn = handle_tags
+                if this_text.lower().startswith('icons:'):
+                    tag_fxn = handle_icons
+                if this_text.strip():
+                    answer.append(tag_fxn(this_text))
+                this_text = ''
+            last_type = type
+            last_list_depth = list_depth
+                
+        elif 'table' in value:
+            pass
+    
+                                      
+    result.append(handle_answer(answer))
+    return json.dumps(result,indent=4)
+
+
+
+
+def scrape_schedule_py():
+    return 1
+    
+    """
+    cur_session = requests.Session() 
+    mygav_url = "https://lum-prod.ec.gavilan.edu/"
+    
+    r1 = requests.get(mygav_url)
+    
+    login_url1 = "https://lum-prod.ec.gavilan.edu/c/portal/login"
+    
+    
+    login_url = "https://eis-prod.ec.gavilan.edu/authenticationendpoint/login.do?commonAuthCallerPath=%2Fsamlsso&forceAuth=false&passiveAuth=false&tenantDomain=carbon.super&sessionDataKey=57203341-6823-4511-b88e-4e104aa2fd71&relyingParty=LP5PROD_LuminisPortalEntity&type=samlsso&sp=Luminis+Portal+PROD&isSaaSApp=false&authenticators=BasicAuthenticator:LOCAL"
+    """
+
+
+
+def scrape_schedule_multi():
+    
+    global SEMESTER, short_sem, semester_begin, filename, filename_html
+    
+    SEMESTER = 'Summer 2022'
+    short_sem = 'su22'
+    semester_begin = strptime('06/13', '%m/%d')
+    filename = 'su22_sched.json'
+    filename_html = 'su22_sched.html'
+    
+    scrape_schedule()
+    
+    SEMESTER = 'Fall 2022'
+    short_sem = 'fa22'
+    semester_begin = strptime('08/22', '%m/%d')
+    filename = 'fa22_sched.json'
+    filename_html = 'fa22_sched.html'
+    
+    scrape_schedule()
+    
+    SEMESTER = 'Spring 2022'
+    short_sem = 'sp22'
+    semester_begin = strptime('01/31', '%m/%d')
+    filename = 'sp22_sched.json'
+    filename_html = 'sp22_sched.html'
+    
+    scrape_schedule()
+    
+
+
+if __name__ == "__main__":
+    
+    print ('')
+    options = { 1: ['Re-create schedule csv and json files from raw html',recent_schedules] ,  
+                2: ['Fetch rosters',fetch_current_rosters] , 
+                3: ['Fetch rosters AND canvas data automatically',fetch_current_rosters_auto] , 
+                4: ['Compute how registration is filling up classes', schedule_filling] , 
+                5: ['Manually convert 3 csv files to joined json enrollment file.', convert_roster_files] , 
+                6: ['Canvas data: interactive sync', interactive ], 
+                7: ['Canvas data: automated sync', sync_non_interactive ], 
+                8: ['Scrape schedule from ssb', scrape_schedule_multi ], 
+                9: ['Test ssb calls with python', scrape_schedule_py ], 
+                10: ['Parse deanza schedule', dza_sched ],
+              }
+    
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+        
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+    
+# Testing
+
+#if __name__ == "__main__":
+    #users = fetch('/api/v1/courses/69/users?per_page=100',1)
+    #print "These are the users: "
+    #print users
+    
+    #getSemesterSchedule()
+    
+    
+    #get_doc()
+    #pass
diff --git a/queries.sql b/queries.sql
new file mode 100644
index 0000000..2e61a94
--- /dev/null
+++ b/queries.sql
@@ -0,0 +1,188 @@
+
+
+# TODO   students enrolled in fall 2020 
+
+
+## Fall 2020 students with how many classes theyre taking
+
+SELECT u.canvasid, u.name, u.sortablename, COUNT(e.id) AS num FROM enrollment AS e
+JOIN users AS u ON e.user_id=u.id
+JOIN courses AS c ON e.course_id=c.id
+WHERE c.sis LIKE "202070-%" 
+AND e.workflow="active"
+AND e."type"="StudentEnrollment"
+GROUP BY u.canvasid
+
+
+
+## All sections offered in Fall 2020
+
+SELECT c.id, c.canvasid, c.name, c.code  FROM courses AS c
+WHERE c.sis LIKE "202070-%"
+AND NOT c.state="deleted"
+ORDER BY c.code ASC
+
+
+
+## All Teachers teaching in Fall 2020
+
+SELECT c.id, c.canvasid AS course_cid, c.name, c.code, u.name, u.sortablename, u.canvasid AS user_cid  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.sis LIKE "202070-%"
+AND NOT c.state="deleted"
+AND e."type"="TeacherEnrollment"
+ORDER BY u.sortablename 
+--- ORDER BY c.code ASC
+
+
+
+
+## All Teachers teaching in Fall 2020 -> how many classes each has
+
+SELECT u.name, u.sortablename, COUNT(c.id) AS num, GROUP_CONCAT(c.code, ", ") AS courses, u.canvasid AS user_cid  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.sis LIKE "202070-%"
+AND NOT c.state="deleted"
+AND e."type"="TeacherEnrollment"
+GROUP BY user_cid
+ORDER BY courses
+--- ORDER BY u.sortablename 
+--- ORDER BY c.code ASC
+
+
+
+-- ## Fall 2020 teachers with NO ACTIVITY
+SELECT c.id AS courseid, u.id AS userid, c.code, u.name  FROM courses AS c
+JOIN enrollment AS e ON e.course_id=c.id
+JOIN users AS u ON u.id=e.user_id
+WHERE c.sis LIKE "202070-%"
+AND NOT c.state="deleted"
+AND e."type"="TeacherEnrollment"
+AND u.id NOT IN (
+	SELECT r.userid FROM requests_sum1 AS r
+)
+ORDER BY u.sortablename 
+
+
+
+-- ## Activity of Fall 2020 teachers
+
+SELECT t.code, t.name, SUM(r.viewcount) FROM requests_sum1 AS r 
+JOIN (
+	SELECT c.id AS courseid, u.id AS userid, c.code, u.name  FROM courses AS c
+	JOIN enrollment AS e ON e.course_id=c.id
+	JOIN users AS u ON u.id=e.user_id
+	WHERE c.sis LIKE "202070-%"
+	AND NOT c.state="deleted"
+	AND e."type"="TeacherEnrollment"
+	--GROUP BY u.id
+	ORDER BY u.sortablename 
+) AS t ON r.userid=t.userid AND r.courseid=t.courseid
+GROUP BY r.userid, r.courseid
+
+
+
+
+######   Students who are new in FALL 2020
+
+SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e
+JOIN users AS u ON e.user_id=u.id
+JOIN courses AS c ON e.course_id=c.id
+WHERE c.sis LIKE "202070-%" 
+AND e.workflow="active"
+AND e."type"="StudentEnrollment"
+AND u.canvasid NOT IN (
+	SELECT u.canvasid FROM enrollment AS e
+	JOIN users AS u ON e.user_id=u.id
+	JOIN courses AS c ON e.course_id=c.id
+	WHERE c.sis NOT LIKE "202070-%" 
+	AND e.workflow="active"
+	AND e."type"="StudentEnrollment"
+	GROUP BY u.canvasid
+)
+GROUP BY u.canvasid
+ORDER BY num DESC, u.sortablename
+
+
+
+
+######   Students who are new in 2020
+
+SELECT u.canvasid, u.name, u.sortablename, GROUP_CONCAT(c.code), COUNT(e.id) AS num FROM enrollment AS e
+JOIN users AS u ON e.user_id=u.id
+JOIN courses AS c ON e.course_id=c.id
+WHERE c.sis LIKE "202070-%" 
+AND e.workflow="active"
+AND e."type"="StudentEnrollment"
+AND u.canvasid NOT IN (
+	SELECT u.canvasid FROM enrollment AS e
+	JOIN users AS u ON e.user_id=u.id
+	JOIN courses AS c ON e.course_id=c.id
+	WHERE c.sis NOT LIKE "202070-%" 
+	AND  c.sis NOT LIKE "202050-%" 
+	AND  c.sis NOT LIKE "202030-%" 
+	AND e.workflow="active"
+	AND e."type"="StudentEnrollment"
+	GROUP BY u.canvasid
+)
+GROUP BY u.canvasid
+ORDER BY num DESC, u.sortablename
+
+
+
+
+
+######   Students who are new in FALL 2020  -> how many students are taking how many classes
+
+SELECT num, COUNT(num) AS class_count FROM (
+	SELECT e.id, COUNT(e.id) AS num FROM enrollment AS e
+	JOIN users AS u ON e.user_id=u.id
+	JOIN courses AS c ON e.course_id=c.id
+	WHERE c.sis LIKE "202070-%" 
+	AND e.workflow="active"
+	AND e."type"="StudentEnrollment"
+	AND u.canvasid NOT IN (
+		SELECT u.canvasid FROM enrollment AS e
+		JOIN users AS u ON e.user_id=u.id
+		JOIN courses AS c ON e.course_id=c.id
+		WHERE c.sis NOT LIKE "202070-%" 
+		AND e.workflow="active"
+		AND e."type"="StudentEnrollment"
+		GROUP BY u.canvasid
+	)
+	GROUP BY u.id
+)
+GROUP BY num
+ORDER BY num DESC
+
+
+
+######   All FALL 2020 students  -> how many students are taking how many classes
+
+SELECT num, COUNT(num) AS class_count FROM (
+	SELECT e.id, COUNT(e.id) AS num FROM enrollment AS e
+	JOIN users AS u ON e.user_id=u.id
+	JOIN courses AS c ON e.course_id=c.id
+	WHERE c.sis LIKE "202070-%" 
+	AND e.workflow="active"
+	AND e."type"="StudentEnrollment"
+	GROUP BY u.id
+)
+GROUP BY num
+ORDER BY num DESC
+
+
+
+
+#####   Students who are NOT enrolled in Fall 2020
+
+SELECT u.canvasid, u.name, u.sortablename FROM enrollment AS e
+JOIN users AS u ON e.user_id=u.id
+JOIN courses AS c ON e.course_id=c.id
+WHERE c.sis NOT LIKE "202070-%" 
+AND e.workflow="active"
+AND e."type"="StudentEnrollment"
+GROUP BY u.canvasid
+
diff --git a/requirements.2019.txt b/requirements.2019.txt
new file mode 100644
index 0000000..778f5d3
--- /dev/null
+++ b/requirements.2019.txt
@@ -0,0 +1,61 @@
+beautifulsoup4==4.6.3
+bs4==0.0.1
+cachetools==3.1.1
+certifi==2018.8.24
+docx==0.2.4
+durable-rules==2.0.11
+google-api-python-client==1.7.11
+google-auth==1.7.1
+google-auth-httplib2==0.0.3
+google-auth-oauthlib==0.4.1
+html2text==2018.1.9
+html5lib==1.0.1
+httplib2==0.14.0
+idna==2.7
+ipython==7.0.1
+ipython-genutils==0.2.0
+jsondiff==1.2.0
+lxml==4.2.5
+Markdown==3.0.1
+numpy==1.17.3
+O365==2.0.5
+oauthlib==3.1.0
+oyaml==0.9
+packaging==19.2
+pampy==0.2.1
+pandas==0.25.2
+paramiko==2.6.0
+pickleshare==0.7.5
+Pillow==6.2.1
+pkginfo==1.5.0.1
+prompt-toolkit==2.0.5
+pyfiglet==0.8
+PyJWT==1.7.1
+PyNaCl==1.3.0
+pyparsing==2.4.5
+pysftp==0.2.9
+PySocks==1.6.8
+python-dateutil==2.8.0
+python-docx==0.8.10
+pytoml==0.1.21
+pytz==2018.9
+PyYAML==5.1.2
+requests==2.19.1
+requests-oauthlib==1.2.0
+rsa==4.0
+simplegeneric==0.8.1
+simpy==3.0.11
+sortedcontainers==2.1.0
+stringcase==1.2.0
+structlog==19.2.0
+tabulate==0.8.6
+textdistance==4.1.5
+toolz==0.10.0
+twilio==6.24.0
+tzlocal==2.0.0
+uritemplate==3.0.0
+urllib3==1.23
+wcwidth==0.1.7
+webencodings==0.5.1
+windows-curses==2.1.0
+yarg==0.1.9
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..739f36e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,288 @@
+alabaster==0.7.10
+anaconda-client==1.6.5
+anaconda-navigator==1.6.9
+anaconda-project==0.8.0
+asn1crypto==0.22.0
+astroid==1.5.3
+astropy==2.0.2
+atomicfile==1.0.1
+attrs==18.2.0
+Automat==0.7.0
+Babel==2.5.0
+backports.shutil-get-terminal-size==1.0.0
+bcrypt==3.1.7
+beautifulsoup4==4.6.0
+bitarray==0.8.1
+bkcharts==0.2
+blaze==0.11.3
+bleach==2.0.0
+blinker==1.4
+bokeh==0.12.10
+boto==2.48.0
+boto3==1.10.20
+botocore==1.13.20
+Bottleneck==1.2.1
+certifi==2019.9.11
+cffi==1.10.0
+chardet==3.0.4
+clfparser==0.3
+click==6.7
+cloudpickle==0.4.0
+clyent==1.2.2
+coinmarketcap==5.0.3
+colorama==0.3.9
+colorlog==4.0.2
+conda==4.3.30
+conda-build==3.0.27
+conda-verify==2.0.0
+constantly==15.1.0
+contextlib2==0.5.5
+cryptography==2.5
+cssselect==1.0.3
+cycler==0.10.0
+cymem==2.0.2
+Cython==0.28.5
+cytoolz==0.9.0.1
+dask==0.15.3
+datashape==0.5.4
+DAWG-Python==0.7.2
+decorator==4.1.2
+deepdiff==5.0.2
+deeppavlov==0.1.6
+dill==0.2.9
+distributed==1.19.1
+dnspython==2.0.0
+docopt==0.6.2
+docutils==0.14
+docx==0.2.4
+durable-rules==2.0.28
+email-validator==1.1.1
+emoji==0.5.4
+en-core-web-lg==2.0.0
+en-core-web-sm==2.0.0
+entrypoints==0.2.3
+et-xmlfile==1.0.1
+fastcache==1.0.2
+filelock==2.0.12
+flasgger==0.9.1
+Flask==1.0.2
+Flask-Caching==1.9.0
+Flask-Cors==3.0.6
+Flask-HTTPAuth==4.1.0
+Flask-Login==0.5.0
+Flask-Mail==0.9.1
+Flask-SocketIO==4.3.1
+Flask-SQLAlchemy==2.4.4
+Flask-User==1.0.2.2
+Flask-WTF==0.14.3
+funcy==1.14
+fuzzywuzzy==0.16.0
+gensim==3.8.3
+gevent==1.2.2
+gevent-websocket==0.10.1
+glob2==0.5
+glob3==0.0.1
+gmpy2==2.0.8
+graphviz==0.15
+greenlet==0.4.12
+h5py==2.8.0
+heapdict==1.0.0
+html2text==2018.1.9
+html5lib==0.999999999
+hyperlink==18.0.0
+idna==2.8
+imageio==2.2.0
+imagesize==0.7.1
+importlib-metadata==2.0.0
+incremental==17.5.0
+inflection==0.3.1
+ipdb==0.13.4
+ipykernel==4.6.1
+ipython==6.1.0
+ipython-genutils==0.2.0
+ipywidgets==7.0.0
+isort==4.2.15
+itsdangerous==0.24
+jdcal==1.3
+jedi==0.10.2
+Jinja2==2.10
+jmespath==0.9.4
+jsondiff==1.2.0
+jsonschema==2.6.0
+jupyter-client==5.1.0
+jupyter-console==5.2.0
+jupyter-core==4.3.0
+jupyterlab==0.27.0
+jupyterlab-launcher==0.4.0
+Keras==2.2.0
+Keras-Applications==1.0.2
+Keras-Preprocessing==1.0.1
+lazy-object-proxy==1.3.1
+llvmlite==0.20.0
+locket==0.2.0
+lxml==4.1.0
+Markdown==3.3.3
+MarkupSafe==1.0
+matplotlib==2.1.0
+mccabe==0.6.1
+mistune==0.7.4
+mmh3==2.5.1
+more-itertools==5.0.0
+mpmath==0.19
+msgpack==0.5.6
+msgpack-numpy==0.4.3.2
+msgpack-python==0.4.8
+multipledispatch==0.4.9
+murmurhash==1.0.2
+mysql-connector==2.1.6
+mysql-connector-python==8.0.15
+navigator-updater==0.1.0
+nbconvert==5.3.1
+nbformat==4.4.0
+ndg-httpsclient==0.5.1
+networkx==2.0
+nltk==3.2.5
+nose==1.3.7
+notebook==5.0.0
+nrepl-python-client==0.0.3
+numba==0.35.0+10.g143f70e90
+numexpr==2.6.2
+numpy==1.14.5
+numpydoc==0.7.0
+odo==0.5.1
+olefile==0.44
+openpyxl==2.4.8
+ordered-set==4.0.2
+ortools==7.1.6720
+overrides==1.9
+packaging==16.8
+paho-mqtt==1.5.0
+pampy==0.3.0
+pandas==0.23.1
+pandas-datareader==0.8.1
+pandocfilters==1.4.2
+paramiko==2.7.1
+parsel==1.5.1
+partd==0.3.8
+passlib==1.7.2
+path.py==10.3.1
+pathlib==1.0.1
+pathlib2==2.3.0
+patsy==0.4.1
+peewee==3.9.5
+pep8==1.7.0
+pervane==0.0.66
+pexpect==4.2.1
+pickleshare==0.7.4
+Pillow==4.2.1
+pkginfo==1.4.1
+plac==0.9.6
+plotly==4.14.1
+ply==3.10
+preshed==2.0.1
+prompt-toolkit==1.0.15
+protobuf==3.7.1
+psutil==5.4.0
+ptyprocess==0.5.2
+py==1.4.34
+pyasn1==0.4.5
+pyasn1-modules==0.2.4
+pycodestyle==2.3.1
+pycosat==0.6.2
+pycparser==2.18
+pycrypto==2.6.1
+pycurl==7.43.0
+pydbus==0.6.0
+PyDispatcher==2.0.5
+pyflakes==1.6.0
+Pygments==2.2.0
+PyHamcrest==1.9.0
+pylint==1.7.4
+pymorphy2==0.8
+pymorphy2-dicts==2.4.393442.3710985
+pymorphy2-dicts-ru==2.4.404381.4453942
+PyNaCl==1.3.0
+pync==2.0.3
+pyodbc==4.0.17
+pyOpenSSL==18.0.0
+pypandoc==1.4
+pyparsing==2.2.0
+pysftp==0.2.9
+PySocks==1.6.7
+pyTelegramBotAPI==3.5.2
+pytest==3.2.1
+python-dateutil==2.6.1
+python-engineio==3.13.2
+python-socketio==4.6.0
+pytz==2017.2
+PyWavelets==0.5.2
+PyYAML==3.12
+pyzmq==16.0.2
+QtAwesome==0.4.4
+qtconsole==4.3.1
+QtPy==1.3.1
+Quandl==3.4.8
+queuelib==1.5.0
+rake-nltk==1.0.4
+readline==6.2.4.1
+regex==2018.1.10
+requests==2.22.0
+requests-cache==0.5.2
+retrying==1.3.3
+rope==0.10.5
+ruamel-yaml==0.11.14
+rusenttokenize==0.0.4
+s3transfer==0.2.1
+schedule==0.6.0
+scikit-image==0.13.0
+scikit-learn==0.19.1
+scipy==1.1.0
+Scrapy==1.6.0
+seaborn==0.8
+service-identity==18.1.0
+simplegeneric==0.8.1
+singledispatch==3.4.0.3
+six==1.12.0
+smart-open==3.0.0
+snowballstemmer==1.2.1
+sortedcollections==0.5.3
+sortedcontainers==1.5.7
+spacy==2.0.18
+Sphinx==1.6.3
+sphinxcontrib-websupport==1.0.1
+spyder==3.2.4
+SQLAlchemy==1.1.13
+statsmodels==0.8.0
+striprtf==0.0.11
+summa==1.2.0
+sympy==1.1.1
+tables==3.4.2
+tblib==1.3.2
+terminado==0.6
+testpath==0.3.1
+textdistance==4.2.0
+thinc==6.12.1
+toolz==0.8.2
+tornado==4.5.2
+tqdm==4.23.4
+traitlets==4.3.2
+Twisted==18.9.0
+typing==3.6.2
+ujson==1.35
+unicodecsv==0.14.1
+urllib3==1.25.7
+w3lib==1.20.0
+wcwidth==0.1.7
+webencodings==0.5.1
+Werkzeug==0.14.1
+widgetsnbextension==3.0.2
+wrapt==1.10.11
+WTForms==2.3.1
+xlrd==1.1.0
+XlsxWriter==1.0.2
+xlwt==1.3.0
+yattag==1.11.1
+youtube-dl==2019.11.5
+zict==0.1.3
+zipp==3.4.0
+zope.interface==4.6.0
diff --git a/sched.py b/sched.py
new file mode 100644
index 0000000..10bbf19
--- /dev/null
+++ b/sched.py
@@ -0,0 +1,94 @@
+import requests, re, csv, json, funcy, sys
+
+
+def dates(s):
+    #print(s)
+    m = re.match(r'(\d\d\d\d)\-(\d\d)\-(\d\d)',s)
+    if m:
+        s = m.group(2) + "/" + m.group(3)
+        #print(s)
+    return s 
+# "Course Code","Start Date","End Date",Term,Delivery,CRN,Status,"Course Name","Course Description","Units/Credit hours","Instructor Last Name","Instructor First Name",Campus/College,"Meeting Days and Times","Pass/No Pass available?","Class Capacity","Available Seats","Waitlist Capacity","Current Waitlist Length","Meeting Locations","Course Notes",ZTC
+# ACCT103,2021-06-14,2021-07-23,"Summer 2021",Online,80386,Active,"General Office Accounting","This course is designed to prepare students for entry-level office accounting positions. Emphasis is on practical accounting applications. This course has the option of a letter grade or pass/no pass. ADVISORY: Eligible for Mathematics 430.","  3.00","Valenzuela Roque",Karla,"Gavilan College"," ",T,"    30","    18","    20","     0",,,
+
+
+def parse_www_csv_sched():
+    old_keys = [ "CRN","Course Code","Units/Credit hours","Course Name","Meeting Days and Times","Class Capacity","Available Seats","Waitlist Capacity","Current Waitlist Length","Instructor Last Name","Start Date","Meeting Locations","ZTC","Delivery","Campus/College","Status","Course Description","Pass/No Pass available?","Course Notes" ]
+
+    # "Instructor First Name","End Date","Term",
+
+    new_keys = [ "crn", "code","cred", "name", "days",  "cap",  "rem", "wl_cap", "wl_act", "teacher", "date", "loc", "ztc", "type", "site","status","desc","pnp","note" ]
+
+    # "time","act","wl_rem",  "partofday",
+
+        
+    url = "https://gavilan.edu/_files/php/current_schedule.csv"
+
+    sched_txt = requests.get(url).text.splitlines()
+    sched = {"Fall 2021":[], "Spring 2022":[], "Winter 2022":[], "Summer 2021":[]}
+    shortsems = {"Fall 2021":"fa21", "Spring 2022":"sp22", "Winter 2022":"wi22", "Summer 2021":"su21","Summer 2022":"su22","Fall 2022":"fa22"}
+    for row in csv.DictReader(sched_txt):
+        d = dict(row)
+        for (old_key,new_key) in zip(old_keys,new_keys):
+            d[new_key] = d.pop(old_key).strip()
+        d['teacher'] = d.pop('Instructor First Name').strip() + " " + d['teacher']
+        d['date'] = dates(d['date']) + '-' + dates(d.pop('End Date').strip())
+        d['term'] = shortsems[d.pop('Term')]
+        if d['cred'] == ".00": 
+            d['cred'] = "0"
+        if d['type'] == "Online": 
+            d["loc"] = "ONLINE"
+            d["site"] = "Online"
+            d["type"] = "online"
+        #d.pop('Instructor First Name').strip() + " " + d['teacher']
+        #d["code"] = d.pop("Course Code")
+        #d["crn"] = d.pop("CRN")
+        sched[row['Term']].append(d)   #print(row)
+
+    print( json.dumps(sched,indent=2))
+    for k,v in sched.items():
+        print("%s: %i" % (k,len(v)))
+        
+    for v in sched["Fall 2021"]:
+        print("%s\t %s\t %s\t %s"    % (  v['code'], v['days'], v['type'], v['loc']  ))    
+        #print("%s\t %s\t %s\t %s"    % (  v['Course Code'], v['Meeting Days and Times'], v['Delivery'], v['Meeting Locations']  ))
+
+def parse_json_test_sched():
+    j2 = open('cache/classes_json.json','r').readlines()
+
+    for L in j2:
+        o3  =  json.loads(L)
+        print(json.dumps(o3,indent=2))
+
+
+
+
+
+
+if __name__ == "__main__":
+
+    print ('')
+    options = { 
+            1: ['fetch and parse the csv on www.', parse_www_csv_sched], 
+            2: ['parse the test json file.', parse_json_test_sched ],
+            }
+
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
+
+
+
+
diff --git a/server.py b/server.py
new file mode 100644
index 0000000..aaac0f8
--- /dev/null
+++ b/server.py
@@ -0,0 +1,679 @@
+import json, codecs, re, markdown, os, pypandoc, striprtf, sqlite3, random, urllib
+import subprocess, html
+from striprtf.striprtf import rtf_to_text
+from flask import render_template, Response
+from flask import send_from_directory
+import hashlib, funcy, platform, requests
+from datetime import datetime
+
+from orgpython import to_html
+
+from localcache import sqlite_file, db  # personnel_meta #  personnel_fetch
+from localcache import user_enrolled_in
+from localcache import arrange_data_for_web, depts_with_classcounts, dept_with_studentviews, course_quick_stats
+
+
+from yattag import Doc
+
+
+LECPATH = "/media/hd2/peter_home_offload/lecture/"
+    host = 'http://192.168.1.6:5000'
+
+
+import paho.mqtt.client as mqtt
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      mqtt
+######
+
+
+
+client = 0
+
+mqtt_offline = 1
+
+mqtt_time = 0.1
+
+
+def mqtt_loop():
+    while 1:
+        if client and not mqtt_offline:
+            client.loop(mqtt_time)
+
+
+# called when MQTT server connects
+def on_connect(client, userdata, flags, rc):
+    print("Connected with result code "+str(rc))
+    client.subscribe("local/#")
+
+# The callback for when a PUBLISH message is received from the server.
+def on_message(client, userdata, msg):
+    now = datetime.now().strftime('%Y %m %d  %H %M')
+    print("    %s  mqtt msg: %s    data: %s"  % (now, msg.topic, msg.payload.decode()))
+
+
+
+while(mqtt_offline):
+    try:
+        client = mqtt.Client()
+        client.on_connect = on_connect
+        client.on_message = on_message
+
+        client.connect("192.168.1.6", 1883, 60)
+
+        mqtt_offline = 0
+
+    except OSError as oe:
+        print('no internet? try again in 5 seconds.')
+        time.sleep(5)
+
+
+
+def displaypi_on():
+    global client
+    msg = 'local/frame_on'
+    client.publish(msg, 'webserver')
+    print("sent %s" % msg)
+
+def displaypi_off():
+    global client
+    msg = 'local/frame_off'
+    client.publish(msg, 'webserver')
+    print("sent %s" % msg)
+
+def desklight():
+    global client
+    
+    msg = 'local/peter/desklamps'
+    client.publish(msg, 'webserver')
+    print("sent %s" % msg)
+
+def clearscreens():
+    global client
+    
+    msg = 'local/clearscreens'
+    client.publish(msg, 'webserver')
+    print("sent %s" % msg)
+
+def screenoff():
+    global client
+    
+    msg = 'local/peter/monitors'
+    client.publish(msg, 'webserver')
+    print("sent %s" % msg)
+
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      writing & knowledgebase
+######
+
+
+news_path = '/media/hd2/peter_home/Documents/scripts/browser/'
+   
+if platform.system() == 'Windows':
+    writing_path = 'c:/users/peter/Nextcloud/Documents/writing/'
+else:
+    writing_path = '/media/hd2/peter_home/Documents/writing/'
+    img_path = '/media/hd2/peter_home/Documents/writing_img/'
+
+if platform.system() == 'Windows':
+    pics_path = 'c:/users/peter/Nextcloud/misc/'
+else:
+    pics_path = '/media/hd2/peter_home/misc/'
+
+br = "<br />"
+nl = "\n"
+style = """<style> body { line-height: 2.3em; margin: 4em; } 
+</style><link rel="stylesheet" href="/data/gui/public/simplemde.min.css">
+<script src="/data/gui/public/simplemde.min.js"></script>"""
+
+
+## I implement the backend of a web based GUI for the canvas functions
+## and dashboard stuff.
+
+
+## Stories / Next Steps
+
+## 1. browse my writings. Follow links in markdown format to other files
+##    in this folder. html, md, rtf are handled so far. 
+
+## 2a. Sort by date, title, topic/tag
+
+## 2b. Use w/ larger topic modeling project to suggest relations
+
+## 2. Run through everything in the folder, and index the 'backlinks'.
+##    Append those to each file.
+
+## 3 (interrupted)  Use vue and implement editing. Q: convert back to
+##   original format? Or maintain new one. A: Convert back. 
+
+## 3. Do similar for tags. 
+
+## 4. Do similar but automatically, using nlp and keywords. 
+
+
+
+def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
+def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
+def a(t,h): return '<a href="%s">%s</a>' % (h,t)
+
+def homepage():
+    #output = pypandoc.convert_file(writing_path+fname, 'html', format="rst")
+    return tag('h1','This is my server.') + "<br />" + \
+            a('Toggle light','/light') + br + br + \
+            a('Clear screens','/clearscreens') + br + br + \
+            a('Toggle monitors','/screensoff') + br + br + \
+            a('Pi frame on','/displaypi/on') + br + br + \
+            a('Pi frame off','/displaypi/off') + br + br + \
+            a('Knowledge Base','/x/writing/index') + br + \
+            a('Graph of users','/x/user/1') + "<br />" + \
+            a('Courses in a dept','/x/dept/csis') + "<br />" + \
+            a('People in a course','/x/roster/10633') + br + br + \
+            a('Summarize courses a user has been seen in','/x/user_course_history_summary/9964') + br + br + \
+            a('Peters Lecture Series','/lectures') + br + br + \
+            a('Reload server','/rl') + "<br />" + \
+            a('want to shut down?','/sd') + "<br />" + \
+            a('', '') + br
+
+def orgline(L):
+    L.strip()
+    if re.search("^\s*$", L): return ""
+
+    a = re.search( '^\*\s(.*)$', L)
+    if a: return "<h2>%s</h2>\n" % a.group(1)
+
+    b = re.search( 'TODO\s\[\#A\](.*)$', L)
+    if b: return "<b><i>Todo - Priority 1</i>: %s</b>" % b.group(1)  + br + nl
+
+    d = re.search( '^\*\*\*\s(.*)$', L)
+    if d: return d.group(1)
+    d = re.search( '^\*\*\s(.*)$', L)
+    if d: L = d.group(1)
+
+    return L + br + nl
+
+def editor(src):
+    return br + br + br + """<textarea name='content' id='editor'>%s</textarea><script>
+var simplemde = new SimpleMDE({ element: document.getElementById("editor") });
+</script>""" % src
+
+def in_form(txt,path):
+    return '<form method="post" action="/save"><input type="hidden" name="what" value="writing">' + \
+            '<input type="hidden" name="path" value="' + path + '" />' + \
+            txt + \
+            '<input type="submit" value="Save" name="Save" /></form>'
+
+def mytime(fname):
+    return os.path.getmtime( os.path.join(writing_path, fname) )
+
+def index():
+    #f = [ os.path.join(writing_path, x) for x in os.listdir(writing_path) ]
+    f = os.listdir(writing_path)
+    f.sort(key=mytime)
+    f.reverse()
+    return "<br /><br />\n".join(  ["<a href='%s'>%s</a> (%s)" % (x,x,datetime.fromtimestamp(mytime(x)).strftime('%Y-%m-%d %H')) for x in f ] )
+
+def writing(fname):
+    
+    if fname == 'index': return index()
+    inp = codecs.open(writing_path + fname, 'r', 'utf-8')
+    ext = fname.split('.')[-1]
+    if ext == "py" or ext == "php":
+        src = inp.read()
+        return "<pre>" + html.escape(src) + "</pre>"
+    if ext == "html":
+        src = inp.read()
+        return src
+    if ext == "md":
+        src = inp.read()
+        return style + markdown.markdown(src) + in_form(editor(src),fname)
+    if ext == "org":
+        src = inp.read()
+        return to_html(src, toc=True, offset=0, highlight=True)
+    if ext == "rtf":
+        text = "<br />\n".join( rtf_to_text(inp.read()).split('\n') )
+        return style + text
+    if ext == "docx":
+        hash = hashlib.sha1("my message".encode("UTF-8")).hexdigest()
+        hash = hash[:10]
+        #output = pypandoc.convert_file('C:/Users/peter/Nextcloud/Documents/writing/' + fname, 'html',
+        output = pypandoc.convert_file(writing_path + fname, 'html',
+                extra_args=['--extract-media=%s' % hash ])     #   file:///c:/Users/peter/Nextcloud/Documents/writing
+        return style + output
+    return style + markdown.markdown( "".join( [ orgline(x) for x in inp.readlines() ] ) )
+
+
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      kiosk display
+######
+
+def dashboard():
+    return open('static/slides.html','r').read()  # tag('h1','Dashboard') + br + a('home', '/')
+
+def dash():
+    return open('static/dashboard.html','r').read()  # tag('h1','Dashboard') + br + a('home', '/')
+
+def mycalendar():
+    ics = 'https://calendar.google.com/calendar/u/0?cid=cGV0ZXIuaG93ZWxsQGdtYWlsLmNvbQ'
+    return dash()
+    
+def most_recent_file_of( target, folder ):
+
+    def finder(st):
+        return re.search(target,st)
+
+    all = os.listdir(folder)
+    all.sort(key=lambda x: os.stat(os.path.join(folder,x)).st_mtime)
+    all.reverse()
+    all = list(funcy.filter( finder, all ))
+
+    print("file list is: " + str(all))
+    if not all:
+        return ''
+    return all[0]
+
+def news():
+    folder = most_recent_file_of( r'\d\d\d\d\d\d\d\d', news_path )
+    pics = os.listdir( news_path + folder )
+    return '/static/news/' + folder + '/' + random.choice(pics)
+
+def randPic():
+    now = datetime.now()
+    
+    if now.minute < 15:
+        return news()
+    return '/static/images/' + random.choice(os.listdir('static/images'))
+
+
+
+
+
+
+def do_img_crop(im):
+    result = subprocess.run(['ls', '-l'], stdout=subprocess.PIPE)
+    rr = result.stdout.decode('utf-8')
+
+
+
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      db info helpers
+######
+
+def sample():
+    return "<h1>I am a sample</h1>"
+
+def sample2(a=""):
+    return "I'm a placeholder"
+
+# Filter a stream of loglines for those that match a course's url / id
+def has_course(stream,courseid):
+    regex = '/courses/%i/' % int(courseid)
+    while True:
+        L = stream.readline()
+        if re.search(regex, L): yield L
+
+def js(s):
+    return json.dumps(s, indent=2)
+
+def sem_from_array_crn(crn):
+    if not crn[2]: return ""
+    if crn[2] == "": return ""
+    return crn[2][:6]
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      db ilearn course / user / hits
+######
+
+def user_courses(uid):
+    return js(user_enrolled_in(uid))
+
+def user_course_history_summary(usr_id):
+    q = """SELECT r.timeblock, r.viewcount, c.sis, c.code, c.canvasid FROM requests_sum1 AS r 
+JOIN users AS u ON r.userid=u.id 
+JOIN courses AS c ON c.id=r.courseid 
+WHERE u.canvasid=%s
+GROUP BY r.courseid ORDER BY r.viewcount DESC;""" % str(usr_id)
+    (conn,cur) = db()
+    cur.execute(q)
+    r = cur.fetchall()
+    return js(r)
+    groups = funcy.group_by(sem_from_array_crn, r)
+    g = {}
+    for K in groups.keys(): g[K] =    [ x[3] for x in groups[K] ] 
+    return js( g )
+
+def roster(crn):
+    q = """SELECT u.name, u.sortablename, u.canvasid as user_id, c.canvasid as course_id, e.workflow, e."type" FROM enrollment AS e 
+JOIN users AS u ON e.user_id=u.id 
+JOIN courses AS c ON c.id=e.course_id 
+WHERE c.canvasid="%s" ;""" % str(crn)
+    (conn,cur) = db()
+    cur.execute(q)
+    return js(cur.fetchall())
+
+
+def user_course_hits(usr,courseid):
+    return list(has_course( codecs.open('cache/users/logs/%s.csv' % usr, 'r', 'utf-8'), courseid))
+    #return "\n".join( [x for x in next(gen)] )
+
+def profiles(id=1,b=2,c=3):
+    import os
+    pics = os.listdir('cache/picsCanvas')
+    return ''.join([ "<img height='45' width='45' hspace='5' vspace='5' src='/cache/picsCanvas/%s' />" % s for s in pics ])
+
+
+# Departments, classes in each, and students (with hits) in each of those.
+def enrollment(a):
+    return js(depts_with_classcounts())
+
+# All the classes in this dept, w/ all the students in each, with count of their views.
+def dept(d=''):
+    if not d: return js(dept_with_studentviews())
+    return js(dept_with_studentviews(d))
+
+
+def user(canvas_id=None):
+    info = json.loads(  codecs.open( 'cache/users/%s.txt' % canvas_id, 'r', 'utf-8').read() )
+    return render_template('hello.html', id=canvas_id, name=info['name'])
+
+
+
+
+
+
+
+
+
+
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      podcast feed
+######
+
+
+def lectures():
+    fi = os.listdir(LECPATH)
+    doc, tag, text = Doc().tagtext()
+    doc.asis('<?xml version="1.0" encoding="UTF-8"?>')
+    doc.asis('<rss xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" version="2.0">')
+    with tag('channel'):
+        with tag('title'): text("Peter's Lecture Series")
+        with tag('description'): text("Since 2019")
+        with tag('link'): text(host)
+        for f in fi:
+            if f.endswith('.mp3'):
+                #print(f)
+                with tag('item'):
+                    name = f.split('.')[0]
+                    ff = re.sub('\s','%20',f)
+                    with tag('title'): text(name)
+                    with tag('guid'): text(f)
+                    b = os.path.getsize(LECPATH+f)
+                    doc.stag('enclosure', url=host+'/podcast/media/'+urllib.parse.quote(ff), type='audio/mpeg',length=b)
+    doc.asis('</rss>')
+    #doc.asis('</xml>')
+    return doc.getvalue()
+
+def web_lectures():
+    fi = os.listdir(LECPATH)
+    output = "<h1>Lectures</h1>\n"
+    for f in fi:
+        if f.endswith('.mp3'):
+            name = f.split('.')[0]
+            ff = urllib.parse.quote(f)
+            #ff = re.sub('\s','%20',f)
+            output += '<a href="%s">%s</a><br />\n' % ( host + '/podcast/media/' + ff, name)
+    return output
+
+
+
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      editing personnel app
+######
+
+
+# personnel_fetch, personnel_meta
+
+# todo: update: dept, title, any of the other fields. 
+#       insert: new dept, new title, 
+
+# update a value: dept id of a personnel id
+def update_pers_title(pid, tid):
+    q = "UPDATE personnel SET `title`='%s' WHERE `id`='%s'" % (str(tid), str(pid))
+    (conn,cur) = db()
+    result = cur.execute(q)
+    conn.commit()
+    return js(  {'result': 'success'} )
+
+# update a value: dept id of a personnel id
+def update_pers_dept(pid, did):
+    q = "UPDATE personnel SET `dept1`='%s' WHERE `id`='%s'" % (str(did), str(pid))
+    (conn,cur) = db()
+    result = cur.execute(q)
+    conn.commit()
+    return js(  {'result': 'success'} )
+
+
+
+def user_edit(canvas_id='2'):
+    info = json.loads(  codecs.open( 'cache/users/%s.txt' % str(canvas_id), 'r', 'utf-8').read() )
+    return render_template('personnel.html', id=canvas_id, name=info['name'])
+    
+def staff_dir(search=''):
+    return render_template('dir.html')
+
+    
+    
+    
+    
+######
+######      handling images
+######
+
+
+def find_goo(n):
+    g = re.search('00(\d\d\d\d\d\d)', n)
+    
+    if g:
+        return g.groups()[0]
+    return ''
+
+
+def byname(x): 
+    if 'conf_name' in x: 
+        return x['conf_name']
+    if 'first_name' in x and 'last_name' in x:
+        return x['first_name'] + " " + x['last_name']
+    return ''
+
+def fn_to_struct( n, staff ):
+    g = find_goo(n)
+    if g: 
+        #print(g)
+        for s in staff:
+            cg = s['conf_goo']
+            if cg == g: 
+                #print("%s - %s - %s" % (n, g, cg) )
+                return s
+        return { "conf_goo":g, "conf_name":"unknown - " + n }
+    return 0
+            
+def image_edit(filename=''):
+    
+    url = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=list/staffsemester"
+    staff = json.loads( requests.get(url).text )
+    
+    badges = 0
+    web = 1
+    
+    if web:
+        files = sorted(os.listdir('cache/picsStaffdir') )
+        done_files = [ x[:-4] for x in sorted(os.listdir('cache/picsStaffdir/cropped') ) ]
+
+    
+    if badges:
+        files = sorted(os.listdir('cache/picsId/originals_20211022') )
+        done_files = [ x[:6] for x in sorted(os.listdir('cache/picsId/2021crop') ) ]
+    
+    files_match = []
+    files_no_match = []
+    raw_filenames = files
+    
+    for f in files:
+        sa = fn_to_struct(f,staff)
+        if sa:
+            ss = sa.copy()
+        else:
+            ss = sa
+        if ss: 
+            ss['filename'] = f
+            files_match.append(ss)
+        else: files_no_match.append(f)
+    
+    
+    fm = json.dumps( sorted(files_match,key=byname) )
+    fnm = json.dumps(files_no_match)
+    sm = json.dumps(staff)
+        
+    
+    return render_template('images.html', staff=sm, matches=fm, nomatches=fnm, checked=done_files)
+
+
+
+def image_crop(filename,x,y,w,h,newname=''):
+    from PIL import Image
+    import piexif
+    
+    badges = 0
+    web = 1
+
+    if not newname: newname = filename
+    
+    if web:
+        im = Image.open('cache/picsStaffdir/%s' % filename)
+        savepath = 'cache/picsStaffdir/cropped/%s.jpg' % newname
+    
+    if badges:
+        im = Image.open('cache/picsId/originals_20211022/%s' % filename)
+        savepath = 'cache/picsId/2021crop/%s.jpg' % newname
+
+    out = { 'im': str(im) }
+    
+    x = int(x)
+    y = int(y)
+    w = int(w)
+    h = int(h)
+    
+    
+    if "exif" in im.info:
+        exif_dict = piexif.load(im.info['exif'])
+        #out['exif'] = exif_dict
+        #print(exif_dict)
+
+        if piexif.ImageIFD.Orientation in exif_dict['0th']:
+            #exif_dict['0th'][piexif.ImageIFD.Orientation] = 3
+            print(piexif.ImageIFD.Orientation)
+            print(exif_dict['0th'])
+            out['rotation'] = 'messed up'
+            
+            if exif_dict['0th'][piexif.ImageIFD.Orientation] == 6:
+                im = im.rotate(270, expand=True)
+                #im.save('cache/picsId/originals_20211022/crotated_%s' % filename, quality=95)
+
+
+
+    
+    im_crop = im.crop((x,y,x+w,y+h))
+    img_resize = im_crop.resize((250, 333))
+    img_resize.save(savepath, quality=95)
+    return json.dumps( out )
+
+
+
+    #if filename=='list':
+    #    #return '<br />\n'.join([ "<a href='/data/picsId/originals_20211022/%s'>%s</a>" % ( x,x ) for x in 
+    #    return '<br />\n'.join([ "<a href='/image/%s'>%s</a>" % ( x,x ) for x in sorted(os.listdir('cache/picsId/originals_20211022')) ])
+    
+
+
+#################################################################################################################
+#################################################################################################################
+######
+######      server infrastructure
+######
+
+
+
+def server_save(key,value):
+    codecs.open(datafile2,'a').write( "%s=%s\n" % (str(key),str(value)))
+
+
+def server_dispatch_json(function_name,arg='', arg2=''):
+    print("Looking for function: %s. arg:%s. arg2:%s." % (function_name, arg, arg2))
+    try:
+        result = "" + globals()[function_name](arg, arg2)
+        print("doing 2 args")
+        return result
+    except Exception as e:
+        print("Error with that: %s" % str(e))
+        try:
+            result = "" + globals()[function_name](arg)     # 
+            print("doing 1 arg")
+            return result
+        except Exception as f:
+            print("Error with that: %s" % str(f))
+            try:
+                result = globals()[function_name]()
+                print("doing 0 arg")
+                return result
+            except Exception as gg:
+                print("Error with that: %s" % str(gg))
+                return json.dumps({'result':'failed: exception', 'e1':str(e), 'e2':str(f), 'e3':str(gg)}, indent=2)
+        
+
+def server_dispatch(function_name,arg='', arg2=''):
+    print("Looking for function: %s. arg:%s. arg2:%s." % (function_name, arg, arg2))
+    try:
+        result = "" + globals()[function_name](arg, arg2)
+        print("doing 2 args")
+        return result
+    except Exception as e:
+        print("Error with that: %s" % str(e))
+        try:
+            result = "" + globals()[function_name](arg)     # 
+            print("doing 1 arg")
+            return result
+        except Exception as f:
+            print("Error with that: %s" % str(f))
+            try:
+                result = globals()[function_name]()
+                print("doing 0 arg")
+                return result
+            except Exception as gg:
+                print("Error with that: %s" % str(gg))
+                return json.dumps({'result':'failed: exception', 'e1':str(e), 'e2':str(f), 'e3':str(gg)}, indent=2)
+        
diff --git a/stats.py b/stats.py
new file mode 100644
index 0000000..5e73a3d
--- /dev/null
+++ b/stats.py
@@ -0,0 +1,223 @@
+
+
+    
+def grades_rundown():
+    global results, users_by_id
+    load_users()
+    results = []
+    all_sem_courses = []
+    ids_out = open('all_teachers_by_goo','w')
+    all_ids = {}
+    # for the current or given semester's shells (really, only active ones)
+    with open('grades_out.csv','wb') as f:
+        w = csv.DictWriter(f, 'id,name,teacher,mean,median,count,count_gt70,grades,avg_activity_time'.split(','))
+        w.writeheader()
+        #for c in all_sem_courses: 
+        courses = getCoursesInTerm(term=23,show=0,active=1)
+        for C in courses: 
+            activity_time_total = 0.0
+            course_info = {'id':str(C['id']),'name':C['name'],'grades':[], 'teacher':[] }
+            #print str(C['id']) + "\t " + C['name']
+            emts = course_enrollment(C['id'])
+            for k,E in emts.items():
+                #print E
+                if E['type'] == 'TeacherEnrollment':
+                    course_info['teacher'].append(users_by_id[E['user_id']]['name'])
+                    all_ids[E['sis_user_id']] = 1
+            """    if 'grades' in E and E['grades']['current_score']:
+                    #print str(E['grades']['final_score']) + ", ",
+                    #print str(E['grades']['current_score']) + ", ",
+                    course_info['grades'].append(E['grades']['current_score'])
+                    activity_time_total += E['total_activity_time']
+            if course_info['grades']:
+                s = pd.Series(course_info['grades'])
+                course_info['mean'] = s.mean()
+                course_info['median'] = s.median()
+                course_info['count'] = len(s.values)
+                course_info['count_gt70'] = (s > 70.0).count()
+                course_info['avg_activity_time'] = activity_time_total / len(s.values)
+            else:
+                course_info['mean'] = 0
+                course_info['median'] = 0
+                course_info['count'] = 0
+                course_info['count_gt70'] = 0
+                course_info['avg_activity_time'] = 0"""
+                
+            #print course_info
+            all_sem_courses.append(course_info)
+            w.writerow(course_info)
+            f.flush()
+        
+    # get a grade (final? current?) for each student
+    for k,v in all_ids.items():
+        if k: ids_out.write(k + ', ')
+    
+    # sanity check to make sure grading is actually happening in the shell
+    
+    # report an average, median, and buckets
+    
+
+
+
+def class_logs():
+    global results
+    # 1. Search the current semester and the misc semesters for a list of courses
+    #    that we want to check for users/activity.
+    #target = url + '/api/v1/accounts/1/terms'       # list the terms
+    target = url + '/api/v1/accounts/1/courses?published=true&enrollment_term_id=14'
+    print "Getting term classes."
+    while target:
+        target = fetch(target)
+
+    print "\n\n\n"
+    
+    term_results = results
+    full_results = []
+    for x in term_results:
+        results = []
+        # now see who's logged in recently:
+        target = url + '/api/v1/courses/' + str(x['id']) + '/recent_students'
+        print "Getting class id: ", str(x['id'])
+        fetch(target)
+        if len(results):
+            #print results
+            LL = [ how_long_ago(z['last_login']) for z in results ]
+            avg = 9999
+            if len(LL): avg = sum(LL) / len(LL)
+            d = { 'id':x['id'], 'avg':avg, 'name':x['name'] }
+            full_results.append(d)
+    sorted_results = sorted(full_results, key=lambda k: k['avg'])
+    for x in sorted_results:
+        print x['id'], "\t", str(x['avg']), "\t", x['name']
+
+        
+        
+        
+        
+
+def user_logs():
+    global url, users_by_id, results
+    target_user = "6357"
+    load_users()
+    results = []
+    target = url + '/api/v1/users/' + target_user + '/page_views?per_page=200'
+    while target:
+        print target
+        target = fetch(target)
+    # have all student's hits. Filter to only this class
+    #results = filter(match59,results)
+    times = []
+    print users_by_id[ int(target_user) ]
+    f.write(str(users_by_id[ int(target_user) ]) + "\n")
+    f.write( "link,updated_at,remote_ip,url,context_type,user_agent,action\n")
+    for hit in results:
+        L = [hit['links']['user'],hit['updated_at'],hit['remote_ip'],hit['url'],hit['context_type'],hit['user_agent'],hit['action']]
+        L = map(str,L)
+        f.write( ",".join(L) + "\n" ) 
+
+        
+        
+
+def recent_logins():
+    global results, url, results_dict
+    p = { 'start_time':'2017-08-31T00:00:00Z', 'end_time':'2017-08-31T00:05:00Z'}
+    target = url + "/api/v1/audit/authentication/accounts/1"
+    results_dict = {}
+    resp = fetch_dict(target,p)
+    print resp
+    print results_dict
+
+
+
+def userHitsThisSemester(uid=2):
+    begin = "20170820T0000"
+    t = url + "/api/v1/users/" + str(uid) + "/page_views?start_time=" + str(begin)
+    while(t): t = fetch(t)
+    print json.dumps(results, indent=4, sort_keys=True)
+    
+
+
+
+def getCurrentActivity():  # a dict
+    # CURRENT ACTIVITY
+    #r = requests.get(url + '/api/v1/accounts/1/analytics/current/activity', headers = header )
+    #t = url + '/api/v1/accounts/1/users?per_page=500'
+    #  analytics/terms/:term_id/activity
+    #t = url + '/api/v1/accounts/1/analytics/current/statistics'
+    global results_dict
+    t = url +  '/api/v1/accounts/1/analytics/terms/11/activity'
+    while(t): t = fetch_dict(t)
+    sp17 = results_dict['by_date']
+    results_dict = {}
+    
+    t = url +  '/api/v1/accounts/1/analytics/terms/14/activity'
+    while(t): t = fetch_dict(t)
+    su17 = results_dict['by_date']
+    results_dict = {}
+    
+    t = url +  '/api/v1/accounts/1/analytics/terms/15/activity'
+    while(t): t = fetch_dict(t)
+    su17b = results_dict['by_date']
+    results_dict = {}
+    
+    t = url +  '/api/v1/accounts/1/analytics/terms/18/activity'
+    while(t): t = fetch_dict(t)
+    fa17 = results_dict['by_date']
+    results_dict = {}
+    
+    t = url +  '/api/v1/accounts/1/analytics/terms/21/activity'
+    while(t): t = fetch_dict(t)
+    sp18 = results_dict['by_date']
+    results_dict = {}
+    
+    t = url +  '/api/v1/accounts/1/analytics/terms/7/activity'
+    while(t): t = fetch_dict(t)
+    cmte = results_dict['by_date']
+    results_dict = {}
+    
+    t = url +  '/api/v1/accounts/1/analytics/terms/6/activity'
+    while(t): t = fetch_dict(t)
+    dev = results_dict['by_date']
+    results_dict = {}
+    
+    master_list_by_date = {}
+    for sem in [sp17,su17,su17b,fa17,sp18,cmte,dev]:
+        #print sem
+        for record in sem:
+            print record
+            date = record['date']
+            if date in master_list_by_date:
+                master_list_by_date[date]['participations'] += record['participations']
+                master_list_by_date[date]['views'] += record['views']
+            else:
+                master_list_by_date[date] = {}
+                master_list_by_date[date]['date'] = date
+                master_list_by_date[date]['participations'] = record['participations']
+                master_list_by_date[date]['views'] = record['views']
+    out = open('canvas/daily.json','w')
+    # want to match the old, funny format
+    by_date = []
+    my_out = {'by_date':by_date}
+    
+    for day in master_list_by_date.keys():
+        by_date.append(master_list_by_date[day])
+    out.write(json.dumps(my_out,indent=2))
+
+
+
+
+
+
+
+def externaltool():        # a list
+
+
+    #mydata = {  "course_navigation[text]": "Video Chat",
+    #            "course_navigation[default]": "false" }
+    #t = url + '/api/v1/accounts/1/external_tools/704?course_navigation[text]=Video Chat&course_navigation[default]=false'
+    #r = requests.put(t,  headers=header)
+    #print r.text
+    t = url + '/api/v1/accounts/1/external_tools/'
+    while(t): t = fetch(t)
+    print results
+    
\ No newline at end of file
diff --git a/tasks.py b/tasks.py
new file mode 100644
index 0000000..611d215
--- /dev/null
+++ b/tasks.py
@@ -0,0 +1,1418 @@
+
+""" Random tasks to classes and canvas,
+
+  Making everyone a teacher
+  Fixing closed class
+  Bulk enrolling people
+  Positive attendance / 20 - 60 calculation
+  Bulk crosslisting
+  
+  sftp upload to web
+  
+  badgr stuff
+  
+"""
+
+import pysftp, os, datetime, requests, re, json, sqlite3, codecs, csv, sys
+import funcy, os.path, shutil, urllib
+from datetime import datetime
+
+from secrets import badgr_target, badgr_hd
+
+
+if os.name != 'posix':
+    import win32com.client
+    import win32com.client as win32
+    import pypandoc
+    from docxtpl import DocxTemplate
+
+from pipelines import header, url, fetch
+
+#from localcache import local_data_folder, sqlite_file, db, user_goo_to_email
+
+
+def survey_answer(q=0):
+
+    if not q:
+        q = int( input("which column? ") )
+        
+    fff = csv.reader( codecs.open('cache/sp21_survey_answers.csv','r','utf-8'), delimiter=',')
+
+    for row in fff:
+        print(row[q])
+
+def survey_organize():
+
+    fff = csv.reader( codecs.open('cache/sp21_survey_answers.csv','r','utf-8'), delimiter=',')
+    
+    ans = []
+    for i in range(27): ans.append([])
+
+    for row in fff:
+        for i,item in enumerate(row):
+            print(item)
+            ans[i].append(item)
+            
+    for i in range(27): ans[i].sort()
+    
+    outp = codecs.open('cache/sp21_answersinorder.txt','w','utf-8')
+    
+    for i in range(27): 
+        outp.write( "\n".join(ans[i]) )
+        outp.write("\n\n\n\n\n")
+
+
+
+def build_quiz(filename=""):
+    if not filename:
+        filename = 'cache/he2.txt'
+
+    quiz_id = "33285"
+    course_id = "10179"
+    quiz_group = 15096
+
+    input_lines = codecs.open(filename,'r', 'utf-8').readlines()
+    qs = []
+    qs_post_data = []
+    this_q = ""
+    this_as = {  }
+    correct_answer = ""
+    
+    state = "q_text"
+
+    for L in input_lines:
+        if state == "q_text":
+            this_q = L.strip()
+            state = "answers"
+        elif state =="answers":
+            m = re.search( '^Answer\:\s(\w)$', L)
+            if m:
+                correct_answer = m.group(1)
+                qs.append( [this_q, this_as, correct_answer ] )
+                state = "q_text"
+                this_as = {  }
+                correct_answer = ""
+                continue
+            m = re.search( '^(\w)\)\s(.*)$', L)
+            if m:
+                print(m.group(1))
+                print(m.group(2))
+                this_as[m.group(1)] = m.group(2)
+    print(json.dumps( qs, indent=2 ))
+
+    i = 1
+    for Q in qs:
+        answers = []
+        for k,v in Q[1].items():
+            answers.append({"answer_text": v, "answer_weight": 100 if k==Q[2] else 0, })
+        this_q = { "question": {"question_name": "q"+str(i),
+                   "position": i,
+                   "question_text": Q[0],
+                   "question_type": "multiple_choice_question",
+                   "points_possible": 1,
+                   "answers": answers}}
+        qs_post_data.append(this_q)
+        i += 1
+
+    for Q in qs_post_data:
+        print(json.dumps(Q, indent=2))
+        if input("enter to upload, or s to skip: ") != "s":
+            u = url + "/api/v1/courses/%s/quizzes/%s/questions" % (course_id, quiz_id)
+            print(u)
+            resp = requests.post( u, headers=header, json=Q ) 
+            print ( resp )
+            print ( resp.text )
+            print()
+
+
+
+# Send an email
+def send_email(fullname, firstname, addr, subj, content):
+    outlook = win32.Dispatch('outlook.application')   #get a reference to Outlook
+    mail = outlook.CreateItem(0)                      #create a new mail item
+    mail.To = addr
+    mail.Subject = subj
+
+    mail.HTMLBody = content
+    mail.Display()
+
+
+def convert_to_pdf(name1, name2):
+    wd = 'C:\\Users\\peter\\Documents\\gavilan\\canvasapp\\'
+    print( wd + name1 )
+    try:
+        word = win32.DispatchEx("Word.Application")
+        worddoc = word.Documents.Open(wd+name1)
+        worddoc.SaveAs(wd+name2, FileFormat = 17)
+        worddoc.Close()
+    except Exception as e:
+        print(e)
+        return e
+    finally:
+        word.Quit()
+
+
+# Build (docx/pdf) certificates for gott graduates
+def certificates_gott_build():
+    #send_email("Peter Howell", "Peter", "phowell@gavilan.edu", "test", "this is a test")
+    
+    g2e = user_goo_to_email()
+    g2name = {}
+    ix = {}     # everyone 
+    ix1 = {}    # only gott 1
+    ix2 = {}    # only gott 2
+
+    cc = csv.reader( open('cache/completers_gott1_su20.csv','r'), delimiter=',')
+    cc2 = csv.reader( open('cache/completers_gott2_su20.csv','r'), delimiter=',')
+
+    for row in cc:
+        # name, goo, section, x, count
+        doc = DocxTemplate("cache/certificates/gott 1 template.docx")
+        doc.render({ 'name' : row[0] })
+        fn = "cache/certificates/gott_1_%s." % re.sub('\s', '_', row[0].lower())
+        print(fn+'docx')
+        try:
+            goo = row[1]
+            email = g2e[ goo ]
+            print(email)
+            g2name[goo] = row[0]
+            ix1[ goo ] = fn+"pdf"
+            ix[ goo ] = email
+        except:
+            print("can't find email")
+        doc.save(fn+'docx')
+        #convert_to_pdf(fn+'docx', fn+'pdf')
+
+    for row in cc2:
+        # name, goo, section, x, count
+        doc = DocxTemplate("cache/certificates/gott 2 template.docx")
+        doc.render({ 'name' : row[0] })
+        fn = "cache/certificates/gott_2_%s." % re.sub('\s', '_', row[0].lower())
+        print(fn+'docx')
+        try:
+            goo = row[1]
+            email = g2e[ goo ]
+            print( email )
+            g2name[goo] = row[0]
+            ix2[ goo ] = fn+"pdf"
+            ix[ goo ] = email
+        except:
+            print("can't find email")
+        doc.save(fn+'docx')
+        #convert_to_pdf(fn+'docx', fn+'pdf')
+    
+    # 
+    g1f = open('cache/gott_emails_1.csv', 'w')
+    g2f = open('cache/gott_emails_2.csv', 'w')
+    g12f = open('cache/gott_emails_12.csv', 'w')
+    for k in ix.keys():
+        if k in ix1 and not k in ix2:
+            print(k + " only gott 1")
+            file1 = ix1[k]
+            email = ix[k]
+            file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1]
+            fname = g2name[k]
+            g1f.write("%s, %s, %s\n" % (fname, email, file1))
+        elif k in ix2 and not k in ix1:
+            print(k + " only in gott 2")
+            file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1]
+            email = ix[k]
+            fname = g2name[k]
+            g2f.write("%s, %s, %s\n" % (fname, email, file2))
+        elif k in ix1 and k in ix2:
+            print(k + " in both")
+            file1 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix1[k].split("/")[-1]
+            file2 = "https://www.gavilan.edu/staff/tlc/certificates/" + ix2[k].split("/")[-1]
+            email = ix[k]
+            fname = g2name[k]
+            g12f.write("%s, %s, %s, %s\n" % (fname, email, file1, file2))
+
+
+# Email experiment
+def mail_test():
+    outlook = win32com.client.Dispatch('outlook.application')   #get a reference to Outlook
+    mail = outlook.CreateItem(0)                      #create a new mail item
+    mail.To = 'executives@bigcompany.com'
+    mail.Subject = 'Finance Status Report '+datetime.today().strftime('%m/%d')  
+
+    mail.HTMLBody = '''
+    <p>Hi Team,</p>
+
+    <p>This email is to provide a status of the our current sales numbers</p>
+
+
+    <img src='C:\\Users\\sam\\Desktop\\EmailAuto\\paste1.png'>
+
+
+    <img src='C:\\Users\\sam\\Desktop\\EmailAuto\\paste2.png'>
+
+
+    <p>Thanks and have a great day!</p>
+    '''
+    mail.Display()
+
+
+# Change LTI Settings. Experimental
+def modify_x_tool():   
+    u2 = "https://gavilan.instructure.com:443/api/v1/accounts/1/external_tools/1462"
+    params = {'course_navigation[default]':'false',  "course_navigation[enabled]": "true",
+    "course_navigation[text]": "NameCoach",
+    "course_navigation[url]": "https://www.name-coach.com/lti/single_page/participants/init",
+    "course_navigation[visibility]": "public",
+    "course_navigation[label]": "NameCoach",
+    "course_navigation[selection_width]": 800,
+    "course_navigation[selection_height]": 400}
+    r2 = requests.put(u2, headers=header, data=params)
+    print ( r2.text )
+
+
+# Upload with sftp to www website folder:   student/online/srt/classfoldername
+def put_file(classfoldername):
+    folder = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    cnopts = pysftp.CnOpts()
+    cnopts.hostkeys = None
+    with pysftp.Connection('www.gavilan.edu',username='cms', password='TODO',cnopts=cnopts) as sftp:
+        sftp.chdir('student/online/srt/'+classfoldername)
+        files = sftp.listdir()
+        print ( folder + "\tI see these files on remote: ", files, "\n" )
+        
+        localf = os.listdir('video_srt/'+classfoldername)
+        
+        print ( "I see these local: ", localf )
+        
+        # copy files and directories from local static, to remote static,
+        # preserving modification times on the files
+        for f in localf:
+            print ( "This local file: " + f + " ", )
+            if not f in files: 
+                sftp.put('video_srt/'+classfoldername+'/'+f, f, preserve_mtime=True)
+                print ( "Uploaded." )
+            else:
+                print ( "Skipped." )
+
+
+        if len(files)==3 and 'users.csv' in files:
+            sftp.get('courses.csv','rosters/courses-'+folder+'.csv')
+            sftp.get('users.csv','rosters/users-'+folder+'.csv')
+            sftp.get('enrollments.csv','rosters/enrollments-'+folder+'.csv')
+            print ( folder + '\tSaved three data files in rosters folder.' )
+            
+            courses = open('rosters/courses-'+folder+'.csv','r')
+            courses.readline()
+            a = courses.readline()
+            print ( a )
+            courses.close()
+            parts = a.split(',')
+            year = parts[1][0:4]
+            ss = parts[1][4:6]
+            #print ( parts[1] )
+            sem = {'30':'spring', '50':'summer', '70':'fall' }
+            this_sem = sem[ss]
+            #print ( this_sem, "", year )
+            print ( folder + '\tbuilding data file...' )
+            convert_roster_files(this_sem,year,folder)
+            print ( folder + '\tmoving files...' )
+            move_to_folder(this_sem,year,folder)
+        else:
+            print ( folder + "\tDon't see all three files." )
+    sftp.close()
+
+
+# Switch everyone in a class to a teacher
+def switch_enrol():
+    global results, header
+    results = []
+    id = raw_input("Id of course? ")
+    url = "https://gavilan.instructure.com:443/api/v1/courses/"+id+"/enrollments?type[]=StudentEnrollment"
+    while (url): url = fetch(url)
+    all_stud = results
+    for S in all_stud:
+        print ( S['user']['name'] )
+        print(  "Switching to teacher." )
+        u2 = "https://gavilan.instructure.com:443/api/v1/courses/"+id+"/enrollments"
+        params = {'course_id':id, 'enrollment[user_id]':S['user_id'],'enrollment[type]':'TeacherEnrollment'}
+        r2 = requests.post(u2, headers=header, data=params)
+        #print(  "Response: ", r2.text )
+        #res = json.loads(r2.text)
+        #if raw_input('continue? y/n ') == 'y':
+        u3 = "https://gavilan.instructure.com:443/api/v1/courses/"+id+"/enrollments/"+str(S['id'])
+        params = {'course_id':id, 'id':S['id'],'task':'delete'}
+        r3 = requests.delete(u3, headers=header, data=params)
+        #print(  "Response: ", r3.text )
+
+# Change dates & term of a class to unrestrict enrollment
+def unrestrict_course():
+    id = raw_input('the course id? ')
+    t1 = url + '/api/v1/courses/' + id
+    course = fetch(t1)
+
+    # CHANGE DATES CHANGE TERM
+    print(  str(course['id']) + "\t", course['name'], "\t", course['workflow_state'] )
+    t2 = url + '/api/v1/courses/' + str(course['id'])
+    data = {'course[end_at]':'','course[restrict_enrollments_to_course_dates]': 'false', 'course[term_id]':'27'}
+    r2 = requests.put(t2, headers=header, params=data)
+    print(  "\t", r2 )
+    print(  'ok' )
+    print(  "\tEnd at: " + str(course['end_at']) )
+    print(  "\tRestricted enrollment: " + str(course['restrict_enrollments_to_course_dates']) )
+    
+    
+    
+    # ADD ENROLLMENT
+    t3 = url + '/api/v1/courses/' + id + '/enrollments' 
+    form = {'enrollment[user_id]':'30286', 'enrollment[type]':'TaEnrollment', 'enrollment[enrollment_state]':'active' }
+    #r3 = requests.post(t3, headers=header, params=form)
+    #print(  "\t", r3.text )
+    print(  '\tok' )
+
+    
+# Bulk enroll users into a course    
+def enroll_accred():
+    global results, results_dict,header
+    # enroll this account in every published course in the semester
+    r = url + '/api/v1/accounts/1/courses?enrollment_term_id=23&perpage=100'
+    all_courses = fetch(r)
+    i = 0
+    #print(  "These courses have custom dates and restricted enrollment:" )
+    for k in all_courses:
+        if k['workflow_state'] in ['completed','available']:
+            i += 1
+            ### Handle courses with custom end date and restricted entry. Turn that off.
+            
+            print ( str(i) + ".\t", str(k['id']) + "\t", k['name'], "\t", k['workflow_state'] )
+            t3 = url + '/api/v1/courses/' + str(k['id']) + '/enrollments' 
+            form = {'enrollment[user_id]':'30286', 'enrollment[type]':'TeacherEnrollment', 'enrollment[enrollment_state]':'active' }
+            r3 = requests.post(t3, headers=header, params=form)
+            print ( "\t", r2.text )
+            print ( '\tok' )
+            
+            
+
+# Calculate attendance stats based on enrollment/participation at 20% of term progressed, then 60% of term progressed.
+def twenty_sixty_stats(li):
+    # actual calcs core. li is a list of lines.
+    cen1_only = []
+    cen2_only = []
+    neither = []
+    both = []
+    for L in li:
+        L = L.strip()
+        parts = L.split(",")
+        
+        # id, lname, fname, before_class_start, before_1st_cen, before_2nd_cen, after_2nd_cen, after_class_end, final_score
+        cen1_yes = int(parts[3]) + int(parts[4])
+        cen2_yes = int(parts[5])
+        
+        if cen1_yes and not cen2_yes:
+            cen1_only.append(parts)
+        elif cen2_yes and not cen1_yes:
+            cen2_only.append(parts)
+        elif not cen1_yes and not cen2_yes:
+            neither.append(parts)
+        elif cen1_yes and cen2_yes:
+            both.append(parts)
+        else:
+            print ( "Error: " + L )
+            
+    #fout = codecs.open('pa_census_'+m.group(1)+'.txt', 'w','utf-8')
+    ret = []
+    ret.append("cen 1 = " + str(len(cen1_only)+len(both)) + ", cen2 = "+ str(len(cen2_only)+len(both)) + ", AVERAGE = " + str( ( len(cen1_only) +len(both) + len(cen2_only)+len(both)  ) / 2.0 ) + "\n\n")
+    ret.append("Census 1 Only: " + str(len(cen1_only)) + "\n")
+    for L in cen1_only:
+        ret.append(",".join(L)+"\n")
+    ret.append("\nCensus 2 Only: " + str(len(cen2_only)) + "\n")
+    for L in cen2_only:
+        ret.append(",".join(L)+"\n")
+    ret.append("\nBoth: " + str(len(both)) + "\n")
+    for L in both:
+        ret.append(",".join(L)+"\n")
+    ret.append("\nNeither: " + str(len(neither)) + "\n")
+    for L in neither:
+        ret.append(",".join(L)+"\n")
+    return ''.join(ret)
+    
+    
+# Older positive attendance hours calculation.
+def hours_calc():
+
+    # open and read enrollments
+    enrol = json.loads( open('semesters/2018fall/roster_fall18.json','r').read() )[2]
+    
+    # {"course_id": "201870-10001", "status": "active", "role": "student", "user_id": "G00256034"}
+    
+    my_sections = '10689,10977,10978,10979,10980,10981,10982,10983,10985,11074,11075,11076'.split(",")
+    
+    enrollments = defaultdict(list)
+    for E in enrol:
+        id = E['course_id'][7:]
+        if id in my_sections:
+            enrollments[id].append(E['user_id'])
+    #print ( json.dumps(enrollments,indent=2) )
+    allout = codecs.open('pa_de_noncred.txt','w','utf-8')
+    
+    for f in os.listdir('.'):
+        m = re.match('pa(\d+)\.txt',f)
+        if m:
+            sec = m.group(1)
+            # split up the combined sections
+            if sec == '10977':
+                possible = '10977,10978,10979'.split(',')
+            elif sec == '10980':
+                possible = '10980,10981,10982,10983'.split(',')
+            elif sec == '10985':
+                possible = '10985,11074,11075,11076'.split(',')
+            else:
+                possible = ['10689',]
+            lines_by_sec = {}
+            for s in possible:
+                lines_by_sec[s] = []
+            fin = codecs.open(f,'r','utf-8').readlines()[1:]
+            for L in fin:
+                parts = L.split(",") # id, lname, fname, before_class_start, before_1st_cen, before_2nd_cen, after_2nd_cen, after_class_end, final_score
+                for s in possible:
+                    if parts[0] in enrollments[s]:
+                        lines_by_sec[s].append(L)
+                        #break
+            #print ( "Split up section " + sec + json.dumps(lines_by_sec,indent=2) )
+            
+            for S,v in lines_by_sec.items():
+                allout.write("\n\nSection " + S + "\n")
+                allout.write(twenty_sixty_stats(v) + "\n - - - - - \n\n")
+
+def course_2060_dates(crn=""):
+    schedfile = 'su20_sched.json'   # TODO
+    schedule = json.loads(open(schedfile,'r').read())
+    ok = 0
+    if not crn:
+        crn = raw_input("What is the CRN?  ")
+    for s in schedule:
+        if s['crn']== crn:
+            ok = 1
+            break
+    if not ok:
+        print ( 'I couldn\'t find that CRN in ' + schedfile )
+    else:
+        a = s['date'].split(' - ')
+        beginT = strptime(a[0],"%b %d, %Y")
+        endT = strptime(a[1],"%b %d, %Y")
+        
+        # Begin and end dates - direct from schedule
+        # Calculate 20% / 60% dates.
+
+        beginDT = datetime.datetime.fromtimestamp(mktime(beginT))
+        endDT = datetime.datetime.fromtimestamp(mktime(endT))
+        seconds_length = mktime(endT) - mktime(beginT)
+        length = datetime.timedelta( seconds=seconds_length )
+        first_cen_date = datetime.timedelta( seconds=(0.2 * seconds_length)) + beginDT
+        second_cen_date = datetime.timedelta( seconds=(0.6 * seconds_length)) + beginDT
+        
+        print ( "Begin: " + str(beginDT) )
+        print ( "End: " + str(endDT) )
+        print ( "The length is: " + str(length) )
+        print ( "First census date is: " + str(first_cen_date) )
+        print ( "Second census date is: " + str(second_cen_date) )
+
+        return (first_cen_date, second_cen_date)
+
+def course_update_all_users_locallogs(course_id=''):
+    if not course_id:
+        course_id = raw_input("ID of course to calculate hours?  ")
+    
+    emts = course_enrollment(course_id)
+    #print(emts)
+
+def hours_calc_pulldata(course_id=''):
+    if not course_id:
+        course_id = raw_input("ID of course to calculate hours?  ")
+    
+    emts = course_enrollment(course_id)
+    #print(emts)
+    
+    # all users in this course
+    results = []
+    t = '/api/v1/courses/' + str(course_id) + '/users'
+    my_users = fetch(t)
+    count = 0
+    
+    pa = codecs.open('cache/pa.txt','w','utf-8')
+    pa.write("id, lname, fname, before_class_start, before_1st_cen, before_2nd_cen, after_2nd_cen, after_class_end, final_score\n")
+    
+    for S in my_users:
+        try:
+            results = []
+            #if count > 3: break
+            count += 1
+            #print ( count )
+            target = url + '/api/v1/users/' + str(S['id']) + '/page_views?per_page=200'
+            while target: target = fetch(target)
+            
+            # have all student's hits. Filter to only this class
+            results = filter(lambda x: str(x['links']['context']) == id,results)
+            bag = { 0:0, 1:0, 2:0, 3:0, 4:0 }
+            
+            if results:
+                for hit in results:
+                    hitT = strptime(str(hit['created_at']),"%Y-%m-%dT%H:%M:%SZ")
+                    hittime = datetime.datetime.fromtimestamp( mktime(hitT) )
+                    
+                    if hittime > endDT: bag[4] += 1
+                    elif hittime > second_cen_date: bag[3] += 1
+                    elif hittime > first_cen_date: bag[2] += 1
+                    elif hittime > beginDT: bag[1] += 1
+                    else: bag[0] += 1
+            
+            record = emts[S['id']]['user']['login_id'] + ", " +emts[S['id']]['user']['sortable_name'] + ", " + str(bag[0]) + ", " + str(bag[1]) + ", " + str(bag[2]) + ", " + str(bag[3]) + ", " + str(bag[4]) + ", " + str(emts[S['id']]['grades']['final_score'])
+            print ( record )
+            pa.write( record + "\n" )
+            pa.flush()
+        except Exception as exp:
+            #errors += S['firstname'] + " " + S['lastname'] + " " + S['id'] + "\n"
+            print ( 'exception with ', )
+            print ( S )
+            print ( exp )
+            pass
+
+
+            
+    #t = url + '/api/v1/accounts/1/courses/' + str(id)
+    #print ( t )
+    #while(t): t = fetch_dict(t)
+    #print ( "These are the results: " )
+    #print ( results_dict )
+    
+    #prettydate = X.strptime("%b %d, %Y")
+    
+
+
+
+def xlist_cwe():
+    this_sem_190_id = 15464   # they get 190s and 290s
+    this_sem_192_id = 15318   # they get 192s
+    
+    
+    this_sem_guid558 = 7592
+    
+    #search_string = "290"
+    search_string = "192"
+    #search_string = "190"
+    the_course_id =this_sem_192_id    # this_sem_190_id
+    
+    this_sem_term = 176    # fa22
+    
+    sem_cache_fn = 'cache/semester_courses_%s.json' % this_sem_term
+    docache = input("Use cache courses? (n = fetch fresh list)   y/n: ")
+    
+    # get all courses this semester. Filter to ___ sections
+    if docache != 'y':
+        t = url + \
+            '/api/v1/accounts/1/courses?enrollment_term_id=%i&perpage=100' % this_sem_term
+        sem_courses = fetch(t,1)
+        out_file = codecs.open(sem_cache_fn, 'w', 'utf-8')
+        out_file.write( json.dumps(sem_courses,indent=2) )
+        out_file.close()
+    else:
+        sem_courses = json.loads(codecs.open(sem_cache_fn,'r','utf-8').read())
+    
+    for R in sem_courses:
+        try:
+            if re.search(search_string, R['name']) and str(R['id']) != str(the_course_id):    
+                
+                # use the course to get the section id
+                print ( R['name'] )
+                u = url + '/api/v1/courses/%i/sections' % R['id']
+                for S in fetch(u):
+                    if (S['id']):
+                        myanswer = input( "-> Should I crosslist: %i\t%s\tsection id: %i (y/n) " % (R['id'],R['name'],S['id'] ))
+                        if myanswer=='y':
+                            # cross list
+                            v = url + "/api/v1/sections/%i/crosslist/%i" % (S['id'],the_course_id)
+                            res = requests.post(v, headers = header)
+                            print( json.dumps( json.loads(res.text), indent=2) )
+
+                print()
+        except Exception as e:
+            print( "Caused a problem: " + str(e) + "\n" + str(R) + "\n" )
+
+            
+            
+            
+            
+            
+            
+            
+            
+            
+def pos_atten():
+    global f, url, results, count, pa, users_by_id, dd
+    errors = ""
+    wr = csv.writer(f,quoting=csv.QUOTE_ALL)
+    pa_wr = csv.writer(pa,quoting=csv.QUOTE_MINIMAL)
+    load_users()
+    
+    # get users in course 59
+    target = url + '/api/v1/courses/3295/users?per_page=100'
+    while target:
+        print ( target )
+        target = fetch(target)
+    
+    students = results
+    results = []
+    
+    count = 1
+    wb = xlwt.Workbook()
+    ws = wb.add_sheet('Course Attendance')
+    
+    ws.write(0,0, "Positive Attendance Report")
+    ws.write(1,0, "2018 Spring Semester")
+    ws.write(2,0, "LIB 732 Lawrence")
+    
+    col = 0
+    row = 5
+    
+    for label in "ID,Lastname Firstname,Hits Total,Sessions Total,Minutes Total,Session Date,Session Hits,Session Minutes".split(","):
+        ws.write(row,col)
+        col +=1
+    col = 0
+    f.write("userid,time,ip,url,context,browser,action\n")
+    pa.write("id,lastname,firstname,hits total,sessions total, minutes total,session date,session hits, session minutes\n")
+    dd.write("[")
+    for S in students:
+        try:
+            results = []
+            ######################
+            if count > 10: break
+            count += 1
+            print ( count )
+            target = url + '/api/v1/users/' + str(S['id']) + '/page_views?per_page=200'
+            while target:
+                print ( target )
+                target = fetch(target)
+            # have all student's hits. Filter to only this class
+            results = filter(match59,results)
+            if results:
+                times = []
+                for hit in results:
+                    L = [hit['links']['user'],hit['updated_at'],hit['remote_ip'],hit['url'],hit['context_type'],hit['user_agent'],hit['action']]
+                    times.insert(0,hit['updated_at'])
+                    wr.writerow(L)
+                print ( times )
+                uu = users_by_id[ S['id'] ]
+                dd.write("{ label: '" + uu['sortable_name'] + "', times: ")
+                part = partition(times)   # also writes to dd
+                dd.write("},\n")
+                
+                # print ( students list of sessions )
+                hits_total = sum(  [h[1] for h in part] )
+                mins_total = sum(  [h[2] for h in part] )
+                lname,fname = uu['sortable_name'].split(",")
+                pa_wr.writerow( [ uu['login_id'], lname,fname, hits_total, str(len(part)),mins_total ] )
+                for xxy in [ uu['login_id'], lname,fname, hits_total, str(len(part)),mins_total ]:
+                    ws.write(row,col,xxy)
+                    col += 1
+                row +=1
+                col = 0
+                for P in part:
+                    pa_wr.writerow([ '','','','','','',P[0],P[1],P[2]])
+                    for xxy in [ '','','','','','',P[0],P[1],P[2]]:
+                        ws.write(row,col,xxy)
+                        col +=1
+                    row += 1
+                    col = 0
+            print ( part )
+            print ( "\n\n" )
+        except Exception as exp:
+            #errors += S['firstname'] + " " + S['lastname'] + " " + S['id'] + "\n"
+            pass
+    dd.write("];\n")    
+    wb.save('pos_atn.xls')
+    
+    err = codecs.open('pa_error.txt','w', encoding='utf-8')
+    err.write(errors)
+  
+
+##
+## Images - profile photos - can exist as:
+##
+##   - picsStaffdir  ... or the images_sm dir on www/staff. 
+##       + alternative copies have 2..3..etc  appended
+##   - the new badge photo folder
+##   - ilearn profile pics
+##
+
+
+
+first_name_subs = """Analisa,Analisa (Lisa)
+Angelic,Angie
+Beatriz,Bea
+Christopher,Chris
+Conception,Connie
+Cynthia,Cindy
+David,Dave
+Deborah,Debbie
+Debra,Debbie
+Desiree,Desiree (Danelle)
+Diana,Diane
+Doug,Douglas
+Frank,Frank (Nick)
+Herbert,Herb
+Irving,Irv
+Isabel,Izzy
+Isela, Isela M.
+Janet,Jan
+Jeffrey,Jeff
+Jiayin,Jiayain
+Joanne,Jo Anne
+Jolynda,JoLynda
+Jonathan,Jon
+Josefina,Josie
+Juan,Juan Esteban
+Kathryn,Katie
+Kenneth,Ken
+Kim,Kimberly
+Lori,Lorraine
+Lucy,Lucila
+Margaret,Margie
+Maria,Maggie
+Maria,Mari
+Maria,Maria (Lupe)
+Mathew,Matthew
+Miriam,Mayra
+Nicholas,Nick
+Osvaldo,Ozzy
+Pam,Pamela
+Ronald,Ron
+Rosangela,Rose
+Sandra,Sandy
+Silvia,Sylvia
+Tamara,Tammy
+Timothy,Craig
+Wong-Lane,Wong
+van Tuyl,Vantuyl""".split("\n")
+
+last_name_subs = """Besson,Besson-Silvia
+Bernabe Perez,Bernabe
+Chargin,Bernstein Chargin
+Dequin,Dequin Bena
+Dufresne,Dufresne Reyes
+Gonzalez,Gonzalez Mireles
+Haehl,Lawton-Haehl
+Hooper,Hooper-Fernandes
+Lacarra,LaCarra
+Larose,LaRose
+MacEdo,Macedo
+Miller,Miller Young
+Najar-Santoyo,Najar
+Rocha-Gaso,Rocha
+Smith,Keys
+Vargas-Padilla,Vargas
+de Reza,DeReza
+del Carmen,Del Carmen""".split("\n")
+
+def lname(x):
+    return x.split(' ')[-1]
+
+def l_initial(x):
+    return x.split(' ')[-1][0]
+
+def job_titles2():
+    inn = open('cache/2020_job_titles.csv','r').readlines()
+    
+    inn = [ x.split(',')[1].strip() for x in inn ]
+    
+    inn = list(funcy.distinct(inn))
+    inn.sort()
+    
+    if 0:
+        ioo = open('cache/2020_job_title_to_ix.csv','w')
+        for x in inn:
+            ioo.write("%s,\n" % x)
+    
+    u1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=get/jobtitles"
+    
+    sss = json.loads(  requests.get(u1).text  )
+    ibb = []
+    for s in sss:
+        ibb.append(s['name'])
+    #print(ibb)
+    
+    print(json.dumps(inn,indent=2))
+    
+    
+    
+def job_titles():
+    title_ids = open('cache/2020_job_title_to_ix.csv','r').readlines()
+    t_ids = [ x.strip().split(',') for x in title_ids ]
+    title_to_id = {}
+    
+    for x in t_ids:
+        #print(x)
+        #ttii = x.split(',')
+        title_to_id[ x[0] ] = x[1]
+        
+    #print(title_to_id)
+    
+    inn = open('cache/2020_job_titles.csv','r').readlines()
+    inn = [ x.split(',') for x in inn ]
+    name_to_title = {}
+    
+    
+    for x in inn: 
+        #print(x[0].strip())
+        parts = x[0].strip().split(' ')
+        fl_name = "%s %s" % ( parts[0], parts[-1] )
+        
+        name_to_title[ x[0] ] = x[1].strip()
+        name_to_title[ fl_name ] = x[1].strip()
+        
+        firstname_variations = []
+        first = parts[0]
+        lastname = " ".join(parts[1:])
+        for fns in first_name_subs:
+            fns_parts = fns.split(',')
+            subbed = re.sub('^'+fns_parts[0]+'$',fns_parts[1].strip(), first)
+            if first != subbed:
+                #print("Subbed %s %s for %s %s" % (subbed,lastname, first, lastname))
+                name_to_title[ subbed + " " + lastname ] = x[1].strip()
+            subbed = re.sub('^'+fns_parts[1].strip()+'$',fns_parts[0], first)
+            if first != subbed:
+                #print("Subbed %s %s for %s %s" % (subbed,lastname, first, lastname))
+                name_to_title[ subbed + " " + lastname ] = x[1].strip()
+        for lns in last_name_subs:
+            fns_parts = lns.split(',')
+            subbed = re.sub('^'+fns_parts[0]+'$',fns_parts[1].strip(), lastname)
+            if lastname != subbed:
+                #print("L Subbed %s %s for %s %s" % (first, subbed, first, lastname))
+                name_to_title[ first + " " + subbed ] = x[1].strip()
+            subbed = re.sub('^'+fns_parts[1].strip()+'$',fns_parts[0], lastname)
+            if lastname != subbed:
+                #print("L Subbed %s %s for %s %s" % (first, subbed, first, lastname))
+                name_to_title[ first + " " + subbed ] = x[1].strip()
+    
+    
+    unmatched_dir_names = []
+    
+    m1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=menus"
+    menus = json.loads(  requests.get(m1).text  )
+    id_to_title = {}
+    for m in menus['titles']:
+        id_to_title[ m['id'] ] = m['name']
+    id_to_dept = {}
+    for m in menus['departments']:
+        id_to_dept[ m['id'] ] = m['name']
+    
+    u1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=list/staffsemester"
+    sss = json.loads(  requests.get(u1).text  )
+    count1 = 0
+    count2 = 0
+    
+    warning = open('cache/missing_ext_row.txt','w')
+
+    for s in sss:
+        easy_name = "%s %s" % (s['first_name'].strip(), s['last_name'].strip())
+        if easy_name in name_to_title:
+            print( " + %s is %s" % (easy_name, name_to_title[easy_name]) )
+            p1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=get/user/%s" % str(s['id'])
+            uuu = json.loads(  requests.get(p1).text  )
+            print("\nFound: %s" % easy_name)
+            print("\tDepartment: %s" % uuu['department'])
+            if not 'ext_id' in uuu:
+                print('\tWARNING no personnel_ext row found!') 
+                warning.write("%s,%s\n" % (easy_name, str(uuu['id'])))
+            if 'dept1' in uuu and uuu['dept1']:
+                print("\tDept1: %s" % id_to_dept[ uuu['dept1'] ])
+            if 'gtitle' in uuu and uuu['gtitle']:
+                print("\tTitle: %s" %  id_to_title[ uuu['gtitle'] ])
+            print("\tDo you want to change the title to %s? y/n " % name_to_title[easy_name])
+            new_title = name_to_title[easy_name]
+            new_title_id = title_to_id[ new_title ]
+            yn = input("\tid: %s   " % str(new_title_id))
+            if yn == 'y':
+                print("...gonna change...")
+                uppy = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=update_xt&cols=%s&vals=%s&id=%s" % (  "gtitle",  str(new_title_id), str(uuu['ext_id'])   )
+                print(uppy)
+                #res = json.loads(  requests.get(uppy).text  )
+                res =  requests.get(uppy).text  
+                print("")
+                print(res)
+                print("")
+
+            #xyz = input()
+            count1 += 1
+        else:
+            print( " - %s " % easy_name )
+            unmatched_dir_names.append(easy_name)
+            count2 += 1
+        #print( json.dumps(s,indent=2) )
+    print("\nMatched %i names, with %i remaining unmatched" % (count1, count2) )
+    print(menus['titles'])
+    
+    return
+    cola = funcy.group_by( l_initial, t_names )
+    colb = funcy.group_by( l_initial, unmatched_dir_names )
+    
+    initials = list(funcy.concat(cola.keys(), colb.keys()))
+    initials = list(funcy.distinct(initials))
+    initials.sort()
+    
+    for i in initials:
+        if i in cola:
+            print('-> title file')
+            for a in cola[i]: print("\t"+a)
+        if i in colb:
+            print('-> dir db')
+            for b in colb[i]: print("\t"+b)
+        print()
+        
+    
+    """longer = max(len(t_names), len(unmatched_dir_names))
+    
+    for i in range(longer):
+        cola = ''
+        colb = ''
+        if len(t_names) > i:
+            cola = t_names[i]
+        if len(unmatched_dir_names) > i:
+            colb = unmatched_dir_names[i]
+            
+        print(" %s\t\t%s" % (cola,colb))
+    """
+        
+
+# an early version, before tearing up...
+def job_titles3():
+    inn = open('cache/2020_job_titles.csv','r').readlines()
+    
+    inn = [ x.split(',') for x in inn ]
+    t_names = []
+    fl_names = []
+    
+    name_to_title = {}
+    fl_to_title = {}
+    
+    for x in inn: 
+        parts = x[0].strip().split(' ')
+        fl_name = "%s %s" % ( parts[0], parts[-1] )
+        
+        t_names.append(  x[0] )
+        fl_names.append(  fl_name)
+        name_to_title[ x[0] ] = x[1].strip()
+        fl_to_title[ fl_name ] = x[1].strip()
+    
+    #print( json.dumps(name_to_title,indent=2) )
+    
+    # t_names has the "state list"
+    t_names.sort( key=lname )
+    
+    unmatched_dir_names = []
+    
+    u1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=list/staffsemester"
+    
+    sss = json.loads(  requests.get(u1).text  )
+    count1 = 0
+    count2 = 0
+    count3 = 0
+    
+    for s in sss:
+        easy_name = "%s %s" % (s['first_name'].strip(), s['last_name'].strip())
+        if easy_name in t_names:
+            print( " + %s is %s" % (easy_name, name_to_title[easy_name]) )
+            t_names.remove(easy_name)
+            count1 += 1
+        elif easy_name in fl_names:
+            print( " + %s is %s" % (easy_name, fl_to_title[easy_name]) )
+            fl_names.remove(easy_name)
+            count3 += 1
+        else:
+            print( " . %s " % easy_name )
+            unmatched_dir_names.append(easy_name)
+            count2 += 1
+        #print( json.dumps(s,indent=2) )
+    print("\nMatched %i names, %i F->L only, with %i remaining unmatched" % (count1,count3, count2) )
+    print()
+    
+    cola = funcy.group_by( l_initial, t_names )
+    colb = funcy.group_by( l_initial, unmatched_dir_names )
+    
+    initials = list(funcy.concat(cola.keys(), colb.keys()))
+    initials = list(funcy.distinct(initials))
+    initials.sort()
+    
+    for i in initials:
+        if i in cola:
+            print('-> title file')
+            for a in cola[i]: print("\t"+a)
+        if i in colb:
+            print('-> dir db')
+            for b in colb[i]: print("\t"+b)
+        print()
+        
+    
+    """longer = max(len(t_names), len(unmatched_dir_names))
+    
+    for i in range(longer):
+        cola = ''
+        colb = ''
+        if len(t_names) > i:
+            cola = t_names[i]
+        if len(unmatched_dir_names) > i:
+            colb = unmatched_dir_names[i]
+            
+        print(" %s\t\t%s" % (cola,colb))
+    """
+
+
+
+def index_pics():
+    dir_staff = 'cache/picsStaffdir/'        #   peter_howell
+    dir_ilearn = 'cache/picsCanvas/'    #   g00102586
+    dir_badge = 'cache/picsId/2021crop/'       #   102586
+    
+    u1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=list/staffsemester"
+    sss = json.loads(  requests.get(u1).text  )
+    
+    by_goo = {}
+    by_fl = {}
+    
+    count1 = 0
+    count2 = 0
+    for s in sss:
+        myflag = 0
+        user = s['first_name'] + " " + s['last_name']
+        fl = s['first_name'].lower() + "_" + s['last_name'].lower()
+        goo_short = ''
+        if 'conf_goo' in s:
+            goo_short = str(s['conf_goo'])
+        goo_long_p = 'g00' + str(goo_short) + ".png"
+        goo_long_j = 'g00' + str(goo_short) + ".jpg"
+        dir1 = fl + ".jpg"
+        dir2 = fl + "2.jpg"
+        dir3 = fl + "3.jpg"
+        
+        if os.path.isfile( dir_staff + dir1 ):
+            print( "%s \t %s" % (user, dir_staff+dir1))
+            count2 += 1
+            myflag = 1
+        if os.path.isfile( dir_staff + dir2 ):
+            print( "%s \t %s" % (user, dir_staff+dir2))
+            count2 += 1
+            myflag = 1
+        if os.path.isfile( dir_staff + dir3 ):
+            print( "%s \t %s" % (user, dir_staff+dir3))
+            count2 += 1
+            myflag = 1
+
+        if os.path.isfile( dir_ilearn + goo_long_p ):
+            print( "%s \t %s" % (user, dir_ilearn + goo_long_p))
+            #try:
+            #    shutil.copyfile(dir_ilearn + goo_long_p, "cache/picsUpload/"+ goo_long_p)
+            #    print("File copied successfully.")
+            #except Exception as e:
+            #    print("Failed to copy...")
+            count2 += 1
+            myflag = 1
+        if os.path.isfile( dir_ilearn + goo_long_j ):
+            print( "%s \t %s" % (user, dir_ilearn + goo_long_j))
+            #try:
+            #    shutil.copyfile(dir_ilearn + goo_long_j, "cache/picsUpload/"+ goo_long_j)
+            #    print("File copied successfully.")
+            #except Exception as e:
+            #    print("Failed to copy...")
+            count2 += 1
+            myflag = 1
+
+        if os.path.isfile( dir_badge + goo_short + '.jpg' ):
+            print( "%s \t %s" % (user, dir_badge + goo_short + '.jpg'))
+            count2 += 1
+            myflag = 1
+
+        count1 += myflag
+        by_goo[ goo_short ] = s
+        by_fl[fl] = s
+    print("Found pics for %i users, a total of %s pics" % (count1,count2))
+    
+
+
+def cmtes():
+    ii = codecs.open('cache/committees-survey.csv','r','utf-8').readlines()
+    
+    ii = [ x.split(',') for x in ii ] 
+    
+    print(  json.dumps(ii,indent=2) )
+    
+
+
+sem_to_short = { 'Summer 2021': 'su21', 'Fall 2021':'fa21', 'Winter 2022':'wi22', 'Spring 2022':'sp22', 'Summer 2022':'su22', 'Fall 2022':'fa22' }
+def strip(x): return x.strip()
+
+def esc_comma(x): return re.sub(',','[CMA]',x)
+
+def by_sem(x): return x['sem']
+    
+def parse_schedule():
+    # "Course Code","Start Date","End Date",Term,Delivery,CRN,Status,"Course Name",
+    # 8 "Course Description","Units/Credit hours","Instructor Last Name",
+    # 11 "Instructor First Name",Campus/College,"Meeting Days and Times",
+    # 14 "Pass/No Pass available?","Class Capacity","Available Seats","Waitlist Capacity",
+    # 18 "Current Waitlist Length","Meeting Locations","Course Notes",ZTC 21
+    
+    oo = codecs.open('cache/fa20_section_notes.txt','w','utf-8')
+    pp = codecs.open('cache/fa20_section_summary.txt','w','utf-8')
+    
+    
+    u0 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=get/sections"
+    existing_sections = json.loads( requests.get(u0).text )
+    
+    existing_sections = funcy.group_by(by_sem,existing_sections)
+    by_sem_crn = {}
+    
+    for sem,sects in existing_sections.items():
+        for s in sects:
+            new_key = sem + '_' + s['crn']
+            by_sem_crn[new_key] = s
+            
+    #print(json.dumps(by_sem_crn,indent=2))
+    mt = open('cache/missed_instructors.txt','w')
+
+    teacher_cache = {}
+    count = 0 
+    stopat = 20000
+    
+    u1 = "https://www.gavilan.edu/_files/php/current_schedule.csv"
+    with requests.Session() as s:
+        download = s.get(u1)
+        decoded_content = download.content.decode('utf-8')
+        cr = csv.reader(decoded_content.splitlines(), delimiter=',')
+        my_list = list(cr)
+        #for row in my_list:
+        #    print(row)    
+        for row in my_list:
+            row = list(map(strip,row))
+            row = list(map(esc_comma,row))
+            if row[3] in sem_to_short:
+                row[3] = sem_to_short[row[3]]
+            if row[20]:
+                oo.write("%s - %s \n" % (row[0], row[20]))
+            summary = "%s %s %s %s \t %s %s\t %s" % (row[4], row[11],row[10],row[6], row[5], row[0], row[7]) 
+            pp.write(summary + "\n")
+            
+            # cancelled?
+            status = row[6]
+            if status != "Active": continue
+                
+            # ignore if exists?   TODO check if i need to update it
+            this_sem_crn = row[3] + '_' + row[5]
+            if this_sem_crn in by_sem_crn:
+                print("\t...already uploaded...skipping %s" % this_sem_crn)
+                continue
+            
+            if count >0 and count<stopat:
+                t_name = "%s %s" % (row[11],row[10])
+                if t_name in teacher_cache:
+                    r = {'sched_alias':1, 'id': teacher_cache[t_name]}
+                else:    
+                    u2 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=get/instructor/name"
+                    p2 = {'inst':  t_name }
+                    print("\tSearching for teacher: %s" % t_name )
+                    r2 = requests.post(u2, data=p2)
+                    r = json.loads(r2.text)
+                    if r and 'id' in r: teacher_cache[t_name] = r['id']
+                    
+                    if not r:
+                        print("\tCouldn't locate teacher: %s %s" % (row[11],row[10]))
+                        u2 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=get/instructor/fuzzyname"
+                        p2 = {'inst':  row[11] + " % " + row[10] }
+                        print("\tFuzzy search for teacher: " + row[11] + " % " + row[10])
+                        r2 = requests.post(u2, data=p2)
+                        r = json.loads(r2.text)
+                        if r and 'id' in r: teacher_cache[t_name] = r['id']
+                
+                if r and 'sched_alias' in r:
+                    row[10] = r['id']
+                    #print("\tfound teacher: " + str(r))
+                    payload = { 'cols':'code,start_date,end_date,sem,delivery,crn,status,name,descr,units,teacher_id,days,pnp,location,note,ztc', 
+                                'vals': u','.join( [ row[j] for j in [0,1,2,3,4,5,6,7,8,9,10,13,14,19,20,21] ] ) }
+                    #print(json.dumps(payload,indent=2))
+                    #print()
+                    r3 = requests.post("https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=add/section", params=payload)
+                    result = json.loads(r3.text)
+                    if result and 'err' in result and result['err']:
+                        print("\n*** Problem? --> %s" % result['err'] )
+                        mt.write("*** Problem? --> %s\n" % result['err'] )
+                else:
+                    print("*** Still Couldn't locate teacher: %s %s" % (row[11],row[10]))
+                    mt.write("Couldn't locate teacher: %s %s\n" % (row[11],row[10]))
+                    print()
+            count += 1
+
+
+
+
+# TODO some weird hour offset issue w/ these activities
+
+def cal():
+    from ics import Calendar
+    
+    u1 = "https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=get/sessions"
+    
+    gav_activities = json.loads(  requests.get(u1).text  )
+    g_by_uid = {}
+    
+    for g in gav_activities:
+        print("\t" + str(g['cal_uid']))
+        if g['cal_uid']:
+            g_by_uid[ g['cal_uid'] ] = g
+    
+    for g in gav_activities:
+        pass
+        #print(g)
+    #return
+    print(g_by_uid)
+
+    url = "https://calendar.google.com/calendar/ical/4aq36obt0q5jjr5p82p244qs7c%40group.calendar.google.com/public/basic.ics"
+    
+    # the plwc cal
+    url = "https://calendar.google.com/calendar/ical/if2r74sfiitva2ko9chn2v9qso%40group.calendar.google.com/public/basic.ics"
+    c = Calendar(requests.get(url).text)
+    
+    for e in list(c.timeline):
+        #print(e)
+        #print()
+        print(e.name)
+        #continue
+        if not str(e.uid) in g_by_uid.keys():
+            year = str(e.begin)
+            year = year[:4]
+            if not year == "2021": continue
+            print("Not in conf_sessions db: \n\t%s\n\t%s" % ( e.name, e.begin ))
+            addit = input("Do you want to add it?  (y/n)  ")
+            if addit=='y':
+                payload = { 'title':str(e.name) , 'length': 1, 'starttime':str(e.begin) , 
+                            'desc': str(e.description), 
+                            'type':220, 'location':str(e.location) , 'cal_uid':str(e.uid) }
+                print(json.dumps(payload,indent=2))
+                print()
+                r = requests.post("https://hhh.gavilan.edu/phowell/map/dir_api_tester.php?a=set/newsession", data=payload)
+                print("RESPONSE --> ")
+                print(r.text)
+                
+        #print("\t%s" % e.uid)
+        #print("\t%s\n\t%s\n\t%s\n\t%s\n" % ( str(e.begin), e.description, e.location, str(e.last_modified)))
+    #c
+    # <Calendar with 118 events and 0 todo>
+    #print(c.events)
+    # {<Event 'Visite de "Fab Bike"' begin:2016-06-21T15:00:00+00:00 end:2016-06-21T17:00:00+00:00>,
+    # <Event 'Le lundi de l'embarqué: Adventure in Espressif Non OS SDK edition' begin:2018-02-19T17:00:00+00:00 end:2018-02-19T22:00:00+00:00>,
+    #  ...}
+    #e = list(c.timeline)[0]
+    #print("Event '{}' started {}".format(e.name, e.begin.humanize()))
+    
+    
+def file_renamer():
+    where = 'cache/picsStaffdir/cropped/'
+    ff = os.listdir(where)
+    
+    for F in ff:
+        nn = re.sub("\.jpg$","",F)
+        print("Old name: %s. New name: %s" % (F, nn))
+        os.rename( where+F, where+nn )
+        print("ok")
+    
+    
+    
+    
+
+if __name__ == "__main__":
+
+    options = { 1: ['Print answers to a single survey question',survey_answer] ,  
+                2: ['Collate survey answers',survey_organize] ,  
+                3: ['X list CWE classes',xlist_cwe] ,  
+                4: ['parse committees survey',cmtes] ,  
+                5: ['job titles',job_titles] ,  
+                6: ['fetch calendar events to conf_sessions db',cal] ,  
+                7: ['job titles workings....',job_titles2] ,  
+                8: ['collate all profile pics for db',index_pics] ,  
+                9: ['process schedule csv file from web',parse_schedule] ,  
+                10: ['dumb rename images mistake',file_renamer] ,  
+              }
+    
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
+
+
+
+    ################################
+    if 0:
+        
+    if (0):
+        out = open('cache/badgr.txt','w')
+        resp = requests.post(badgr_target, data = badgr_hd)
+        print ( resp )
+        print ( resp.text )
+        out.write(resp.text)
+        auth = json.loads(resp.text)
+    
+    if (0):
+        auth = json.loads(open('cache/badgr.txt','r').read())
+        print ( auth )
+
+    if (0):
+        mail_test()
+
+    if (0):
+        build_quiz()
+
+    if (0):
+        certificates_gott_build()
+
+    if (0):
+        blueprint_semester()
+    
+
+  
+"""
+
+1       Default Term
+6       Professional Development
+7       Committees
+8       Practice Courses
+9       Miscellaneous
+24      OEI Courses
+27      Prof Dev Drafts
+169     Library
+170     Incompletes
+172     2021 Fall
+171     2021 Summer
+168     2021 Spring
+167     Accreditation
+65      2020 Fall
+64      2020 Summer
+62      2020 Spring
+63      2020 Winter
+61      2019 Fall
+60      2019 Summer
+25      2019 Spring
+26      2019 Winter
+23      2018 Fall
+22      2018 Summer
+21      2018 Spring
+18      2017 Fall
+14      2017 Summer
+15      2017 Early Start Summer
+10      2017 Spring (Practice)
+11      2017 Spring
+
+
+"""
\ No newline at end of file
diff --git a/temp.py b/temp.py
new file mode 100644
index 0000000..52d2a2e
--- /dev/null
+++ b/temp.py
@@ -0,0 +1,52 @@
+"""
+fname = 'cache/teacherdata/activity/G00101483.json'
+import json
+import codecs
+from collections import defaultdict as ddict
+from dateutil.parser import parse as parse_dt
+allact = json.loads( codecs.open(fname,'r','utf-8').read() )
+
+
+unique_urls = set(funcy.pluck('url',allact))
+date_hits = sorted(funcy.pluck('updated_at',allact))
+
+
+date_hits = list(map(parse_dt, date_hits))
+
+dontcare = open('cache/urls_i_dont_care.txt','r').readlines()
+
+dd = ddict(int)
+for k in allact: dd[ k['url'] ] += 1
+
+dits = ddict(int)
+for j in 
+
+urls_by_freq = [ (k, v) for k, v in sorted(ddd.items(), key=lambda item: item[1],reverse=True)]
+top_five = [ (k, v) for k, v in sorted(ddd.items(), key=lambda item: item[1],reverse=True)][:5]
+
+"""
+
+import csv
+
+ilearn_version = csv.reader(open('cache\teacherdata\staff_main_table.csv','r').read())
+
+
+
+old_dir = csv.reader(open('cache/personnel2020_04_12.csv'), delimiter=',')
+dept1_crxn = {r[0]:r[1] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
+dept2_crxn = {r[0]:r[2] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
+title_crxn = {r[0]:r[3] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
+
+newfile = open('cache/dir_new.txt','w')
+
+depts = []
+for r in old_dir:
+    old_dept = r[2]
+    if old_dept in dept1_crxn:
+        new_one = dept1_crxn[old_dept]
+        if dept2_crxn[old_dept]: new_one += '/' + dept2_crxn[old_dept]
+        if title_crxn[old_dept]: new_one += '/' + title_crxn[old_dept]
+        r[2] =  new_one
+    newfile.write('\t'.join(r) + '\n')
+    
+ 
\ No newline at end of file
diff --git a/tempget.py b/tempget.py
new file mode 100644
index 0000000..79193af
--- /dev/null
+++ b/tempget.py
@@ -0,0 +1,136 @@
+#
+#
+#  Fetcher for my otter and pinterest accounts. And whatever else.
+
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.ui import WebDriverWait, Select
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import re
+
+import time
+
+from secrets import banner_url1, banner_url2, GOO, GOO_PIN, otter_login, otter_pw
+
+
+
+# Use Firefox and log in to ssb and get full schedule
+def login():
+    #todo: my data here.... secret
+    url = banner_url2
+    un = GOO
+    pw = GOO_PIN
+    text = ''
+    try:
+        driver = webdriver.Firefox()
+        driver.get(url)
+        driver.find_element_by_id("username").clear()
+        driver.find_element_by_id("username").send_keys(un)
+        driver.find_element_by_name("password").send_keys(pw)
+        driver.find_element_by_name("loginForm").submit()
+        driver.implicitly_wait(5)
+        
+        print(driver.title)
+        
+        driver.find_element_by_link_text("Students").click()
+        driver.implicitly_wait(5)
+        print(driver.title)
+        
+        driver.find_element_by_link_text("Registration").click()
+        driver.implicitly_wait(5)
+        print(driver.title)
+        
+        driver.find_element_by_link_text("Search for Classes").click()
+        driver.implicitly_wait(15)
+        print(driver.title)
+        
+        dd = Select(driver.find_element_by_name("p_term"))
+        if (dd):
+            dd.select_by_visible_text(SEMESTER)
+            driver.find_element_by_xpath("/html/body/div/div[4]/form").submit()
+            driver.implicitly_wait(15)
+            print(driver.title)
+
+        driver.find_element_by_xpath("/html/body/div/div[4]/form/input[18]").click()   
+        driver.implicitly_wait(10)
+        print(driver.title)
+        
+        driver.find_element_by_name("SUB_BTN").click()
+        driver.implicitly_wait(10)
+        print(driver.title)
+        text = driver.page_source
+        
+        
+    except Exception as e:
+        print("Got an exception: ", e)
+    finally:
+        print("")
+        driver.quit()
+    return text
+
+
+
+
+
+def filename_friendly(str):
+    str1 = re.sub(r'\s+','_',str)
+    return "".join([c for c in str1 if c.isalpha() or c.isdigit() or c==' ']).rstrip()
+
+def otter():
+    driver = webdriver.Firefox()
+    driver.get("https://otter.ai/signin")
+    #assert "Python" in driver.title
+    elem = driver.find_element_by_css_selector('#mat-input-0')
+    elem.clear()
+    elem.send_keys(otter_login)
+    elem.send_keys(Keys.RETURN)
+    
+    
+    elem = WebDriverWait(driver, 10).until(
+        EC.presence_of_element_located((By.CSS_SELECTOR, "#mat-input-1")))
+    elem.clear()
+    elem.send_keys(otter_pw)
+    elem.send_keys(Keys.RETURN)
+    time.sleep(5)
+    #driver.implicitly_wait(15)
+    
+    driver.get("https://otter.ai/my-notes")
+    driver.implicitly_wait(10)
+    
+    items = driver.find_elements_by_css_selector('div.__conversation-title')
+    print("I found %i conversations" % len(items))
+    titles = []
+    for i in items:
+        print(i.text)
+        titles.append(i.text)
+    
+    count = len(items)
+    
+    n = 0
+    while n < count:
+        items[n].click()
+        element = WebDriverWait(driver, 15).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR, "div.conversation-detail__content")))
+        
+        date_elem = driver.find_element_by_css_selector('.conversation-detail__title__meta')
+        kw_elem = driver.find_element_by_css_selector('.conversation-detail__title__keywords-list')
+        
+        myfile = filename_friendly(date_elem.text) + '_' + filename_friendly(titles[n]) + '.txt'
+        ff = open('otter/%i.txt' % n, 'w')
+        ff.write("Title: %s\n" % titles[n])
+        ff.write("Keywords: %s\n\n" % kw_elem.text)
+        ff.write(element.text)
+        ff.close()
+        
+        driver.get("https://otter.ai/my-notes")
+        driver.implicitly_wait(10)
+    
+        items = driver.find_elements_by_css_selector('div.__conversation-title')
+        n += 1
+    driver.close()
+    print("OK")
+
+#otter()
+
+print(login() )
\ No newline at end of file
diff --git a/templates.py b/templates.py
new file mode 100644
index 0000000..2413563
--- /dev/null
+++ b/templates.py
@@ -0,0 +1,444 @@
+
+import os, re, codecs
+
+from pipelines import get_doc, get_doc_generic, put_file
+
+# build web pages from fragments
+
+output_type = ".php"    # use php when uploading
+output_type2 = ".html"    # use php when uploading
+
+which_template = "/template.html"
+which_template2 = "/template2.html"
+
+
+masonry_template = """  <div class="masonry-item">
+    <div class="masonry-content">
+      <a href="%%LINK%%">
+      <img src="%%IMGSRC%%" 
+        alt="%%IMGALT%%"></a>
+      <h3 class="masonry-title">%%TITLE%%</h3>
+      <p class="masonry-description">%%CARDBODY%%</p>
+    </div>
+  </div>"""
+  
+def item_to_masonry(item):
+    #  link  title  img  imgalt  desc
+    #print(item)
+    #print(item[1] + "\n" )
+    ix = {'TITLE': "<a href='%s'>%s</a>" % (item[0], item[1]),
+          'IMGSRC': item[2] or 'http://www.gavilan.edu/_files/img/blank.gif',  'IMGALT': item[3],
+          'CARDBODY': item[4], 'LINK': item[0] }
+    output = ''
+    for L in masonry_template.split('\n'):
+        match = re.search(r'%%(\w+)%%',L)
+        if match:
+            tag = match.group(1)
+            #print("Found a tag: %s" % tag)
+            line = re.sub(r'%%\w+%%', ix[tag], L)
+            output += line
+        else:
+            output += L        
+    return output
+    
+    
+def try_untemplate():
+    from bs4 import BeautifulSoup as bs
+    import bs4
+    
+    dir1 = 'C:/Users/peter/Documents/gavilan/www mirror'
+    j = 0
+    j_in_dir = []
+    for x in os.listdir(dir1):
+        if x in ['student','staff']:
+            for xy in os.listdir(dir1+'/'+x):
+                j+= 1
+                print("%i.\t%s" % (j,x+'/'+xy))
+                j_in_dir.append(x+'/'+xy)
+        else:
+            j+= 1
+            print("%i.\t%s" % (j,x))
+            j_in_dir.append(x)
+    
+    dir2 = j_in_dir[int(input("Choose a folder to look in: "))-1]
+    
+    dir = dir1 + '/' + dir2
+    
+    i = 0
+    f_in_dir = []
+    for x in os.listdir(dir):
+        if x.endswith('php'):
+            i+= 1
+            print("%i.\t%s" % (i,x))
+            f_in_dir.append(x)
+    
+    choices = input("Choose inputs. Separate with a space: ")
+    
+    for C in choices.split(" "):
+    
+        #choice = int( input("Choose a page to make into template: ") ) - 1
+        choice = int( C ) - 1
+        print(f_in_dir[choice])
+        
+        raw_html_in = open(dir + "/" + f_in_dir[choice],'r').read()
+        php_sig = '!!!PHP!!!'
+        php_elements = []
+
+        def php_remove(m):
+            php_elements.append(m.group())
+            return php_sig
+
+        def php_add(m):
+            return php_elements.pop(0)
+
+        # Pre-parse HTML to remove all PHP elements
+        html = re.sub(r'<\?php.*?\?>', php_remove, raw_html_in, flags=re.S+re.M)
+        
+        # Try just poppin the first php tag. We probably leave it behind...
+        php_elements.pop(0)
+
+        bb = bs(html,'html.parser')
+        
+        if not os.path.isdir(dir + '/template'):
+            os.mkdir(dir + '/template',0o777)
+            
+        output_f = '.'.join(f_in_dir[choice].split('.')[:-1]) + '.html'
+        output = open( dir + '/template/' + output_f, 'w', encoding='utf-8')
+        
+        b = bb.find(id='breadcrumbs').get_text()
+        b = re.sub(r'\s+',' ',b)
+        parts = b.split(' > ')
+        b = parts[-1]
+        
+        c = bb.find('h1',class_='page-heading').get_text()
+        
+        a = bb.find('article')
+        a.div.extract()   # the first div has the h1 header
+        
+        a_out = ""
+        for ea in a.contents:
+            try: 
+                a_out +=  ea.prettify(formatter="html")
+            except:
+                if type(ea) == bs4.element.Comment:
+                    a_out += "<!-- %s -->\n" % ea.string
+                else:      
+                    a_out += ea.string + "\n"
+        
+        # some article cleanup
+        a_out = re.sub( r'\n{3,}','\n\n',a_out)
+        a_out = re.sub( r'(&#160;)+',' ',a_out)
+        
+        a_out = re.sub(php_sig, php_add, a_out)
+        
+        print("breadcrumb: %s" % b)
+        print("\n\ntitle: %s\n" % c)
+        #print("\n\narticle: %s" % a_out.strip())
+        
+        output.write("BREADCRUMB=%s\n" % b)
+        output.write("TITLE=%s\n" % c)
+        output.write("ARTICLE=%s" % a_out)
+        output.close()
+
+
+def do_template(temp,source,side):
+    subs = {'BANNER':'http://www.gavilan.edu/_files/img/blank.gif', 'SIDEBAR':''.join(side),}
+    state = 0
+    items = ""
+    output = ""
+    for L in source:
+        if state: 
+            if re.search('%%ITEMS%%',L):
+                subs['ARTICLE'] += items
+            else:
+                subs['ARTICLE'] += L
+        else:
+            parts = L.split('=',1)
+            if parts[0] == 'ITEM':
+                i_parts = parts[1].split('|')
+                items += item_to_masonry(i_parts) + "\n"
+            if parts[0] == 'ARTICLE': 
+                subs['ARTICLE'] = ""
+                state = 1
+            else: subs[parts[0].strip()] = parts[1].strip()
+    #subs['ITEMS'] = items
+    #print("Building page with this: " + str(subs))
+
+    for L in temp:
+        if len(L)<200:
+            match = re.search(r'%%(\w+)%%',L)
+            if match:
+                tag = match.group(1)
+                line = re.sub(r'%%\w+%%', subs[tag], L)
+                output += line
+            else:
+                output += L
+        else:
+            output += L
+    return output
+
+def remove_filetype(f):
+    parts = f.split(r'.')
+    return '.'.join(parts[:-1])
+
+def make():
+    dir1 = 'C:/Users/peter/Documents/gavilan/www mirror'
+    j = 0
+    j_in_dir = []
+    for x in os.listdir(dir1):
+        if x in ['student','staff']:
+            for xy in os.listdir(dir1+'/'+x):
+                j += 1
+                print("%i.\t%s" % (j,x+'/'+xy))
+                j_in_dir.append(x+'/'+xy)
+        else:
+            j+= 1
+            print("%i.\t%s" % (j,x))
+            j_in_dir.append(x)
+    dir2 = j_in_dir[int(input("Choose a folder to look in: "))-1]
+    in_dir = dir1 + '/' + dir2
+
+    #in_dir = r"C:/Users/peter/Documents/gavilan/www mirror/finaid_2019"
+    
+    # how many slashes? Whats the depth?  Any more than zero, start adding ../ s.
+    depth = dir2.count("/")
+    print("Depth is %i\n\n" % depth)
+    
+    src = in_dir + r"/template/"
+    
+    sidebar = ""
+    template = dir1 + which_template
+    if depth:
+        template = dir1 + which_template2
+    pages = []
+    for F in os.listdir(src):
+        if re.search(r'sidebar',F):
+            sidebar = F
+        #elif re.search(r'template',F):
+        #    template = F
+        elif F.endswith('.html'):
+            pages.append(F)
+    print("Template: %s\nSidebar: %s\nPages: %s" % (template,sidebar,str(pages)))
+    
+    
+    template_text = open(template,'r').readlines()
+    side_txt = open(src + sidebar, 'r').readlines()
+    for P in pages:
+        in1_text = open(src + P, 'r').readlines()    
+        out_file = open(in_dir + "/" + remove_filetype(P) + output_type , 'w')
+        out_file2 = open(in_dir + "/" + remove_filetype(P) + output_type2 , 'w')
+        print(P)
+        out_file.write(  do_template( template_text, in1_text, side_txt) )
+        out_file.close()
+        out_file2.write(  do_template( template_text, in1_text, side_txt) )
+        out_file2.close()
+        
+    
+def txt_2_table():
+    input = open("C:/Users/peter/Documents/gavilan/www mirror/counseling_2019/who_call.txt",'r').readlines()
+    output = ''
+    state = ''
+    
+    for L in input:
+        parts = L.split(r' ')
+        
+        if state=='begintable':  state= 'intable'
+        
+        if state=='': output += "<p>"
+        
+        if state=='intable':                                # in a table and a line is beginning
+            output += "<tr><td>"
+            
+        for P in parts:
+            P = P.strip()
+            print(P)
+            if P=='NT':
+                if state=='intable':
+                    output += "</td></tr></table>\n"
+                output += "<br /><table><tr><td>\n"
+                state = 'begintable'
+                continue
+            
+            elif P=='ET':
+                output += '</td></tr></table>\n'
+                state = ''
+                continue
+                
+            elif P=='|':                                    # between cells
+                output += "</td><td>"
+                continue
+                
+            output += " " + P                               # the normal case - input to output
+            
+        if state=='intable':                                # in a table and a line just ended
+            output += "</td></tr>\n"
+            
+        if state=='begintable':                                # in a table and the header just ended
+            output += "</td></tr>\n"
+            state = 'intable'
+        
+        if state=='': output += "</p>"
+        
+    output = open("C:/Users/peter/Documents/gavilan/www mirror/counseling_2019/who_call_out.txt.html",'w').write(output)
+            
+
+# https://docs.google.com/document/d/1Jw3rSGxuCkujMLrm-5p_zxSzCQavfwo_7Esthjzg0rQ/edit?usp=sharing
+
+def studenttech_faq():
+    """f = "../www mirror/student/online/template/tech_faq.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    for L in lines:
+        output.write(L )
+        if re.search('<!--START-->',L):
+            break
+        
+    output.write( get_doc('1Jw3rSGxuCkujMLrm-5p_zxSzCQavfwo_7Esthjzg0rQ', 1) )"""
+    codecs.open('qanda_student/public/questions.json','w','utf-8').write( \
+        get_doc_generic('1Jw3rSGxuCkujMLrm-5p_zxSzCQavfwo_7Esthjzg0rQ', bracket=0,verbose=0))
+    put_file('/gavilan.edu/student/', 'qanda_student/public/', 'questions.json')
+    print("I uploaded the questions, but remember to do the images too if they changed.")
+
+# https://docs.google.com/document/d/1tI_b-q75Lzu25HcA0GCx9bGfUt9ccM8m2YrrioDFZcA/edit?usp=sharing
+def de_faq():
+    """f = "cache/faq_template.html"
+    input = codecs.open(f,'r','utf-8')
+    lines = input.readlines()
+    input.close()
+    
+    output = codecs.open('cache/de_teach_faq.html','w','utf-8')
+    for L in lines:
+        output.write(L )
+        if re.search('<!-- BEGIN -->',L):
+            output.write( get_doc_generic('1tI_b-q75Lzu25HcA0GCx9bGfUt9ccM8m2YrrioDFZcA', bracket=0,verbose=1))
+    """
+    codecs.open('qanda/public/questions.json','w','utf-8').write( \
+        get_doc_generic('1tI_b-q75Lzu25HcA0GCx9bGfUt9ccM8m2YrrioDFZcA', bracket=0,verbose=0))
+    put_file('/gavilan.edu/staff/tlc/canvas_help/', 'qanda/public/', 'questions.json')
+    print("I uploaded the questions, but remember to do the images too if they changed.")
+    
+def degwork_faq():
+    f = "../www mirror/counseling_2019/template/degreeworks.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    for L in lines:
+        output.write(L )
+        if re.search('<!--START-->',L):
+            break
+        
+    output.write( '<meta charset="utf-8" />\n' + get_doc('1ctmPkWwrIJ1oxlj8Z8UXYjijUzMW2VxnsVDSE1KfKME') )
+
+def vrc_faq():
+    # https://docs.google.com/document/d/1anAmnSusL-lTSAz-E4lcjlzq1CA8YJyUfUHxnKgmJEo/edit?usp=sharing
+    f = "../www mirror/student/veterans/template/faq.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    for L in lines:
+        output.write(L )
+        if re.search('<!--START-->',L):
+            break
+        
+    output.write( '<meta charset="utf-8" />\n' + get_doc('1anAmnSusL-lTSAz-E4lcjlzq1CA8YJyUfUHxnKgmJEo',verbose=1) )
+
+def counseling_faq():
+    f = "../www mirror/counseling_2019/template/faq.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    for L in lines[0:3]:
+        output.write(L)
+        
+    output.write( get_doc('101iOplZearjv955FX2FX9AM6bUnkcryo7BShKuzE9tI') )
+
+def finaid_faq():
+    f = "../www mirror/finaid_2019/template/faq.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    i = 0
+    for L in lines[0:3]:
+        #print("%i, %s" % (i,L))
+        output.write(L)
+        i+=1
+        
+    output.write( get_doc('1-FarjfyzZceezdSBXDHpP2cF_vaa9Qx6HvnIqwipmA4') )
+    
+def coun_loc():
+    f = "../www mirror/counseling_2019/template/location.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    i = 0
+    for L in lines[0:3]:
+        #print("%i, %s" % (i,L))
+        output.write(L)
+        i+=1
+        
+    output.write( get_doc('1hxQZ9iXMWvQQtaoVlRgor9v4pdqdshksjeHD2Z4E6tg') )
+    
+def tutor_faq():
+    f = "../www mirror/student/learningcommons/template/faq.html"
+    input = open(f,'r')
+    lines = input.readlines()
+    input.close()
+    
+    output = open(f,'w')
+    i = 0
+    for L in lines[0:3]:
+        #print("%i, %s" % (i,L))
+        output.write(L)
+        i+=1
+        
+    output.write( get_doc('1gCYmGOanQ2rnd-Az2HWFjYErBm_4tp_RuJs6a7MkYrE',1) )
+    
+def test_repl():
+    from interactive import MyRepl
+    
+    c = MyRepl()
+    c.set_my_dict( { "Peter": "thats me", "Mike": "a VP", "Pablo": "isn't here", "Mary": "Far away" })
+    c.inputloop()
+    
+    
+        
+if __name__ == "__main__":
+    
+    print ('')
+    options = { 1: ['Build www pages', make] ,
+                2: ['De-template an existing page', try_untemplate], 
+                3: ['Text to table', txt_2_table], 
+                4: ['Pull the Counseling FAQ from gdocs', counseling_faq] , 
+                5: ['Pull the DegreeWorks FAQ from gdocs', degwork_faq] , 
+                6: ['Pull the Finaid FAQ from gdocs', finaid_faq] , 
+                7: ['Pull the Tutoring FAQ from gdocs', tutor_faq] , 
+                8: ['Pull the Counseling Location page from gdocs', coun_loc] , 
+                9: ['Pull the student tech faq page from gdocs', studenttech_faq] , 
+                10: ['Pull the DE faq page from gdocs', de_faq] , 
+                11: ['Pull the VRC faq page from gdocs', vrc_faq] , 
+                12: ['Test a REPL', test_repl ],
+              }
+    
+    for key in options:
+        print(str(key) + '.\t' + options[key][0])
+    
+    print('')
+    resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+    
+            
\ No newline at end of file
diff --git a/templates/dir.html b/templates/dir.html
new file mode 100644
index 0000000..b46d603
--- /dev/null
+++ b/templates/dir.html
@@ -0,0 +1,171 @@
+<style>
+body, select { font-family: arial; font-size: 0.75rem; }
+.line span { display: inline-block; width: 16%; }
+div.line { border-bottom: 1px solid lightgrey; margin-bottom: 4px; }
+th { text-align:left; border-bottom: 1px solid lightgrey; cursor: pointer; }
+.linehead span { font-weight: bold; }
+</style>
+<!--<script src='/data/gui/lib/lodash.js'></script>-->
+<script src='/data/gui/lib/underscore-min.js'></script>
+<script src='/data/gui/lib/vue.js'></script>
+<script src='/data/gui/lib/jacks.min.js'></script>              <!-- https://github.com/jccazeaux/jacks -->
+<script src='/data/gui/lib/vue-good-table.min.js'></script> 
+<script>
+    var margin = {top: 20, right: 20, bottom: 70, left: 40}
+    var appdata = { users:[], depts:[], titles:[],       
+        columns: [
+        {
+          label: ' ',
+          field: 'picture',
+        },
+        {
+          label: 'Last',
+          field: 'last_name',
+        },
+        {
+          label: 'First',
+          field: 'first_name',
+        },
+        {
+          label: 'Department',
+          field: 'd1',
+        },
+        {
+          label: 'Phone',
+          field: 'phone_number',
+        },
+        {
+          label: 'Email',
+          field: 'email',
+        },
+        {
+          label: 'Room',
+          field: 'room',
+        },
+      ],
+      }
+    Vue.config.devtools = true;
+
+</script>
+<div id="staff_dir"> 
+
+    <vue-good-table
+      :columns="columns"
+      :rows="users" 
+      :search-options="{
+        enabled: true}">
+  </vue-good-table>
+
+        <br /><br />
+  <!--<div class="line linehead">
+  <span>Name</span>
+  <span>Title</span>
+  <span>Department</span>
+  <span>Old Department</span>
+  <span>Email</span>
+  <span>Phone</span>
+  </div><user-line v-for="p,i in users" v-bind:user="p"
+  v-bind:key="'usr_'+i"></user-line>-->
+</div>
+
+<script>
+
+Vue.component('popup-field', {
+    props: ['fieldname', 'values',   // an array of objs: {'text':'t', 'value':'v' }
+            'init_val', 'userid'],
+    data: function() { return {'selected':this.init_val, 'a':'b'} },
+    delimiters: ['[[', ']]'],
+    methods: {
+        new_val: function() { 
+            jacks().get("/api/update_pers_dept/" + this.userid + "/" + this.selected)
+                .header("Accepts", "application/json")
+                .send( function(resp) { console.log("Tried to update department.")
+                                        console.log(resp) } )
+                  },
+    },
+    template: `<select v-model="selected" @change="new_val()" >
+        <option v-for="k in values" :value="k.id">[[ k.name ]]</option>
+    </select>`
+})
+
+Vue.component('popup-title-field', {
+    props: ['fieldname', 'values', 
+            'init_val', 'userid'],
+    data: function() { return {'selected':this.init_val, 'a':'b'} },
+    delimiters: ['[[', ']]'],
+    methods: {
+        new_val: function() { 
+            jacks().get("/api/update_pers_title/" + this.userid + "/" + this.selected)
+                .header("Accepts", "application/json")
+                .send( function(resp) { console.log("Tried to update title.")
+                                        console.log(resp) } )
+                  },
+    },
+    template: `<select v-model="selected" @change="new_val()" >
+        <option v-for="k in values" :value="k.id">[[ k.name ]]</option>
+    </select>`
+})
+
+Vue.component('user-line', {
+  props: [ 'user', ],
+  data: function () {
+      return { "a": "a", "b": "b" }
+  },
+  delimiters: ['[[', ']]'],
+  methods: {
+      update2: _.debounce( function(column, row, newval) {         
+        jacks().get("/api/update_pers_dept/" + row + "/" + newval)
+            .header("Accepts", "application/json")
+            .send( function(resp) { console.log("Tried to update department.")
+                                    console.log(resp) } )
+      }, 300),
+      update: function(column, row, newval) {         }
+  },
+  template: `<div class="line">
+  <span>[[ user.first_name ]] [[ user.last_name ]]</span>
+  <span>
+    <popup-title-field :fieldname="'title'" :values="this.$root.titles"
+        :init_val="user.titleid" :userid="user.id" ></popup-title-field> 
+  </span>
+  <span>
+    <popup-field :fieldname="'dept1'" :values="this.$root.depts"
+        :init_val="user.dept1" :userid="user.id" ></popup-field> 
+  </span>
+  <span>[[ user.old_dept ]]</span>
+  <span>[[ user.email ]]</span>
+  <span>[[ user.phone_number ]]</span>
+</div>`
+}) 
+
+
+
+var vm = new Vue({
+    data: appdata, 
+    el: '#staff_dir',
+    delimiters: ['[[', ']]'],
+    methods: { 
+      pretty_session: function(ses) { },
+	  remove: function (event) { console.log(event); console.log(this);  
+      }},
+    computed: {
+        }
+}); 
+
+jacks().get("/api/personnel_fetch")
+    .header("Accepts", "application/json")
+    .send( function(resp) { vm.users = JSON.parse(resp.response)
+                            console.log(vm.users) } )
+
+jacks().get("/api/personnel_meta")
+    .header("Accepts", "application/json")
+    .send( function(resp) { var meta = JSON.parse(resp.response)
+                            vm.depts =  meta.depts
+                            vm.titles = meta.titles
+                            console.log(vm.depts) } )
+
+console.log(filename)
+    
+</script>
+
+
+
diff --git a/templates/hello.html b/templates/hello.html
new file mode 100644
index 0000000..f540101
--- /dev/null
+++ b/templates/hello.html
@@ -0,0 +1,112 @@
+<!doctype html>
+<title>Welcome To Gavilan College</title>
+{% if name %}
+<h1>iLearn Hits for: {{ name }}</h1>
+<p>  <b>canvas id:</b> {{ id }}
+{% else %}
+<h1>Hello, World!</h1>
+{% endif %}
+
+<a href="/sd">Shutdown</a>
+    <script src='/data/gui/lib/d3.js'></script>
+    <script src='/data/gui/lib/lodash.js'></script>
+    <script>
+    var margin = {top: 20, right: 20, bottom: 70, left: 40},
+
+    filename= "/data/users/logs/{% if id %}{{ id }}{% else %}241{% endif %}.csv",
+
+    width = 600 - margin.left - margin.right,
+    height = 300 - margin.top - margin.bottom;
+    var	parseDate = d3.timeParse("%Y-%m-%dT%H:%M:%S%Z")     /*  2020-07-13T12:44:54Z,2020-07-13T12:44:54Z  */
+    var	parseDate2 = d3.timeParse("%Y-%m-%d")     /*  2020-07-13  */
+
+    // Parse the date / time
+
+    var x = d3.scaleTime().range([0, width]);
+    var y = d3.scaleLinear().range([height, 0]);
+
+    var svg = d3.select("body").append("svg")
+        .attr("width", width + margin.left + margin.right)
+        .attr("height", height + margin.top + margin.bottom)
+      .append("g")
+        .attr("transform",
+              "translate(" + margin.left + "," + margin.top + ")");
+    
+    d3.csv(filename).then(function(data) {
+        data.forEach(function(d) {
+            d.day = d.created_at.substring(0,10) 
+            d.date = parseDate(d.created_at);
+            d.value = 1;
+        });
+      
+        var by_day = d3.nest().key( function(d) { return d.day }).rollup(function(v){return v.length;})
+            .entries(data);
+        by_day.forEach(function(d) {
+            d.key = parseDate2(d.key) 
+        });
+        by_day = by_day.filter(function(d) { return d.key > parseDate2("2020-01-01")} )
+      console.log(by_day)
+
+      // Scale the range of the data
+      x.domain(d3.extent(by_day, function(d) { return d.key; }));
+      y.domain([0, d3.max(by_day, function(d) { return d.value; })]);
+
+      // Add the valueline path.
+      /*svg.append("path")
+          .data([by_day])
+          .attr("class", "line")
+          .attr("d", valueline);
+      */
+      // Add the X Axis
+      svg.append("g")
+          .attr("class", "axis")
+          .attr("transform", "translate(0," + height + ")")
+          .call(d3.axisBottom(x)
+                  .tickFormat(d3.timeFormat("%Y-%m-%d")))
+          .selectAll("text")	
+            .style("text-anchor", "end")
+            .attr("dx", "-.8em")
+            .attr("dy", ".15em")
+            .attr("transform", "rotate(-65)");
+
+      // Add the Y Axis
+      svg.append("g")
+          .attr("class", "axis")
+          .call(d3.axisLeft(y));
+
+      /*svg.append("g")
+          .attr("class", "x axis")
+          .attr("transform", "translate(0," + height + ")")
+          .call(xAxis)
+        .selectAll("text")
+          .style("text-anchor", "end")
+          .attr("dx", "-.8em")
+          .attr("dy", "-.55em")
+          .attr("transform", "rotate(-90)" );
+
+      svg.append("g")
+          .attr("class", "y axis")
+          .call(yAxis)
+        .append("text")
+          .attr("transform", "rotate(-90)")
+          .attr("y", 6)
+          .attr("dy", ".71em")
+          .style("text-anchor", "end")
+          .text("Value ($)");
+      */
+      svg.selectAll("bar")
+          .data(by_day)
+        .enter().append("rect")
+          .style("fill", "steelblue")
+          .attr("x", function(d) { return x(d.key); })
+          .attr("width", 10)
+          .attr("y", function(d) { return y(d.value); })
+          .attr("height", function(d) { console.log(height); console.log(d); console.log(height - y(d.value)); return height - y(d.value); });
+
+    });
+    /*d3.csv("/data/users/logs/241.csv").then(function(data) {
+      console.log(data[0]);
+    }); */
+
+
+    </script>
diff --git a/templates/images.html b/templates/images.html
new file mode 100644
index 0000000..d74d505
--- /dev/null
+++ b/templates/images.html
@@ -0,0 +1,134 @@
+<!doctype html>
+<title>Welcome To Gavilan College</title>
+<link  href="/data/gui/cropper.css" rel="stylesheet">
+<script src="/data/gui/cropper.js"></script>
+
+<style>
+    body, select { font-family: arial; font-size: 0.75rem; }
+    .line span { display: inline-block; width: 16%; }
+    div.line { border-bottom: 1px solid lightgrey; margin-bottom: 4px; }
+    .linehead span { font-weight: bold; }
+    .clicky { cursor:pointer; padding:0.3em; }
+    button { padding:0.5em; margin:0.5em;  }
+    #myapp img { max-width:100%; max-height:30rem; }
+    #list { width:30%; display:inline-block; position:absolute; top:0; left:0; height:100%; overflow:scroll; }
+    #ed { width:60%; display:inline-block; position:absolute; top:0; left:30%; height:90%;}
+</style>
+
+<script src='/data/gui/lib/lodash.js'></script>
+<script src='/data/gui/lib/vue-max.js'></script>
+
+
+<div id="pic_editor"> 
+
+<div id="list">
+    <h1>Photo Cropper</h1>
+    <div class="clicky" v-for="ff,i in files_match" v-on:click="swap_image(i)">
+    <span v-if="is_checked(ff)"><b>x </b></span>
+    [[ff.conf_name]]
+    </div>
+    <div class="clicky" v-for="ff,i in files_no_match" v-on:click="swap_image_b(i)"><span v-if="is_checked(ff)"><b>x </b></span> [[ff]]</div>
+</div>
+<div id="ed">
+    <img id='target' src='' />
+    <button v-on:click="save_crop">Make Crop</button>
+</div>
+</div>
+
+
+<script>
+
+{% autoescape false %}
+    var files_match = {{matches}}
+    var files_no_match = {{nomatches}}
+    var staff = {{staff}}
+    var checked = {{checked}}
+{% endautoescape %}
+
+var app = new Vue({ 
+  el: '#pic_editor',
+  data: {  
+    msg: 'hello', active: false,
+    current_file: 0,
+    //current_filename: '',
+    files_match: files_match,
+    files_no_match: files_no_match,
+    checked: checked,
+    picDir: '/data/picsStaffdir/',                         //   '/data/picsId/originals_20211022/'
+    staff: staff,
+    cropper: '',
+    image: '',
+  },
+  delimiters: ['[[', ']]'],
+  watch: {  },
+  methods: { 
+    // current_filename: function() { return this.files_match[this.current_file].filename },
+    current_filename: function() { return this.files_no_match[this.current_file] },
+    is_checked: function(x) { console.log(x); return this.checked.includes(x) },
+    swap_image: function(i) {
+        console.log("Swap: " + i)
+        this.current_file = i
+        this.cropper.replace(this.picDir + this.files_match[i].filename)
+        //this.cropper.replace('/data/picsUpload/' + this.files_match[i].filename)
+    },
+    swap_image_b: function(i) {
+        console.log("Swap b: " + i)
+        // this.current_file = i
+        this.current_file = this.files_no_match[ i ]
+        this.cropper.replace(this.picDir + this.files_no_match[i])
+    },
+    save_crop: function() {
+        var self = this
+        console.log(this.cropper.getData(true))
+        cr = this.cropper.getData(true)
+        
+        
+        fetch('/imagecrop/' + this.current_file + '/' + cr.x + '/' + cr.y + '/' + cr.width + '/' + cr.height
+            + '/' + this.current_file, { method: 'GET' }).then(function (response) {
+
+
+        // fetch('/imagecrop/' + this.current_filename() + '/' + cr.x + '/' + cr.y + '/' + cr.width + '/' + cr.height
+        //     + '/' + this.files_match[ this.current_file ].conf_goo, { method: 'GET' }).then(function (response) {
+
+
+            // The API call was successful!
+            if (response.ok) {
+                response.json().then( function(r2) { 
+                    console.log('I saved it yo')
+                    console.log(r2.text)
+                    self.checked.push(self.current_file)
+                } )
+            } else { return Promise.reject(response) }
+        }).then(function (data) {
+        }).catch(function (err) { console.warn('Something went wrong.', err); });
+    },
+  },
+  computed: {  },
+  mounted: function() { 
+        this.image = document.getElementById('target');
+        this.cropper = new Cropper(this.image, {
+          aspectRatio: 3 / 4,
+          crop(event) { console.log(event) },
+        });
+        /*var self = this;
+        fetch('dir_api_tester.php', { method: 'GET' }).then(function (response) {
+            // The API call was successful!
+            if (response.ok) {
+                response.json().then( function(r2) { 
+                    self.user = r2; 
+                } )
+            } else { return Promise.reject(response) }
+        }).then(function (data) {
+        }).catch(function (err) { console.warn('Something went wrong.', err); });
+        */
+  }
+})
+
+
+
+
+
+</script>
+
+
+
diff --git a/templates/personnel.html b/templates/personnel.html
new file mode 100644
index 0000000..7c3728e
--- /dev/null
+++ b/templates/personnel.html
@@ -0,0 +1,197 @@
+<!doctype html>
+<title>Welcome To Gavilan College</title>
+{% if name %}
+<h1>Editor for: {{ name }}</h1>
+<p>  <b>canvas id:</b> {{ id }}
+{% else %}
+<h1>Hello, World!</h1>
+{% endif %}
+
+<p>
+This is a page. Vue is:   1. set up your data. fetch json of either 1 item or the whole list. 
+</p>
+
+<style>
+body, select { font-family: arial; font-size: 0.75rem; }
+.line span { display: inline-block; width: 16%; }
+div.line { border-bottom: 1px solid lightgrey; margin-bottom: 4px; }
+.linehead span { font-weight: bold; }
+</style>
+<script src='/data/gui/lib/d3.js'></script>
+<script src='/data/gui/lib/lodash.js'></script>
+<script src='/data/gui/lib/vue.js'></script>
+<script src='/data/gui/lib/jacks.min.js'></script>              <!-- https://github.com/jccazeaux/jacks -->
+<script src='/data/gui/lib/zeroupload.min.js'></script>         <!-- https://github.com/jhuckaby/zeroupload  -->
+<script src='/data/gui/lib/vue-good-table.min.js'></script> 
+<script>
+    var margin = {top: 20, right: 20, bottom: 70, left: 40}
+    var filename= "/data/users/logs/{% if id %}{{ id }}{% else %}241{% endif %}.csv"
+    var appdata = { users:[], depts:[], titles:[],       
+        columns: [
+        {
+          label: 'Name',
+          field: 'name',
+        },
+        {
+          label: 'Age',
+          field: 'age',
+          type: 'number',
+        },
+        {
+          label: 'Created On',
+          field: 'createdAt',
+          type: 'date',
+          dateInputFormat: 'yyyy-MM-dd',
+          dateOutputFormat: 'MMM do yy',
+        },
+        {
+          label: 'Percent',
+          field: 'score',
+          type: 'percentage',
+        },
+      ],
+      rows: [
+        { id:1, name:"John", age: 20, createdAt: '2020-09-08', score: 0.03343 },
+        { id:2, name:"Jane", age: 24, createdAt: '2011-10-31', score: 0.03343 },
+        { id:3, name:"Susan", age: 16, createdAt: '2011-10-30', score: 0.03343 },
+        { id:4, name:"Chris", age: 55, createdAt: '2011-10-11', score: 0.03343 },
+        { id:5, name:"Dan", age: 40, createdAt: '2011-10-21', score: 0.03343 },
+        { id:6, name:"John", age: 20, createdAt: '2011-10-31', score: 0.03343 },
+      ], }
+    Vue.config.devtools = true;
+
+</script>
+
+<h3>I'm making a vue app. Again. And I like it.  </h3>
+
+<p>1.1 Make your main div with id, and custom tags in it.</p>
+
+<div id="sample"> 
+
+    <vue-good-table
+      :columns="columns"
+      :rows="rows"></vue-good-table>
+
+        <br /><br />
+  <div class="line linehead">
+  <span>Name</span>
+  <span>Title</span>
+  <span>Department</span>
+  <span>Old Department</span>
+  <span>Email</span>
+  <span>Phone</span>
+  </div><user-line v-for="p,i in users" v-bind:user="p" v-bind:key="'usr_'+i"></user-line>
+</div>
+
+
+<p>2. Make some components</p>
+
+<script>
+
+Vue.component('popup-field', {
+    props: ['fieldname', 'values',   // an array of objs: {'text':'t', 'value':'v' }
+            'init_val', 'userid'],
+    data: function() { return {'selected':this.init_val, 'a':'b'} },
+    delimiters: ['[[', ']]'],
+    methods: {
+        new_val: function() { 
+            jacks().get("/api/update_pers_dept/" + this.userid + "/" + this.selected)
+                .header("Accepts", "application/json")
+                .send( function(resp) { console.log("Tried to update department.")
+                                        console.log(resp) } )
+                  },
+    },
+    template: `<select v-model="selected" @change="new_val()" >
+        <option v-for="k in values" :value="k.id">[[ k.name ]]</option>
+    </select>`
+})
+
+Vue.component('popup-title-field', {
+    props: ['fieldname', 'values', 
+            'init_val', 'userid'],
+    data: function() { return {'selected':this.init_val, 'a':'b'} },
+    delimiters: ['[[', ']]'],
+    methods: {
+        new_val: function() { 
+            jacks().get("/api/update_pers_title/" + this.userid + "/" + this.selected)
+                .header("Accepts", "application/json")
+                .send( function(resp) { console.log("Tried to update title.")
+                                        console.log(resp) } )
+                  },
+    },
+    template: `<select v-model="selected" @change="new_val()" >
+        <option v-for="k in values" :value="k.id">[[ k.name ]]</option>
+    </select>`
+})
+
+
+</script>
+
+
+<p>3. Including the one that corresponds to the html / main div above. </p>
+
+<script>
+
+Vue.component('user-line', {
+  props: [ 'user', ],
+  data: function () {
+      return { "a": "a", "b": "b" }
+  },
+  delimiters: ['[[', ']]'],
+  methods: {
+      update2: _.debounce( function(column, row, newval) {         
+        jacks().get("/api/update_pers_dept/" + row + "/" + newval)
+            .header("Accepts", "application/json")
+            .send( function(resp) { console.log("Tried to update department.")
+                                    console.log(resp) } )
+      }, 300),
+      update: function(column, row, newval) {         }
+  },
+  template: `<div class="line">
+  <span>[[ user.first_name ]] [[ user.last_name ]]</span>
+  <span>
+    <popup-title-field :fieldname="'title'" :values="this.$root.titles"
+        :init_val="user.titleid" :userid="user.id" ></popup-title-field> 
+  </span>
+  <span>
+    <popup-field :fieldname="'dept1'" :values="this.$root.depts"
+        :init_val="user.dept1" :userid="user.id" ></popup-field> 
+  </span>
+  <span>[[ user.old_dept ]]</span>
+  <span>[[ user.email ]]</span>
+  <span>[[ user.phone_number ]]</span>
+</div>`
+}) 
+
+
+
+var vm = new Vue({
+    data: appdata, 
+    el: '#sample',
+    delimiters: ['[[', ']]'],
+    methods: { 
+      pretty_session: function(ses) { },
+	  remove: function (event) { console.log(event); console.log(this);  
+      }},
+    computed: {
+        }
+}); 
+
+jacks().get("/api/personnel_fetch")
+    .header("Accepts", "application/json")
+    .send( function(resp) { vm.users = JSON.parse(resp.response)
+                            console.log(vm.users) } )
+
+jacks().get("/api/personnel_meta")
+    .header("Accepts", "application/json")
+    .send( function(resp) { var meta = JSON.parse(resp.response)
+                            vm.depts =  meta.depts
+                            vm.titles = meta.titles
+                            console.log(vm.depts) } )
+
+console.log(filename)
+    
+</script>
+
+
+
diff --git a/templates/sample-simple-vue-starter.html b/templates/sample-simple-vue-starter.html
new file mode 100644
index 0000000..90fddb0
--- /dev/null
+++ b/templates/sample-simple-vue-starter.html
@@ -0,0 +1,194 @@
+<!doctype html>
+<title>Welcome To Gavilan College</title>
+{% if name %}
+<h1>Editor for: {{ name }}</h1>
+<p>  <b>canvas id:</b> {{ id }}
+{% else %}
+<h1>Hello, World!</h1>
+{% endif %}
+
+<p>
+This is a page. Vue is:   1. set up your data. fetch json of either 1 item or the whole list. 
+</p>
+
+
+<script>
+    Vue.config.devtools = true;
+
+    var appdata = { "allsessions": [],
+                "goo":"", "name":"", "uid":"", 
+                "mysessions": [], 
+                "questions": [],
+                "times":times, "headers":headers, "colors":colors, };
+
+	appdata.goo=x['user']['goo']; appdata.name = x['user']['name']; 
+	appdata.uid = x['user']['id']; appdata.mysessions = x['mysessions'];
+    appdata.other = x['other'];
+    appdata.questions = _.groupBy(x['questions'], function(z) { return z.ses_id; });
+
+</script>
+
+
+<p>1.1 Make your main div with id, and custom tags in it.</p>
+
+<div id="sample">
+  <post-session v-for="ses in orderedSessions" :ses="ses" :qmodel="questions[parseInt(ses.session)]"></post-session>
+</div>
+
+
+<p>2. Make some components</p>
+
+<script>
+
+// A single text style question
+Vue.component('t-question', {
+  props: [ 'qq' ],
+  data: function () {
+    return {
+      "answer": ""
+    }
+  },
+  watch: {
+    "qq.answer": function (val, oldVal) { this.$emit('update', this.qq.qid, val); },
+  },
+  template: `<div><span class="question">{{ qq.question }}</span><br />
+        <textarea v-model="qq.answer"></textarea>
+        <br />
+</div>`
+});
+
+</script>
+
+
+<p>3. Including the one that corresponds to the html / main div above. </p>
+
+<script>
+
+Vue.component('post-session', {
+  props: [ 'ses','qmodel' ],
+  data: function () {
+      var st = "wait";
+      if ( this.passed(this.ses) && this.ses.surveyed ) { st = "ok";}    
+      else if ( this.passed(this.ses) ) { st = "need"; }
+      
+      var is_cert = true;
+      if (this.ses.certified_at == null) { is_cert = false; }
+      
+      return { "is_cert": is_cert, "state": st }
+  },
+  methods: {
+      docert: function(c) { var card = this;
+            var d = 1;
+            if (!c) { d = null; }
+            var save_answer_url = "conf3.php?u=" + card.ses.user + "&cert=" + card.ses.ses_id + "&q=" + d;
+            console.log(save_answer_url);
+            $.ajax({ 
+              url: save_answer_url,         
+              context: document.body
+            }).done(function(r) {
+              card.state = "ok_saved";
+              console.log('ajax returned'); console.log(r);
+            });      },
+      update: _.debounce( function(qid,newval) {         var card = this;
+        var save_answer_url = "conf3.php?u=" + card.ses.user + "&s=" + card.ses.ses_id + "&q=" + qid + "&answer=" + encodeURI(newval);
+        console.log(save_answer_url);
+        
+        $.ajax({
+          url: save_answer_url,         // u   s   q   answer
+          context: document.body
+        }).done(function(r) {
+          card.state = "ok_saved";
+          // console.log('ajax returned'); console.log(r);
+        });
+      }, 300),
+      
+      format_dt: format_dt, 
+      passed: function(s) { 
+        var t = s['starttime'].split(/[- :]/);
+        var d = new Date(t[0], t[1]-1, t[2], t[3], t[4], t[5]);
+        var milli = Date.now() - d;
+        if (milli > 0) { return true; }
+        return false; },
+      needs_svy(s) {
+          if ( this.passed(s) && ! s.surveyed ) { return true;}
+          return false;
+      }
+  },
+  template: `<div id="single_session" class="ses_card">
+    <div class="card_flags">
+    <template v-if="state=='need'">
+        <span class="badge badge-warning">Needs Survey</span>
+        <img src="map/images/icons/alert-circle.svg" alt="" width="22" height="22" title="Click to answer survey">
+    </template>    
+    <template  v-if="state=='ok'">
+        <span class="badge badge-success">OK</span>
+        <img src="map/images/icons/check.svg" alt="" width="22" height="22" title="OK">
+    </template>    
+    <template  v-if="state=='wait'">
+        <span class="badge badge-success">Not yet</span>
+        <img src="map/images/icons/check.svg" alt="" width="22" height="22" title="Wait till after session">
+    </template>    
+    <template  v-if="state=='loading'">
+        <span class="badge badge-success">OK</span>
+        <img src="map/images/spinner.gif" alt="" width="22" height="22" title="OK">
+    </template>    
+    <template  v-if="state=='ok_saved'">
+        <span class="badge badge-success">SAVED</span>
+        <img src="map/images/icons/check.svg" alt="" width="22" height="22" title="Saved">
+    </template>    
+    </div>
+    <h3>Event: {{ ses.title }} </h3> 
+    <span class="minor">Date:</span> {{ format_dt(ses) }}  &nbsp; &nbsp; &nbsp; 
+    <span class="minor">Location:</span> {{ ses.location }} <br />
+    
+    <span class="minor">{{ ses.desc }}</span><br /><br />
+    <template v-for="q in qmodel">
+    <t-question v-if="q.type==1" :qq="q" @update="update"></t-question><br />
+    <n-question v-if="q.type==2" :qq="q" @update="update"></n-question><br />
+    </template><br />
+        <icertify :c="is_cert"></icertify>
+    <!--<template v-if=" ! passed(ses)">
+    </template>
+    <p v-else>This session hasn't occured yet. Check back after it to do the survey.</p>-->
+    
+</div>`
+}) 
+
+
+
+var vm = new Vue({
+  data: appdata, el: '#sample',
+  methods: { 
+      format_dt: format_dt,
+      pretty_track: pretty_track,
+      pretty_session: function(ses) { },
+	  remove: function (event) { console.log(event); console.log(this);  
+		if (g_login) {
+			var re = /\w+_(\d+)/;
+			var match = re.exec(event.target.id);
+			remove(match[1]); }  }, },
+    computed: {
+        orderedSessions: function () {
+            return _.sortBy(this.mysessions, 'starttime'); 
+        },
+        groupedSessions: function () {
+            var only = _.without( this.allsessions, _.findWhere(this.allsessions, { id:"1002" }));
+            return _.sortBy ( _.groupBy(    _.sortBy(only, 'track'),   'starttime' ), function(x) { return x[0].starttime; }); 
+        }
+    }
+}); 
+
+</script>
+
+
+
+<script src='/data/gui/lib/d3.js'></script>
+<script src='/data/gui/lib/lodash.js'></script>
+<script src='/data/gui/lib/vue.js'></script>
+<script>
+    var margin = {top: 20, right: 20, bottom: 70, left: 40}
+
+    var filename= "/data/users/logs/{% if id %}{{ id }}{% else %}241{% endif %}.csv"
+    console.log(filename)
+    
+</script>
diff --git a/timer.py b/timer.py
new file mode 100644
index 0000000..a084727
--- /dev/null
+++ b/timer.py
@@ -0,0 +1,35 @@
+from threading import Timer
+import time, datetime
+
+mm = 18
+
+t = datetime.datetime.today()
+future = datetime.datetime(t.year,t.month,t.day,23,mm)
+diff = future - t
+delta = diff.total_seconds()
+
+print("waiting until 11:%i PM, which is %i seconds from now." % (mm,delta))
+
+    
+    
+    
+ 
+def func(a, b):
+    print("Called function")
+    return a * b
+ 
+# Schedule a timer for 5 seconds
+# We pass arguments 3 and 4
+t = Timer(delta, func, [3, 4])
+ 
+start_time = time.time()
+ 
+# Start the timer
+t.start()
+ 
+end_time = time.time()
+ 
+if end_time - start_time < 5.0:
+    print("Timer will wait for sometime before calling the function")
+else:
+    print("%i seconds already passed. Timer finished calling func()" % mm)
\ No newline at end of file
diff --git a/token.pickle b/token.pickle
new file mode 100644
index 0000000..f58564b
Binary files /dev/null and b/token.pickle differ
diff --git a/users.py b/users.py
new file mode 100644
index 0000000..bf946aa
--- /dev/null
+++ b/users.py
@@ -0,0 +1,2203 @@
+
+import json, codecs, requests, re, pdb, csv, textdistance
+import sys, csv, string, funcy, math, shutil, imghdr, os
+import pytz, time
+import pandas as pd
+import matplotlib.pyplot as plt
+
+#from pandas import TimeGrouper
+from collections import defaultdict
+from pipelines import fetch, fetch_stream, getSemesterSchedule, header, url, FetchError, put_file
+from courses import course_enrollment, users_in_semester
+from localcache import users_this_semester_db, unwanted_req_paths, timeblock_24hr_from_dt, dt_from_24hr_timeblock
+from localcache import teachers_courses_semester
+from util import dept_from_name, most_common_item
+from os.path import exists, getmtime
+
+#from localcache import users_file, com_channel_dim
+
+from dateutil import parser
+from datetime import datetime as dt
+from datetime import timedelta
+import datetime
+
+import queue
+from threading import Thread
+from os import path
+
+# for NLP
+import spacy
+from gensim import corpora, models, similarities, downloader, utils
+from nltk import stem
+
+
+# todo: these constants
+
+#last_4_semesters = 'fall2020 summer2020 spring2020 fall2019'.split(' ')
+#last_4_semesters_ids = [62, 60, 61, 25]
+last_4_semesters = 'spring2021 fall2020 summer2020 spring2020'.split(' ')
+last_4_semesters_ids = [168, 65, 64, 62]
+
+log_default_startdate = "2021-08-23T00:00:00-07:00"
+lds_stamp = parser.parse(log_default_startdate)
+
+recvd_date = '2021-08-23T00:00:00Z'
+num_threads = 25
+max_log_count = 250000
+
+
+##########
+##########
+##########    GETTING USER DATA
+##########
+##########
+
+# All users to a cache file     cache/allusers.json
+def fetchAllUsers():
+
+    if exists('cache/allusers.json'):
+        time = date_time = dt.fromtimestamp( getmtime('cache/allusers.json') )
+        newname = 'cache/allusers_'+ time.strftime('%Y%m%d') + ".json"
+        print("renaming old data file to %s" % newname)
+        os.rename('cache/allusers.json', newname)
+
+
+
+    out1 = codecs.open('cache/allusers.json','w','utf-8')
+    out2 = codecs.open('cache/allusers_ids.json','w','utf-8')
+    all_u = fetch_stream(url + '/api/v1/accounts/1/users?per_page=100', 1)
+    
+    ids = []
+    main_list = []
+    for this_fetch in all_u:
+        for U in this_fetch:
+            ids.append(U['id'])
+            main_list.append(U)
+    
+    ids.sort()
+    out2.write( json.dumps(ids, indent=2))
+    out1.write( json.dumps(main_list, indent=2))
+    out2.close()
+    out1.close()
+    return ids
+    
+
+
+##########
+##########
+##########    TEACHERS LIST AND LOCAL USERS FILE
+##########
+##########
+
+# Fetch teacher users objects from local cache
+def teacherRolesCache():               # I used to be load_users
+    users_raw = json.load(open('cache/ilearn_staff.json','r'))
+    users = {}
+    users_by_id = {}
+    for U in users_raw:
+        users[ U['login_id'] ] = U
+        users_by_id[ U['id'] ] = U
+    return users, users_by_id
+
+
+
+
+
+
+# Outputs:  cache/ilearn_staff.json    
+# Canvas:   Fetch all people with gavilan.edu email address        
+def teacherRolesUpdateCache():    # I used to be get_users
+    t = fetch('/api/v1/accounts/1/users?per_page=500&search_term=%40gavilan.edu&include[]=email')
+    g = open('cache/ilearn_staff.json','w')
+    g.write( json.dumps(t) )
+    g.close()
+    #put_file('/gavilan.edu/staff/flex/2020/','cache/','ilearn_staff.json')
+    print("Wrote to 'cache/ilearn_staff.json'")
+    return teacherRolesCache()
+    
+    
+# Fetch preferred email address for a given user id.   ( Canvas )
+def getEmail(user_id):
+    results = fetch("/api/v1/users/" + str(user_id) + "/communication_channels")
+    for r in results:
+        if r['type']=='email':
+            return r['address']
+    return ''
+
+  
+##########
+##########
+##########    TEACHERS AND OTHER STAFF
+##########
+##########
+#
+# Gather all my info, CRM style, in the folder teacherdata
+#
+# 
+# Typical actions: For everyone with a teacher role:
+# - What are the courses they taught for the last X semesters?
+# - What's their activity level each semester?
+# - Which of those courses are Online, Hybrid or Face2face?
+#     + column for each semester:  OHLOHL
+# - How many online classes have they taught in the past?
+# - Are they brand new, or brand new online?# further...
+# - what's their department?
+# - what's their badges and 'tech level?'
+# - 
+
+
+# All teachers in a particular course
+def getAllTeachers(course_id=59):  # a list 
+    qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
+    t = url + qry
+    while(t): t = fetch(t)
+#
+def classType(t):
+    if t == 'lecture': return 'L'
+    if t == 'online': return 'O'
+    if t == 'hours': return 'R'
+    if t == 'lab': return 'A'
+    if t == 'hybrid': return 'H'
+    else: return 'L'                   # todo: fix bug in schedule parser so non-online classes have a type field
+    
+def my_blank_string(): return "no data"
+def my_blank_dict(): return {'name':'NoName','email':'noemail@gavilan.edu'}
+def my_empty_dict(): return defaultdict(my_blank_string)
+
+def get_email_from_rec(name,name_to_record):
+    #print "Looking up: " + name
+    try:
+        return name_to_record[name]['email']
+    except Exception as e:
+        print("Missing Teacher %s" % name)
+        return 'noemail@gavilan.edu'
+
+
+
+
+# Pull the staff directory on the webpage. Convert to pandas dataframe
+def staff_dir(get_fresh=False):
+    """
+    if get_fresh:
+        url = "http://www.gavilan.edu/staff/dir.php"
+        regex = "var\slist=(\[.*\]);"
+        response = requests.get(url).text
+        m = re.search(regex,response)
+        if m: 
+            output = '{"staff":' + m.group(1) + '}'
+            of = open('cache/teacherdata/staff_dir.json','w')
+            of.write(output)
+            js = json.loads(output)
+            df = pd.DataFrame(js['staff'])
+            return df
+            print("Wrote cache/teacherdata/staff_dir.json")
+        else:
+            print("Failed on staff directory scrape")
+            return ''
+    else:
+        input = json.loads(open('cache/teacherdata/staff_dir.json','r').read())
+        df = pd.DataFrame(input['staff'])
+    return df
+    """
+    
+    # TODO lol get fresh again... 
+    
+    old_dir = csv.reader(open('cache/personnel2020_04_12.csv'), delimiter=',')
+    dept1_crxn = {r[0]:r[1] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
+    dept2_crxn = {r[0]:r[2] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
+    title_crxn = {r[0]:r[3] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
+    revised_dir = [  ]
+    columns = next(old_dir)
+    
+    for r in old_dir:
+        old_dept = r[2]
+        if old_dept in dept1_crxn:
+            new_one = dept1_crxn[old_dept]
+            if dept2_crxn[old_dept]: new_one += '/' + dept2_crxn[old_dept]
+            if title_crxn[old_dept]: new_one += '/' + title_crxn[old_dept]
+            r[2] =  new_one
+        revised_dir.append(r)
+    print(revised_dir)            
+    return pd.DataFrame(revised_dir,columns=columns)    
+
+
+#
+#
+#
+#    ###
+#    ###   TEACHER   CRM   FUNCTIONS
+#    ###
+#
+
+def schedForTeacherOverview(long,short):
+    sem = getSemesterSchedule(short)
+    sem['type'] = sem['type'].apply(classType)
+    #sem['code'] = sem[['code','type']].apply(' '.join,axis=1)
+    sem['sem'] = short
+    sem = sem.drop(['time','loc','name','date','days'],axis=1)  # ,'crn'
+    return sem
+
+
+
+
+# Return a dataframe of the last 4 semester schedules put together
+def oneYearSchedule():
+    sp19 = schedForTeacherOverview('2019spring','sp19')
+    su19 = schedForTeacherOverview('2019summer','su19')
+    fa19 = schedForTeacherOverview('2019fall','fa19')
+    sp20 = schedForTeacherOverview('2020spring','sp20')
+    
+    # The four-semester schedule
+    a = pd.concat([sp19,su19,fa19,sp20], sort=True, ignore_index=True)
+    a = a.drop(['cap','cmp','extra','rem','sec','cred','act'], axis=1)
+    a.to_csv('cache/one_year_schedule.csv')
+    return a
+
+def num_sections_last_year(line):
+    #if not type(line)=='str': return 0
+    parts = line.split(' ')
+    return len(parts)
+
+def sec_type_stats(line):
+    #print(type(line))
+    #if not type(line)=='str': return {'fail':1}
+    #print("in sts: " + str(line))
+    parts = line.split(' ')
+    output = defaultdict(int)
+    for p in parts: output[p] += 1
+    return output
+
+def prct_online(line):
+    d = sec_type_stats(line)
+    #print(d)
+    total = 0
+    my_total = 0
+    for k,v in d.items():
+        total += v
+        if k == 'O': my_total += v
+    return int(100 * ((1.0)*my_total / total))
+
+def prct_lecture(line):
+    #print(line)
+    d = sec_type_stats(line)
+    #if 'fail' in d: return 0
+    total = 0
+    my_total = 0
+    for k,v in d.items():
+        total += v
+        if k == 'L': my_total += v
+    return int(100 * ((1.0)*my_total / total))
+
+
+def prct_hybrid(line):
+    d = sec_type_stats(line)
+    #if 'fail' in d: return 0
+    total = 0
+    my_total = 0
+    for k,v in d.items():
+        total += v
+        if k == 'H': my_total += v
+    return int(100 * ((1.0)*my_total / total))
+
+# Given the names of teachers in last year's schedules, fill in email, etc. from ilearn files
+def teacher_basic_info(sched, from_ilearn, names):
+    bi = from_ilearn    # pd.DataFrame(from_ilearn)
+    bi.rename(columns={'id':'canvasid','login_id':'goo'}, inplace=True)
+    # bi.drop(['name',],axis=1,inplace=True)
+    
+    #print(bi)
+    #input('xx')
+    
+    sp20 = schedForTeacherOverview('2020spring','sp20')
+    
+    
+    codes_sp20 = sp20.groupby('teacher')['code'].apply( lambda x: ' '.join(funcy.distinct(x)) )
+    crns_sp20 = sp20.groupby('teacher')['crn'].apply( lambda x: ' '.join( map( str, funcy.distinct(x))) )
+    codes_sp20.rename(columns={'code':'sp20code'}, inplace=True)
+    codes_sp20.to_csv('cache/trash/codes_sp20.csv',header=True)
+    crns_sp20.rename(columns={'crn':'sp20crn'}, inplace=True)
+    crns_sp20.to_csv('cache/trash/crns_sp20.csv',header=True)
+
+                   
+    a = sched.groupby('teacher')['code'].apply( lambda x: ' '.join(funcy.distinct(x)) )
+    a = pd.DataFrame(a)
+    a.reset_index(inplace=True)
+    a['dept'] = a.apply(guessDept,axis=1)
+    print(a)
+    
+    def find_that_name(x):
+        #print(x)
+        if 'teacher' in x: return names(x['teacher'])
+        #print('name not found?')
+        return ''
+    
+    a['ilearn_name'] = a.apply( find_that_name, axis=1) 
+    
+    a.rename(columns={'code':'courses'}, inplace=True)
+    #print(type(a))
+    a.reset_index(inplace=True)
+    
+    a = pd.merge(a,codes_sp20.rename('sp20courses'), on='teacher')
+    a = pd.merge(a,crns_sp20.rename('sp20crns'), on='teacher')
+    a.to_csv('cache/trash/sched_w_sp20.csv',header=True)
+    print(a)
+    
+    a['canvasid'] = a['teacher'].map(names)
+    #print(a)
+    c = pd.merge(bi, a, left_on='name', right_on='ilearn_name', how='outer')
+    c.to_csv('cache/trash/basic.csv',header=True)
+    #print(c)
+    return c
+
+
+# what percentage of their sections were online / hybrid /lecture ?
+# Consumes:  output/semesters/fa19_sched.json and etc for 1 year 
+# Outputs:   cache/teacher_by_semester.csv, 
+def teacherModalityHistory(sched=[],names=[]):
+    if not len(sched): 
+        sched = oneYearSchedule()
+        names = match_username()
+    
+    # How many classes a teacher taught lect/online/hybrid/hours
+    sec_type = sched.groupby(['teacher','sem'])['type'].apply(' '.join)
+    sec_type.to_csv('cache/teacherdata/teacher_by_semester.csv',header=True)       
+                                                                ## THIS IS THE LIST of how many
+                                                                ## lecture, hybrid, online they've taught   
+    
+    #sec_type = pd.read_csv('cache/teacherdata/teacher_by_semester.csv')
+    
+    sec_grp = sec_type.groupby('teacher').aggregate( ' '.join )
+    #sec_grp.to_csv('cache/trash/sec_grp_3.csv',header=True)
+    
+    #sec_grp = sec_grp.iloc[1:]   ## I'm seeing bad items on the first 2
+    #sec_grp.drop(index='teacher')
+    #sec_grp.to_csv('cache/trash/sec_grp_0.csv',header=True)
+    
+    #
+    sec_grp = pd.DataFrame(sec_grp)
+    #print(type(sec_grp))
+    sec_grp['prct_online']  = sec_grp['type'].map(prct_online) 
+    
+    sec_grp['prct_lecture'] = sec_grp['type'].map(prct_lecture) 
+    sec_grp['prct_hybrid']  = sec_grp['type'].map(prct_hybrid) 
+    sec_grp['num_sections_last_year'] = sec_grp['type'].map(num_sections_last_year) 
+    sec_grp.drop('type',axis=1,inplace=True)
+    sec_grp.reset_index(inplace=True)
+    sec_grp.to_csv('cache/teacherdata/modality_history.csv')
+    return sec_grp
+    
+    
+
+def teacherCourseHistory(a,names):
+    pass
+    # actually not using this. moved to _basic_info
+    
+    # YEEEAH
+    sched = a.groupby(['teacher','code'])
+    #for name,group in sched:
+    #    print(name)
+    #print(sched.count())
+    return
+    a['name'] = a.apply(lambda x: records_by_sname[x['teacher']]['name'],axis=1)
+    a['email'] = a.apply(lambda x: records_by_sname[x['teacher']]['email'],axis=1)
+    a.sort_values(by=['dept','teacher','codenum'],inplace=True)
+    a = a.drop(['teacher'],axis=1)  
+    a.to_csv('cache/teacherdata/courses_taught.csv')
+    
+    return a
+    """
+    d = a.groupby(['teacher'])                     # ,'dept','codenum','codeletter'
+    
+    out1 = open('teacherdata/courses_taught.csv','w')  
+    by_dept = {}                                 # x todo: sort by dept also
+    for name, group in d:
+        #print name
+        if re.search(r'^\d+',name) or name=='TBA':
+            print("Skipping weird name: ", name)
+            continue
+        rec = {'email':'xx'}
+        try:
+            rec = records_by_sname[name]
+            #print rec
+        except Exception as e:
+            print("Missing Teacher %s" % name)
+            continue
+        out1.write(name+"\t"+rec['email'])
+        s = set()
+        #print group
+        for idx,r in group.iterrows():
+            s.add( str(r[1]) + str(r[2]) + str(r[3]))
+        for clas in sorted(s):
+            d = dept_from_name(clas)
+            if d in by_dept:
+                if name in by_dept[d]:
+                    by_dept[d][name].append(clas)
+                else:
+                    by_dept[d][name] = [ clas, ]
+            else:
+                by_dept[d] = { name: [ clas, ] }
+                
+            out1.write("\n\t"+str(clas))
+        out1.write("\n")
+    out1.write( json.dumps(by_dept,indent=2))"""
+
+
+
+# Consumes:  output/semesters/fa19_sched.json and etc for 1 year 
+# Outputs:   cache/course_teacher_combos.csv, 
+def teacherSharedCourses(a=[]):
+    if not len(a): a = oneYearSchedule()
+    
+    # List of classes. Group by teacher/format. Shows who has historically
+    # taught a class and who teaches it most often.
+    c = a.drop(['code','partofday','sem','site','type'],axis=1)   #,'dept','codeletter'
+    c = c.groupby(['dept','codenum','codeletter'])  #,'teacher'
+    c = c.aggregate(lambda x: set(x))
+    c.to_csv('teacherdata/course_teacher_combos.csv')      ## THIS is the list of teachers who
+                                                     ## share courses
+    return c
+
+
+
+# Consumes:  output/semesters/fa19_sched.json and etc for 1 year 
+# Outputs:   cache/num_courses_per_dept.csv                 (not teacher_course_oer_deptcount)
+# How many courses in each department were taught in the last year?
+def departmentCountCourses(a=[]):
+    if not len(a): a = oneYearSchedule()
+    
+    tt = a.drop(['code','partofday','sem','site','type'],axis=1)   #,'dept','codeletter'
+
+    records_by_sname = defaultdict(my_empty_dict, match_usernames())
+    tt.drop_duplicates(keep='first',inplace=True)
+    tt['name'] = tt.apply(lambda x: records_by_sname[x['teacher']]['name'],axis=1)
+    tt['email'] = tt.apply(lambda x: records_by_sname[x['teacher']]['email'],axis=1)
+    tt = tt.drop(['teacher'],axis=1)  
+    tt.sort_values(by=['dept','name','codenum'],inplace=True)
+    count = tt['dept'].value_counts()
+    count.to_csv('cache/num_courses_per_dept.csv', header=True)
+
+
+def clean_nonprint(s):
+    return re.sub(f'[^{re.escape(string.printable)}]', '', s)
+
+def read_cmte(names):
+    output = []
+    out2 = defaultdict(list)
+    input = codecs.open('cache/teacherdata/committees_2018_2019.csv','r','utf-8')
+    with input as csvfile:
+        cmtereader = csv.reader(csvfile, delimiter=',', quotechar='"')
+        for row in cmtereader:
+            for R in row:
+                R = R.strip()
+                R = clean_nonprint(R)
+            (fname,lname,cmtes) = row
+            a = re.split(",\s*",cmtes)
+            if len(a)>1:
+                cmtes = a
+            else:
+                cmtes = a
+            
+            name1 = lname + ", " + fname
+            name2 = fname + " " + lname
+            name = name1
+            realname = names(name1)
+            if not realname:
+                realname = names(name2)
+                name = name2
+            if realname:
+                for cmm in cmtes:
+                    output.append( [realname, cmm] )
+                    out2[realname].append(cmm)
+            else:
+                print("committee participant name failed: %s / %s:\t%s" % (name1,name2,str(a)))
+                print(type(name1))
+    #print(out2)            
+    return output,out2
+
+def read_training_records():
+    myinput = open('cache/teacherdata/more_2018_2019_training_attendance.txt','r').readlines()
+    current_sesh = ""
+    ppl_in_sesh = {}
+    all_ppl = set()
+    
+    for L in myinput:
+        L = L.strip()
+        if L:
+            if L.startswith('#'):
+                ma = re.search(r'^\#\s(.*)$',L)
+                if ma:
+                    current_sesh = ma.group(1)
+                else:
+                    print("-- read_training_records: Couldn't find training set? " + L)
+            else:
+                if current_sesh in ppl_in_sesh:
+                    ppl_in_sesh[current_sesh].append(L)
+                else:
+                    ppl_in_sesh[current_sesh] = [ L, ]
+                all_ppl.add(L)
+    if 0:
+        print(ppl_in_sesh)
+        print(all_ppl)
+    
+    # Want to pivot the dict, so key is a name, value is another dict, where k2 is session name, v2 is Y/N
+    d_of_d = defaultdict(dict)
+    
+    for k,v in ppl_in_sesh.items():
+        for user in v:
+            d_of_d[user][k] = 'Y'
+    
+    return d_of_d    
+
+# open a file and mark the people with their ids given. Return a dataframe
+def read_bootcamp1(filename):
+    a = pd.read_csv(filename)
+    #print(a)
+    b = a.loc[:, ['canvas_id','grade','last_activity']]
+    b.rename(columns={'canvas_id':'bc1canvasid','grade':'bootcamp_grade','last_activity':'bootcamp_date'}, inplace=True)
+    #print(b)
+    return b
+
+# open a file and mark the people with their ids given. Return a dataframe
+def read_bootcamp2(filename):
+    a = pd.read_csv(filename)
+    #print(a)
+    b = a.loc[:, ['canvas_id','grade','last_activity']]
+    b.rename(columns={'canvas_id':'bc2canvasid','grade':'bootcamp_progress','last_activity':'bootcamp_date'}, inplace=True)
+    #print(b)
+    return b
+
+
+def not_blank_or_pound(L):
+    if L.startswith("#"): return False
+    L = L.strip()
+    if L == "": return False
+    return True
+
+def temp1(x):
+    #print(x[1])
+    return x[1]
+
+def add_realnames(df,names):    # the surveys. raw name is in 2nd column
+    df['ilearn_name'] = df.apply( lambda x: names(temp1(x),1), axis=1)
+    return df
+
+def compareToughNames(a,b):
+    # search for a in b
+    m = re.search(a, b)
+    if m: return True
+    return False
+        
+
+def compareNames(a,b,verbose=0):
+    if a == b: return True
+        
+    cnDBG = 0
+    try:
+        parts_a = [ W.lower() for W in re.split("[\s,]", a) ]
+        [ x.strip() for x in parts_a ]
+        
+        parts_b = [ W.lower() for W in re.split("[\s,]", b) ]
+        [ x.strip() for x in parts_b ]
+        
+        pa2 = sorted([ parts_a[0], parts_a[-1] ])
+        pb2 = sorted([ parts_b[0], parts_b[-1] ])
+        
+        if pa2 == pb2: 
+            if cnDBG: print("->Match: %s, %s" % (a,b)) 
+            return True
+        if pa2[0] == pb2[0] or pa2[-1] == pb2[-1]:
+            if cnDBG: print("--->Near match: %s" % b) 
+            return False
+    
+    except Exception as e:
+        #print("Problem with compareNames %s , %s" % (a,b))
+        #print(e)
+        return False
+        
+    if len(pa2[0])>3 and len(pb2[0])>3:
+        if pa2[0][0] == pb2[0][0]:
+            if pa2[0][1] == pb2[0][1]:
+                if pa2[0][2] == pb2[0][2]:
+                    if cnDBG: print("===> Near match (first 3): %s, %s, %s, %s" % (a, b, pa2[0], pb2[0])) 
+                    pass
+    
+    b = b.lower()
+    a = a.lower()
+    
+    #if verbose: print("searching: %s / %s" % (a,b))
+    if re.search( b, a):
+        #print("REGEX MATCH: %s | %s" % (a,b))
+        return True
+    if re.search( a, b):
+        #print("REGEX MATCH: %s | %s" % (a,b))
+        return True
+    return False    
+
+def find_ilearn_record(ilearn_records,manual_records, othername,verbose=0):
+    # manual records are ('name':'canvas_id')
+    #print(ilearn_records)
+    if not othername: return ""
+    if type(othername) == type(1.25): return ""
+    #if math.isnan(othername): return False
+        
+    if othername in manual_records:
+        a = funcy.first( funcy.where( ilearn_records, id=int(manual_records[othername]) ))
+        if a:
+            return a['name']
+
+    for x in ilearn_records:
+        #print('f_i_r')
+        #print(othername)
+        #print(x)
+        if compareNames(othername,x['name'],verbose):
+            return x['name']
+    
+    for k,v in manual_records.items():
+        #print(k)
+        #print(othername)
+        #print(type(othername))
+        b = re.search( k, othername)
+        if b:
+            a = funcy.first( funcy.where( ilearn_records, id=int(manual_records[k]) ))
+            if a:
+                return a['name']
+    return ""
+
+
+def manualNamesAndDept():
+    # copied from // getTeachersInfoMain ....   
+    
+    schedule_one_yr = oneYearSchedule()
+    from_ilearn = list(  map(  lambda y: funcy.select_keys( lambda z: z in ['name','id','email','login_id','sortable_name'], y), \
+                    json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) ) )
+    manual_names = manualNames()
+    names_lookup = funcy.partial(find_ilearn_record, from_ilearn, manual_names)
+    teacher_info = teacher_basic_info(schedule_one_yr, from_ilearn, names_lookup)
+    # till here
+    
+    
+    # the staff directory
+    dr = staff_dir(False)
+    print(dr)
+    print(dr.columns)
+    print( dr['department'].unique() )
+    
+    # now to reconcile and combine these....
+    #
+    # we want:
+    #  - alternate names of academic / other depts, with one preferred
+    #  - some people are PT Fac, FT Fac, Director, assistant, spec, and some titles are unknown. 
+    #  - sometimes the hierarchy is of departments, and sometimes of people. try not to confuse that.
+    #
+    
+    
+    # eventually, want to get pics or other info from other sources too, o365, cranium cafe, etc
+    #
+    
+
+
+def manualNames():
+    mm = dict([ x.strip().split(',') for x in \
+            open('cache/teacherdata/teacher_manual_name_lookup.csv','r').readlines()])  
+    mz = {}
+    for k,v in mm.items():
+        mz[k] = v
+        mz[k.lower()] = v
+        parts = k.split(" ")
+        if len(parts)==2:
+            mz[  parts[1] + ", " + parts[0] ] = v
+            mz[  parts[1] + "," + parts[0] ] = v
+    #print(mz)
+    return mz
+
+# given a list of class codes, return the most common (academic) department
+def guessDept(d_list):
+    li = str(d_list.code).split(" ")
+    count = defaultdict(int)
+    #print(str(d_list.code))
+    for i in li:
+        m = re.search(r'^([A-Z]+)$',i)
+        if m:
+            count[m.group(1)] += 1
+    mmax = 0
+    max_L = ''
+    for k,v in count.items():
+        #print("  %s:%i,   " % (k,v), end='')
+        if v > mmax: 
+            mmax = v
+            max_L = k
+    print("")
+    return max_L
+
+"""
+# Faculty Info Plans
+
+
+
+bootcamp_active.csv				Started bootcamp. Remind them to finish it?
+
+bootcamp_passed.csv				Badge'd for BC. Online and Hybrid teachers not on this list need reminding. 
+
+courses_taught.csv				x
+
+course_teacher_combos.csv		Teachers who share the teaching of a course. Courses in common. 
+
+emails_deans+chairs.txt         Just a email list
+
+FA2017 Faculty Survey.csv       Look at answers for video, helpful formats, and comments
+
+faculty_main_info.csv           Has percentage mix of a teachers' online/hybrid/lecture history
+
+historical_shells_used.json     x
+
+SP2019 Faculty Survey.csv       Look at rate tech skills, topics interested in, would add video, and comments
+
+committees 2018 2019.csv       Committees people serve on.
+
+
+
+Not so useful:
+
+teacher_by_semester.csv         precursor to faculty_main_info. Has semesters separated.   
+
+"""
+#
+#
+#
+# Call all the teacher info / CRM gathering stuff
+# Make one big csv file of everything I know about a teacher    
+def getTeachersInfoMain():
+
+    schedule_one_yr = oneYearSchedule()
+    #print(schedule_one_yr)
+    #if input('q to quit ')=='q': return
+    
+    # comes from  teacherRolesUpdateCache  ...  search for  @gavilan.edu in email address
+    from_ilearn = list(  map(  lambda y: funcy.select_keys( lambda z: z in ['name','id','email','login_id','sortable_name'], y), \
+                    json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) ) )
+    #names_from_ilearn = list( [x.lower() for x in map( str, sorted(list(funcy.pluck('name',from_ilearn)))) ] )
+    from_ilearn_df = pd.DataFrame(from_ilearn)
+    
+    
+    manual_names = manualNames()
+    names_lookup = funcy.partial(find_ilearn_record, from_ilearn, manual_names)
+    #print(from_ilearn_df)
+    #if input('q to quit ')=='q': return
+    
+    
+    #print(schedule_one_yr)
+    #print("This is one year schedule.")
+    #input('\npress enter to continue')
+    
+    teacher_info = teacher_basic_info(schedule_one_yr, from_ilearn_df, names_lookup)
+    #print(teacher_info)
+    #input('\nThis is teacher info.\npress enter to continue')
+
+    modality_history = teacherModalityHistory(schedule_one_yr,names_lookup)
+    print(modality_history)
+    #print("This is teacher modality history.")
+    #input('\npress enter to continue')
+
+    
+    master = pd.merge( modality_history, teacher_info, on='teacher', how='outer')
+    print(master)
+    master.to_csv('cache/trash/joined1.csv')
+    print(master.columns)
+    #input('\nThis is Joined 1.\npress enter to continue')
+    
+    wp = read_bootcamp1('cache/teacherdata/bootcamp_passed.csv')
+    #print(wp)
+    master2 = pd.merge( master, wp, left_on='canvasid_x', right_on='bc1canvasid', how='outer')
+    master2.to_csv('cache/trash/joined2.csv')
+    print(master2)
+    print(master2.columns)
+    #input('\nThis is Joined 2.\npress enter to continue')
+    
+    
+    wp = read_bootcamp2('cache/teacherdata/bootcamp_active.csv')
+    master3 = pd.merge( master2, wp, left_on='canvasid_x', right_on='bc2canvasid', how='outer')
+    master3.to_csv('cache/trash/joined3.csv')
+    print(master3)
+    print(master3.columns)
+    #input('\nThis is Joined 3.\npress enter to continue')
+    
+    
+    # THE VIEWS / HISTORY. UPDATE with    get_recent_views()   .... check it for appropriate dates....
+    views = json.loads( codecs.open('cache/teacherdata/activitysummary.json','r','utf-8').read() )
+    vdf = pd.DataFrame.from_dict(views,orient='index',columns=['cid','cname','views','goo','dates','dateviews'])
+    print(vdf)
+    #input('k')
+    
+    #master3.set_index('canvasid_x')
+    master3 = pd.merge(master3, vdf, left_on='canvasid_x', right_on='cid',how='outer')
+    
+    dir_records = pd.DataFrame(staff_dir())
+    dir_records['email'] = dir_records['email'].str.lower()
+    master3['email'] = master3['email'].str.lower()
+    
+    print(dir_records)
+    master3 = pd.merge(master3, dir_records, on='email',how='outer')
+    print(master3)
+    #if input('q to quit ')=='q': return
+    
+    #master3.fillna(0, inplace=True)
+    #master3['views'] = master3['views'].astype(int)
+    #master3['num_sections_last_year'] = master3['num_sections_last_year'].astype(int)
+    
+    
+    #cmte = pd.read_csv('cache/teacherdata/committees_2018_2019.csv')
+    cmte,cmte_by_name = read_cmte(names_lookup)
+    cmte_str_by_name = {}
+    for k in cmte_by_name.keys():
+        #print(k)
+        #print(cmte_by_name[k])
+        cmte_str_by_name[k] = ",".join(cmte_by_name[k])
+    cc = pd.DataFrame.from_dict(cmte_str_by_name,orient='index',columns=['committees'])   # 'teacher',
+    cc.reset_index(inplace=True)
+    master4 = pd.merge(master3, cc, left_on='name', right_on='index', how='outer')
+    master4.to_csv('cache/trash/joined4.csv')
+    
+    master4.drop(['teacher','ilearn_name','canvasid_y','bc1canvasid','bc2canvasid','cid','cname','index_y'],axis=1,inplace=True)
+    
+    # Exclude surveys for now
+    """
+    survey_2017 = pd.read_csv('cache/teacherdata/FA2017 Faculty Survey.csv')
+    survey_2017 = add_realnames(survey_2017,names_lookup)
+    survey_2017.to_csv('cache/trash/survey1.csv')
+    master5 = pd.merge(master4, survey_2017, left_on='name', right_on='ilearn_name', how='left')
+    master5.to_csv('cache/trash/joined5.csv')
+    
+    survey_2019 = pd.read_csv('cache/teacherdata/SP2019 Faculty Survey.csv')
+    survey_2019 = add_realnames(survey_2019,names_lookup)
+    master6 = pd.merge(master5, survey_2019, left_on='name', right_on='ilearn_name', how='left')
+    master6.to_csv('cache/trash/joined6.csv')
+
+
+    newnames = [ x.strip() for x in open('cache/poll_question_names.txt','r').readlines() ]
+    namedict = {}
+    for i,n in enumerate(newnames):
+        if i%3==1: newname = n
+        if i%3==2: namedict[oldname] = newname
+        if i%3==0: oldname = n
+    master6 = master6.rename(columns=namedict)
+    master6.to_csv('cache/teacherdata/staff_main_table.csv')
+    master6.to_csv('cache/teacherdata/staff_main_table.csv')
+    """
+    
+    
+    master4.to_csv('cache/teacherdata/staff_main_table.csv')
+    master4.to_csv('gui/public/staff_main_table.csv')    
+
+    other_training_records = read_training_records()
+    #print(json.dumps(other_training_records,indent=2))
+    #print("This is misc workshops.")
+    tt = pd.DataFrame.from_dict(other_training_records,orient='index')
+    tt = tt.fillna("")
+    #print(tt)
+    #input('\npress enter to continue')
+    
+    
+   
+    #teacherSharedCourses(schedule_one_yr)
+    #getAllTeachersInTerm()
+    
+
+
+def enroll_staff_shell():
+    staff = users_with_gavilan_email()
+    for i,s in staff.iterrows():
+        print(s['canvasid'],s['name'])
+        u = url + '/api/v1/courses/8528/enrollments'
+        param = {
+            'enrollment[user_id]':s['canvasid'],
+            'enrollment[type]': 'StudentEnrollment',
+            'enrollment[enrollment_state]': 'active',
+        }
+         
+        res = requests.post(u, headers = header, data=param)
+        print(res.text)
+
+#"Jun 28 2018 at 7:40AM" -> "%b %d %Y at %I:%M%p"
+#"September 18, 2017, 22:19:55" -> "%B %d, %Y, %H:%M:%S"
+#"Sun,05/12/99,12:30PM" -> "%a,%d/%m/%y,%I:%M%p"
+#"Mon, 21 March, 2015" -> "%a, %d %B, %Y"
+#"2018-03-12T10:12:45Z" -> "%Y-%m-%dT%H:%M:%SZ"
+
+
+# take a list of raw hits. 
+def activity_summary(hits):
+    #infile = "cache/teacherdata/activity/G00101483.json"
+    #data = json.loads(open(infile,'r').read())
+    #hits = data['raw']
+    if not hits:
+        return [ [], [], ]
+    dt_list = []
+    
+    one_week = datetime.timedelta(days=14)                       # actually two....
+    today = dt.now().replace(tzinfo=pytz.timezone('UTC'))
+    
+    target = today - one_week
+    
+    for h in hits:
+        the_stamp = parser.parse(h['created_at'])
+        if the_stamp > target:
+            dt_list.append(the_stamp)
+    df = pd.DataFrame(dt_list,  columns=['date',])
+    df.set_index('date', drop=False, inplace=True)
+    df.rename(columns={'date':'hits'}, inplace=True)
+    #df.resample('1D').count().plot(kind='bar')
+    #return df.resample('1D').count().to_json(date_format='iso')
+    #print(hits)
+    #print(df)
+    if not df.size:
+        return [ [], [], ]
+    bins = df.resample('1D').count().reset_index()
+    bins['date'] = bins['date'].apply(str)
+    #print(bins)
+    return [bins['date'].to_list(), bins['hits'].to_list()]
+    
+    #plt.show()
+    
+    #df = df.groupby([df['date'].dt.to_period('D')]).count().unstack()
+    #df.groupby(TimeGrouper(freq='10Min')).count().plot(kind='bar')
+    #df.plot(kind='bar')
+
+
+
+# next step
+# 1. save timestamp of the fetch
+#
+# 2. parse it and only fetch since then. afterwards, pull out non-hits. Summarize day/week/month stats. 
+#
+# 2a. merge old and new records, and re-summarize.
+#
+# 3. Next improvements in GUI. hook up to python server backend.
+#
+# Get views counts on current teachers. todo: month is hardcoded here
+def get_recent_views(id=1):
+    dt_format = "%Y-%m-%dT%H:%M:%SZ"
+    default_start_time = dt.strptime("2020-08-14T00:00:00Z", dt_format)
+    default_start_time = default_start_time.replace(tzinfo=pytz.timezone('UTC'))
+    end_time = dt.now(pytz.utc)
+    print("End time is: %s" % str(end_time))
+    myheaders = "x,teacher,prct_online,prct_lecture,prct_hybrid,num_sections_last_year,canvasid_x,name,sortable_name,goo,email,index_x,courses,dept,ilearn_name_x,canvasid_y,canvasid_x,bootcamp_grade,bootcamp_date_x,canvasid_y,bootcamp_progress,bootcamp_date_y,index_y,committees".split(",")
+    
+    teachers = [row for row in csv.reader(open('cache/teacherdata/staff_main_table.csv','r'))][1:]
+    
+    #tt = teachers[6:10]
+    
+    summary = {}
+    
+    for t in teachers:
+        name = t[1]
+        if name=="" or name=="TBA": continue
+        if not t[6]: continue
+        the_id = int(float(t[6]))
+        if the_id == 290: continue   # STAFF STAFF
+        goo = t[9]
+        print(goo)
+        
+        # read log of this person:
+        try:
+            prev_logf = codecs.open('cache/teacherdata/activity/%s.json' % goo,'r','utf-8')
+            prev_log = json.loads(prev_logf.read())
+            prev_logf.close()
+        except:
+            print("Exception happened on reading previous temp logs.")
+            prev_log = ''
+        
+        if type(prev_log) == dict:
+            lastfetch = dt.strptime(prev_log['meta']['lastfetch'], dt_format)
+            lastfetch = lastfetch.replace(tzinfo=pytz.timezone('UTC'))
+            print("last fetch is: " + str(lastfetch))
+            print("Hits BEFORE was: %i" % len(prev_log['raw']))
+        else:
+            lastfetch = default_start_time
+            prev_log = { "raw":[], }
+        
+        end_time = dt.now(pytz.utc)
+        u = url + "/api/v1/users/%s/page_views?start_time=%s&end_time=%s&per_page=100" % (str(the_id),lastfetch.strftime(dt_format), end_time.strftime(dt_format))
+        #print(u)
+        #input('getting this url')
+        
+        print(name + "\t",end='\n')
+        if 1:    # get fresh data?
+            r = fetch(u)
+            prev_log['raw'].extend( r )
+        summ = activity_summary(prev_log['raw'])
+        mydata = {'meta':{'lastfetch':end_time.strftime(dt_format)},'summary':summ,'raw':prev_log['raw']}
+        codecs.open('cache/teacherdata/activity/%s.json' % goo,'w','utf-8').write( json.dumps(mydata,indent=2))
+        summary[the_id] = [the_id, name, len(prev_log['raw']),goo, summ ,mydata['meta']]
+        print("Hits AFTER is: %i" % len(prev_log['raw']))
+    codecs.open('cache/teacherdata/activitysummary.json','w','utf-8').write( json.dumps(summary,indent=2) )
+    codecs.open('gui/public/activitysummary.json','w','utf-8').write( json.dumps(summary,indent=2) )
+
+
+        
+# Have they taught online or hybrid classes?        
+def categorize_user(u):
+    global role_table, term_courses
+    their_courses = get_enrlmts_for_user(u, role_table)
+    num_s = 0
+    num_t = 0
+    type = 's'
+    online_only = 1
+    is_online = []
+    #print their_courses
+    for x in their_courses.iterrows():
+        if len(x):
+            ttype = x[1]['type']
+            if ttype=='StudentEnrollment': num_s += 1
+            if ttype=='TeacherEnrollment': num_t += 1
+            cid = x[1]['course_id']
+            current_term = term_courses[lambda x: x['id']==cid]
+            if not current_term.empty:
+                is_online.append(current_term['is_online'].values[0])
+            else: online_only = 0
+        else: online_only = 0
+    if num_t > num_s: type='t'
+    if len(is_online)==0: online_only = 0
+    
+    for i in is_online: 
+        if i==0: online_only = 0
+    #print "Type: " + type + " All online: " + str(online_only) + " Number courses this term: " + str(len(is_online))
+    return (u[0],type, online_only, len(is_online))
+    
+
+        
+##########
+##########
+##########    PHOTOS
+##########
+##########    # todo: threaded
+
+# Doest the account have a photo loaded?
+def checkForAvatar(id=2):
+    try:
+        t = url + '/api/v1/users/%s?include[]=last_login' % str(id)
+        r2 = requests.get(t, headers = header)
+        result = json.loads(r2.text)
+        codecs.open('cache/users/%s.txt' % str(id),'w','utf-8').write( json.dumps(result,indent=2) )
+        
+        if 'avatar_url' in result:
+            if re.search(r'avatar\-50',result['avatar_url']): return 0 
+            else: return (result['login_id'], result['avatar_url'], result['name'])
+    except Exception as e:
+        print("Looking for an avatar / profile pic had a problem: %s" % str(e))
+    return 0
+
+# Grab em. Change the first if when continuing after problems....
+def downloadPhoto():
+    pix_dir = 'cache/picsCanvas2022/'
+    # Update the list of all ilearn users?
+    i_last_ix = '-1'
+    photo_log_f = ''
+    if 0:                           ##  CHANGE TO 0 IF CRASHED / RESUMING....
+        ii = fetchAllUsers()
+        photo_log_f = open("cache/fotolog.txt", "w")
+    else:
+        ii = json.loads(codecs.open('cache/allusers_ids.json','r').read())
+        photo_log_f = open("cache/fotolog.txt", "r+")
+        i_last_ix = -1
+        try:
+            ab = photo_log_f.read()
+            print(ab)
+            ac = ab.split("\n")
+            print(ac)
+            i_last_ix = ac[-2]
+            print(i_last_ix)
+        except:
+            i_last_ix = -1
+        i_last_ix = int(i_last_ix)
+
+    
+    print("Last user index checked was: %s, which is id: %s" %  \
+            (i_last_ix, ii[i_last_ix] ))
+
+    print("Max index is: %i" % len(ii))
+
+
+    i_last_ix += 1
+    for index in range(i_last_ix, len(ii)):
+        i = ii[index]
+        photo_log_f.write("\n%i" % i )
+
+        a = checkForAvatar(i)
+        if a:
+            print(str(i) + ":\t" + str(a[0]) + "\t" + str(a[2]) )
+            
+            try:
+                r = requests.get(a[1], stream=True)
+                if r.status_code == 200:
+                    r.raw.decode_content = True
+                    h=r.raw
+                    with open(pix_dir + a[0].lower(), 'wb') as f:
+                        shutil.copyfileobj(h, f) 
+                    # rename to right file extension          
+                    img_type = imghdr.what(pix_dir + a[0].lower())
+                    if img_type == 'jpeg': img_type = 'jpg'
+                    try:
+                        shutil.move(pix_dir + a[0].lower(),pix_dir + a[0].lower()+'.'+img_type)
+                    except Exception as e:
+                        print("   \tCouldn't rewrite file")
+                else:
+                    print(str(i) + ":\t didn't get expected photo")
+            except Exception as e:
+                print("   \tProblem with download " + str(e))
+        else:
+            print(str(i) + ":\tno user or no photo")
+            pass
+            
+
+def mergePhotoFolders():
+    
+    staff = [ row for row in csv.reader(  open('cache/teacherdata/staff_main_table.csv','r') ) ]
+    
+    headers = staff[0]
+    staff = staff[1:]
+    
+    activestaff = []
+    
+    for i,h in enumerate(headers):
+        #print("%i. %s" % (i,h) )
+        pass
+    
+    for S in staff:
+        if S[7] and S[15]:   #   if teacher (name present) and sp20crns (taught in sp20)
+            activestaff.append(S[9].lower())
+    activestaffset=set(activestaff)
+    
+    #return
+    
+    a = 'cache/picsCanvas'
+    b = 'gui/public/picsCanvas2018'
+    c = 'gui/public/picsCanvasAll'
+    
+    
+    # I want a big list of who has an avatar pic. 
+    
+    # and i want to know how many updated since last DL, and how many are in only one or the other. 
+    
+    
+    old = os.listdir(b)
+    count = defaultdict(int)
+    
+    oldset = set()
+    newset = set()
+    
+    for O in old:
+        if O.endswith('.jpg') or O.endswith('.png'):
+            g = O.split(r'.')[0]
+            oldset.add(g)
+    
+    for N in os.listdir(a):
+        if N.endswith('.jpg') or N.endswith('.png'):
+            g = N.split(r'.')[0]
+            newset.add(g)
+    
+    """print("Active SP20 Teachers")
+    print(activestaffset)
+
+    print("Old Avatars")
+    print(oldset)
+
+    print("New Avatars")
+    print(newset)"""
+    
+    updated_set = oldset.union(newset)
+    
+    tch_set = updated_set.intersection(activestaffset)
+    
+    only_old = oldset.difference(newset)
+    
+    only_new = newset.difference(oldset)
+    
+    print("Tch: %i  Old: %i New: %i" % (len(activestaffset),len(oldset),len(newset)))
+    
+    print("All avatars: %i  Teachers: %i   Only in old: %i   Only in new: %i" % (  len(updated_set), len(tch_set), len(only_old), len(only_new)))
+
+    allpics = os.listdir(c)
+    
+    haveapic = {}
+    for A in allpics:
+        if A.endswith('.jpg') or A.endswith('.png'):
+            g = (A.split(r'.')[0]).upper()
+            
+            haveapic[g] = A
+    outie = codecs.open('gui/public/pics.json','w').write( json.dumps( haveapic,indent=2))
+            
+        
+def mergePhotoFolders2():
+    
+    staff = [ row for row in csv.reader(  open('cache/teacherdata/staff_main_table.csv','r') ) ]
+    
+    headers = staff[0]
+    staff = staff[1:]
+    
+    activestaff = []
+    
+    for i,h in enumerate(headers):
+        #print("%i. %s" % (i,h) )
+        pass
+    
+    for S in staff:
+        if S[5]:
+            activestaff.append(S[9].lower())
+    
+    a = 'cache/picsCanvas'
+    b = 'gui/public/picsCanvas2018'
+    c = 'gui/public/picsCanvasAll'
+    
+    old = os.listdir(b)
+    count = defaultdict(int)
+    for N in os.listdir(a):
+        if N.endswith('.jpg') or N.endswith('.png'):
+            g = N.split(r'.')[0]
+            if g in activestaff:
+                count['s'] += 1
+            if N in old:
+                #print( "Y - %s" % N)
+                count['y'] += 1
+            else:
+                #print( "N - %s" %N )
+                count['n'] += 1
+        else:
+            #print("x - %s" % N)
+            count['x'] += 1
+    print("Of the 2020 avatars, %i are in the 2018 folder, and %i are new." % (count['y'],count['n']))
+    print("Of %i active teachers, %i have avatars." % (len(activestaff),count['s']))
+    #print(json.dumps(count,indent=2))
+        
+    
+
+# Go through my local profile pics, upload any that are missing.
+def uploadPhoto():
+    files = os.listdir('pics2017')
+    #print json.dumps(files)
+    pics_i_have = {}
+    #goo = "g00188606"
+    canvas_users = json.loads(open('canvas/users.json','r').read())
+    t = url +  '/api/v1/users/self/files'
+    i = 0
+    j = 0
+    pics_dir = 'pics2017/'
+    
+    for x in canvas_users:
+        j += 1
+        if x['login_id'].lower() + '.jpg' in files:
+            #print x['login_id'] + " " + x['name']
+            i += 1
+            pics_i_have[x['id']] = x
+    
+    print('Canvas users: ' + str(j))
+    print('Pic matches: ' + str(i))
+    account_count = 0
+    ids_i_uploaded = []
+    
+    for id, target in list(pics_i_have.items()):
+        #if account_count > 50:
+        #    print 'Stopping after 5.'
+        #    break
+            
+        print('trying ' + target['name'] + '(' + str(id) + ')')
+        if checkForAvatar(id):
+            print("Seems to have avatar loaded.")
+            continue
+        
+        goo = target['login_id'].lower()
+        local_img = pics_dir + goo + '.jpg'
+        inform_parameters = {
+            'name':goo + '.jpg',
+            'size':os.path.getsize(local_img), # read the filesize
+            'content_type':'image/jpeg',
+            'parent_folder_path':'profile pictures',
+            'as_user_id':'{0}'.format(id)
+        }
+         
+        res = requests.post(t, headers = header, data=inform_parameters)
+        print("Done prepping Canvas for upload, now sending the data...")
+        json_res = json.loads(res.text,object_pairs_hook=collections.OrderedDict)
+        files = {'file':open(local_img,'rb').read()}
+      
+        _data = list(json_res.items())
+        _data[1] = ('upload_params',list(_data[1][1].items()))
+        print("Yes! Done sending pre-emptive 'here comes data' data, now uploading the file...")
+        upload_file_response = requests.post(json_res['upload_url'],data=_data[1][1],files=files,allow_redirects=False)
+        # Step 3: Confirm upload
+        print("Done uploading the file, now confirming the upload...")
+        confirmation = requests.post(upload_file_response.headers['location'],headers=header)
+        if 'id' in confirmation.json():
+            file_id = confirmation.json()['id'] 
+        else:
+            print('no id here')
+        #print(confirmation.json())
+        print("upload confirmed...nicely done!")
+        
+        time.sleep(1)
+        # Make api call to set avatar image to the token of the uploaded imaged (file_id)
+        params = { 'as_user_id':'{0}'.format(id)}
+        avatar_options = requests.get("https://%s/api/v1/users/%s/avatars"%(domain,'{0}'.format(id)),headers=header,params=params)
+        #print "\nAvatar options: " 
+        #print avatar_options.json()
+        for ao in avatar_options.json():
+            #print ao.keys()
+            if ao.get('display_name')==goo + '.jpg':
+                #print("avatar option found...")
+                #print((ao.get('display_name'),ao.get('token'), ao.get('url')))
+                params['user[avatar][token]'] = ao.get('token')
+                set_avatar_user = requests.put("https://%s/api/v1/users/%s"%(domain,'{0}'.format(id)),headers=header,params=params)
+                if set_avatar_user.status_code == 200:
+                    print(('success uploading user avatar for {0}'.format(id)))
+                    account_count += 1
+                    ids_i_uploaded.append(id)
+                else:
+                    print('some problem setting avatar')
+            else:
+                pass #print 'didnt get right display name?'
+    print("Uploaded these guys: " + json.dumps(ids_i_uploaded))
+    
+
+
+
+##########
+##########
+##########    EMAILING PEOPLE
+##########
+##########
+
+
+    
+def test_email():
+    send_z_email("Peter Howell", "Peter", "phowell@gavilan.edu", ['CSIS85','CSIS42'])
+    
+
+def create_ztc_list():
+    course_combos = pd.read_csv('cache/teacher_course_oer_email_list.csv')
+    course_combos.fillna('',inplace=True)
+    
+    # read this file and make it a dict (in one line!)
+    dept_counts = { x[0]:x[1].strip() for x in [ y.split(',') for y in open('cache/teacher_course_oer_deptcount.csv','r').readlines() ][1:] }
+    
+    
+    course_template = "<a href='%s'>%s</a> &nbsp; &nbsp;"
+    url_template = "https://docs.google.com/forms/d/e/1FAIpQLSfZLQp6wHFEdqsmpZ7jz2Y8HtKLo8XTAhrE2fyvTDOEgquBDQ/viewform?usp=pp_url&entry.783353363=%s&entry.1130271051=%s"    #   % (FULLNAME, COURSE1)
+
+
+
+    # list depts
+    mydepts = sorted(list(set(course_combos['dept'] )))
+    i = 0
+    outp = open("output/oer_email_list.csv","w")
+    outp.write("fullname,firstname,email,link,courses\n")
+    
+    ones_i_did = [ int(x) for x in "40 38 31 21 7 12 24 25 1 13 18 22 44 55 56 51 20 16 2 3 4 5 6 8 9 10 11 14 15 17 23 53 52 50 30 48 39 37 54 49 47 46 45 43 42 41 33 32 29 28 27 26".split(" ") ]
+    
+    for D in mydepts:
+        i += 1
+        extra = ''
+        if D in dept_counts:
+            extra = " (%s)" % dept_counts[D]
+        extra2 = ''
+        if i in ones_i_did: 
+            extra2 = "xxxx "
+        print("%s  %i. %s %s" % (extra2,i,D,extra))
+    choice_list = input("Which department? (for multiple, separate with spaces) ").split(' ')
+    
+    all_people_df = []
+    
+    for choice in choice_list:
+        is_cs = course_combos['dept']==mydepts[int(choice)-1] 
+        filtered =  pd.DataFrame(course_combos[is_cs])
+        if len(all_people_df): all_people_df = pd.concat([filtered,all_people_df])
+        else: all_people_df = filtered
+        print(mydepts[int(choice)-1])
+        print(all_people_df)
+        print(' ')
+    all_people_df.sort_values(by=['name'],inplace=True)    
+    print(all_people_df)
+    
+    b = all_people_df.groupby(['name'])
+    for name,group in b:
+        if name == 'no data': continue
+        nameparts = name.split(', ')
+        fullname = nameparts[1] + ' ' + nameparts[0]
+        firstname = nameparts[1]
+        
+        outp.write(fullname + ',' + firstname + ',')
+        email = ''
+        link = ''
+        courses = []
+        flag = 1
+        for i in group.iterrows():
+            g = i[1]   # wtf is this shi.....
+            this_course = g.dept + ' ' + str(g.codenum) + g.codeletter
+            courses.append( this_course ) #print(g)
+            email = g.email
+            if flag:
+                link = url_template % (fullname, this_course)
+            flag = 0
+        
+        outp.write(email + ',' + link + "," + "    ".join(courses) + "\n")
+        
+    outp.close()
+    
+
+##########
+##########
+##########    FORENSICS TYPE STUFF
+##########
+##########
+
+# better name for this standard fetch. so they stay together in alpha order too....
+
+def get_user_info(id):
+    u = fetch( '/api/v1/users/%i' % id )
+    ff = codecs.open('cache/users/%i.txt' % id, 'w', 'utf-8')
+    ff.write( json.dumps(u, indent=2))
+    return u
+
+
+# these are any messages that get pushed out to their email
+def comm_mssgs_for_user(uid=0):
+    if not uid:
+        uid = input('Canvas id of the user? ')
+    u = url + '/api/v1/comm_messages?user_id=%s&start_time=%s&end_time=%s' % (uid,'2021-01-01T01:01:01Z','2021-08-01T01:01:01Z')     # &filter[]=user_%s' % uid
+    convos = fetch(u,1)
+    
+    oo = codecs.open('cache/comms_push_user_%s.txt' % str(uid), 'w')
+    oo.write('USER %s\n' % uid)
+    oo.write(json.dumps(convos, indent=2))
+    
+    print(convos)
+
+
+#
+def convos_for_user(uid=0):
+    if not uid:
+        uid = input('Canvas id of the user? ')
+    u = url + '/api/v1/conversations?include_all_conversation_ids=true&as_user_id=%s' % uid     # &filter[]=user_%s' % uid
+    convos = fetch(u,1)
+    
+    oo = codecs.open('cache/convo_user_%s.txt' % str(uid), 'w')
+    oo.write('USER %s\n' % uid)
+    oo.write(json.dumps(convos, indent=2))
+    
+    convo_ids_list = convos["conversation_ids"]
+    print(convo_ids_list)
+    
+    u2 = url + '/api/v1/conversations?include_all_conversation_ids=true&scope=archived&as_user_id=%s' % uid     # &filter[]=user_%s' % uid
+    archived_convos = fetch(u2,1)
+    try:
+        aconvo_ids_list = archived_convos["conversations_ids"]
+        print(aconvo_ids_list)
+    except:
+        print("didnt seem to be any archived.")
+        aconvo_ids_list = []
+    
+    u3 = url + '/api/v1/conversations?include_all_conversation_ids=true&scope=sent&as_user_id=%s' % uid     # &filter[]=user_%s' % uid
+    sent_convos = fetch(u3,1)
+    try:
+        sconvo_ids_list = sent_convos["conversations_ids"]
+        print(sconvo_ids_list)
+    except:
+        print("didnt seem to be any sent.")
+        sconvo_ids_list = []
+    
+    convo_ids_list.extend(aconvo_ids_list)
+    convo_ids_list.extend(sconvo_ids_list)
+    
+    
+    ##
+    ## Now get all the messages in each of these conversations
+    ##
+    
+    for cid in convo_ids_list:
+        print("Fetching conversation id: %s" % cid)
+        oo.write("\n\n----------------\nconversation id: %s\n\n" % cid)
+        
+        u4 = url + '/api/v1/conversations/%s?as_user_id=%s' % (cid,uid)       #     ' % (cid, uid
+        coverstn = fetch(u4,1)
+        oo.write("\n%s\n\n" % json.dumps(coverstn,indent=2))
+        
+        
+    
+    
+    
+    """
+    for c in convos:
+        c['participants'] = ", ".join([ x['name'] for x in c['participants'] ])
+    includes = tuple("last_message subject last_message_at participants".split(" "))
+    convos = list( \
+            reversed([ funcy.project(x, includes) for x in convos ]))
+    """
+    
+    #
+    
+    #print(json.dumps(convos, indent=2))
+
+
+# single q sub
+def quiz_get_sub(courseid, quizid, subid=0):
+    u = url + "/api/v1/courses/%s/quizzes/%s/submissions/%s" % ( str(courseid), str(quizid), str(subid) )
+
+    u = url + "/api/v1/courses/%s/quizzes/%s/questions?quiz_submission_id=%s" % \
+            ( str(courseid), str(quizid), str(subid) )
+
+    u = url + "/api/v1/courses/%s/assignments/%s/submissions/%s?include[]=submission_history" % \
+            ( str(courseid), str(quizid), str(subid) )
+
+    u = url + "/api/v1/courses/%s/students/submissions?student_ids[]=all&include=submission_history&grouped=true&workflow_state=submitted" % str(courseid)
+    return fetch(u)
+
+    #?quiz_submission_id=%s"  
+
+# quiz submissions for quiz id x, in course id y
+def quiz_submissions(courseid=9768, quizid=32580):
+    #subs = quiz_get_sub(courseid, quizid)
+    #print( json.dumps( subs, indent=2 ) )
+
+    if 1:
+        # POST
+        data = { "quiz_report[includes_all_versions]": "true", "quiz_report[report_type]": "student_analysis" }
+
+        u = url + "/api/v1/courses/%s/quizzes/%s/reports?" % ( str(courseid), str(quizid) )
+        res = requests.post(u, headers = header, data=data)
+        print(res.content)
+
+    #u2 = url + "/api/v1/courses/%s/quizzes/%s/reports" % ( str(courseid), str(quizid) )
+    #res2 = fetch(u2)
+    #print( json.dumps(res2.content, indent=2))
+
+    jres2 = json.loads( res.content )
+    print(jres2)
+    if jres2['file'] and jres2['file']['url']:
+        u3 = jres2['file']['url']
+        r = requests.get(u3, headers=header, allow_redirects=True)
+        open('cache/quizreport.txt', 'wb').write(r.content)
+    return
+
+    for R in res2:
+        if R['id'] == 7124:
+            u3 = R['url']
+            r = requests.get(u3, headers=header, allow_redirects=True)
+            open('cache/quizreport.txt', 'wb').write(r.content)
+    return
+
+    u3 = url + "/api/v1/courses/%s/quizzes/%s/reports/%s" % ( str(courseid), str(quizid), res2[''] )
+
+    oo = codecs.open('cache/submissions.json','w', 'utf-8')
+    oo.write('[\n')
+    for s in subs:
+        if len(s['submissions']):
+            j = json.dumps(s, indent=2) 
+            print(j)
+            oo.write(j)
+            oo.write('\n')
+
+    oo.write('\n]\n')
+    return 0
+
+
+    #u = url + "/api/v1/courses/%s/quizzes/%s/submissions?include[]=submission" % (str(courseid), str(quizid))
+    u = url + "/api/v1/courses/%s/quizzes/%s/submissions" % (str(courseid), str(quizid))
+    subs = fetch(u, 0)
+    print( json.dumps( subs, indent=1 ) )
+
+    for S in subs['quiz_submissions']:
+        print(json.dumps(S))
+        submis = quiz_get_sub(courseid, quizid, S['id'])
+        print(json.dumps(submis, indent=2))
+
+
+
+# return (timeblock, course, read=0,write=1)
+def requests_line(line,i=0):
+    try:
+        L = line  # strip?
+        if type(L) == type(b'abc'): L = line.decode('utf-8')
+        for pattern in unwanted_req_paths:
+            if pattern in L: 
+                return 0
+        i = 0
+        line_parts = list(csv.reader( [L] ))[0]
+        #for p in line_parts:
+        #    print("%i\t%s" % (i, p))
+        #    i += 1
+
+        d = parser.parse(line_parts[7])
+        d = d.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
+        d = timeblock_24hr_from_dt(d)
+
+        #r = re.search('context\'\:\s(\d+)', line_parts[22])
+        #c = 0
+        #if r:
+        #    c = r.groups(1)
+        str1 = line_parts[20]
+        str2 = str1.replace("'",'"')
+        str2 = str2.replace("None",'""')
+        #print(str2)
+        j = json.loads(str2  )
+        c = j['context']
+        a = line_parts[5]
+        #print( str( (d, c, a) ))
+        return (d, str(c), a)
+    except Exception as e:
+        #print("Exception: " + str(e))
+        return 0
+
+
+# 
+def report_logs(id=0):
+    if not id:
+        L = ['10531', ]
+    else:
+        L = [ id, ]
+    report = []
+    for id in L:
+        emt_by_id = course_enrollment(id)
+        for U in emt_by_id.values():
+            user_d = defaultdict( int )
+            print( "Lookin at user: %s" % U['user']['name'] )
+            report.append( "User: %s\n" % U['user']['name'] )
+            log_file_name = 'cache/users/logs/%i.csv' % U['user']['id']
+            if path.exists(log_file_name):
+                print("Log file %s exists" % log_file_name)
+                temp = open(log_file_name, 'r').readlines()
+                for T in temp[1:]:
+                    #print(T)
+                    result = requests_line(T)
+                    if result:
+                        (d, c, a)  = result
+                        if c == id:
+                            user_d[d] += 1
+            print(json.dumps(user_d, indent=2))
+            for V in sorted(user_d.keys()):
+                report.append( "\t%s: %i\n" % ( dt_from_24hr_timeblock(V), user_d[V]) )
+        report.append("\n\n")
+    return report
+
+
+def track_users_in_sem():
+    L = users_this_semester_db()
+    sL = list(L)
+    sL.sort(reverse=True)
+    fetch_queue = queue.Queue()
+    
+    for i in range(num_threads):
+        worker = Thread(target=track_user_q, args=(i,fetch_queue))
+        worker.setDaemon(True)
+        worker.start()
+
+    for U in sL:
+        print( "adding %s to the queue" % U )
+        fetch_queue.put( U )
+
+    fetch_queue.join()
+    print("Done.")
+
+
+def track_users_in_class(L=[]):
+    if len(L)==0:
+        #id = '10531'
+        ids = input("Course ids, separated with comma: ")
+        L = [x for x in ids.split(',')]
+    print("Getting users in: " + str(L))
+
+    fetch_queue = queue.Queue()
+    
+    for i in range(num_threads):
+        worker = Thread(target=track_user_q, args=(i,fetch_queue))
+        worker.setDaemon(True)
+        worker.start()
+    
+
+    users_set = set()
+    for id in L:
+        emt_by_id = course_enrollment(id)
+        print(emt_by_id)
+        for U in emt_by_id.values():
+            if not U['user_id'] in users_set:
+                print(U)
+                print( "adding %s to the queue" % U['user']['name'] )
+                fetch_queue.put( U['user_id'] )
+                users_set.add(U['user_id'])
+
+    all_reports = []
+    fetch_queue.join()
+    print("Done with %i users in these courses." % len(users_set))
+    for id in L:
+        rpt = report_logs(id)
+        all_reports.append(rpt)
+        outp = codecs.open('cache/courses/report_%s.txt' % id, 'w', 'utf-8')
+        outp.write(''.join(rpt))
+        outp.close()
+    return all_reports
+
+def track_user_q(id, q):
+    while True:
+        user = q.get()
+        print("Thread %i: Going to download user %s" % (id, str(user)))
+        try:
+            track_user(user, id)
+        except FetchError as e:
+            pass
+        q.task_done()
+
+
+# honestly it doesn't make much sense to get full histories this way if they're
+# already in the canvas data tables....
+
+# just the most recent hits or a short period
+# 
+# Live data would be better.
+
+# Maintain local logs. Look to see if we have some, download logs since then for a user.
+def track_user(id=0,qid=0):
+    global recvd_date 
+    L = [id,]
+    if not id:
+        ids = input("User ids (1 or more separated by comma): ")
+        L = [int(x) for x in ids.split(',')]
+    print("Getting users: " + json.dumps(L))
+
+
+    for id in L:
+        id = int(id)
+        # Open info file if it exists, check for last day retrived
+        try:
+            infofile = open("cache/users/%i.txt" % id, 'r')
+            info = json.loads( infofile.read() )
+
+            # TODO: set up this info file if it isn't there. check any changes too. it
+            # was written where?....
+            infofile.close()
+        except Exception as e:
+            print("failed to open info file for user id %i" % id)
+
+            info = get_user_info(id)
+
+        print("(%i) Student %i Info: " % (qid,id))
+        #print( json.dumps(info, indent=2))
+
+        url_addition = ""
+        
+        if 1:    # hard code dates
+        
+            url_addition = "?start_time=%s&end_time=%s" % ( '2022-06-15T00:00:00-07:00',  '2022-12-31T00:00:00-07:00' )
+        elif 'last_days_log' in info:
+            print("There's existing log data for %s (%s)" % (info['name'] , info['sis_user_id']))
+            print("Last day logged was: %s" % info['last_days_log'])
+            url_addition = "?start_time=%s" % info['last_days_log']
+            the_stamp = parser.parse(info['last_days_log']) 
+            the_stamp = the_stamp.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
+            now = dt.now()
+            now = now.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
+            dif = now - the_stamp
+            print("It was %s ago" % str(dif))
+            if the_stamp < lds_stamp:
+                print("Too long, taking default")
+                url_addition = "?start_time=%s" % log_default_startdate
+            
+            #lds_stamp = parser.parse(log_default_startdate)
+
+##########
+        else:
+            url_addition = "?start_time=%s" % log_default_startdate
+            #if dif.days > 1:
+
+        url = "/api/v1/users/%i/page_views%s" % (id, url_addition)
+        print(url)
+
+        try:
+
+            api_gen = fetch_stream(url,0)
+            
+            log_file_name = 'cache/users/logs/%i.csv' % id
+            if path.exists(log_file_name):
+                print("Log file %s exists" % log_file_name)
+                temp = open(log_file_name, 'a', newline='')
+                csv_writer = csv.writer(temp) 
+            else:
+                print("Creating new log file: %s" % log_file_name)
+                temp = open(log_file_name, 'w', newline='')    ### TODO  
+                csv_writer = csv.writer(temp) 
+            
+            
+            count = 0
+            for result in api_gen:
+                if count == 0 and len(result):
+                    header = result[0].keys() 
+                    csv_writer.writerow(header) 
+                    # results come in newest first....
+                    recvd_date = result[0]['updated_at']
+                    print("(%i) Most recent hit is %s" % (qid,recvd_date))
+
+                count += len(result)
+                indent = "    " * qid
+                #print("(%i) Got %i records, %i so far" % (qid,len(result),count))
+                print("(%s - %i) %s %i" % (qid, id, indent, count))
+                if count > max_log_count:
+                    print("Too many logs, bailing. sorry.")
+                    break
+           
+                for R in result:
+                    csv_writer.writerow(R.values()) 
+
+            latest = parser.parse(recvd_date)
+            #last_full_day = (latest - timedelta(days=1)).isoformat()
+            info['last_days_log'] = recvd_date    #last_full_day
+
+            infofile = open("cache/users/%i.txt" % id, 'w')
+            infofile.write(json.dumps( info, indent=2 ))
+            infofile.close()
+              
+            print("(%i) Output to 'cache/users/log/%i.csv'" % (qid,id))
+        except FetchError as e:
+            print("Getting a 502 error.")
+            raise FetchError()
+        except Exception as e2:
+            print("Got an error receiving logs: %s" % str(e2))
+
+#
+def track_users_by_teacherclass():
+    all_teachers = teachers_courses_semester()
+
+    skip_to = "Punit Kamrah"
+    skipping = 1
+
+    grouped = funcy.group_by( lambda x: x[4], all_teachers )
+    g2 = {}
+    for k,v in grouped.items():
+        print(k)
+        if skipping and skip_to != k:
+            print("skipping")
+            continue
+        skipping = 0
+
+        g2[k] = list(funcy.distinct( v, 1 ))
+        print("\n\n\n\n\n")
+        print(k)
+        print("\n\n\n\n\n")
+
+        teacherfile = codecs.open('cache/teacherdata/reports/%s.txt' % k.replace(" ","_"),'w','utf-8')
+        class_ids = funcy.lpluck(1,v)
+        class_names = funcy.lpluck(2,v)
+        print(class_ids)
+        print(class_names)
+        
+        rpts = track_users_in_class(class_ids)
+
+        for i, R in enumerate(rpts):
+            teacherfile.write('\n\n\n---\n\n%s \n\n' % class_names[i])
+            teacherfile.write(''.join(R))
+            teacherfile.flush()
+        teacherfile.close()
+
+
+
+    print(json.dumps(g2, indent=2))
+ 
+
+def nlp_sample():
+    # Stream a training corpus directly from S3.
+    #corpus = corpora.MmCorpus("s3://path/to/corpus")
+    
+    stemmer = stem.porter.PorterStemmer()
+    
+    strings = [
+    "Human machine interface for lab abc computer applications",
+    "A survey of user opinion of computer system response time",
+    "The EPS user interface management system",
+    "System and human system engineering testing of EPS",
+    "Relation of user perceived response time to error measurement",
+    "The generation of random binary unordered trees",
+    "The intersection graph of paths in trees",
+    "Graph minors IV Widths of trees and well quasi ordering",
+    "Graph minors A survey",
+]
+    processed = [ [ stemmer.stem(y) for y in utils.simple_preprocess(x, min_len=4)] for x in strings]
+    print(processed)
+    dictionary = corpora.Dictionary( processed )
+    dct = dictionary
+    print(dictionary)
+    
+    corpus = [dictionary.doc2bow(text) for text in processed]
+    
+    print(corpus)
+    
+    # Train Latent Semantic Indexing with 200D vectors.
+    lsi = models.LsiModel(corpus, num_topics=4)
+    print(lsi.print_topics(-1))
+
+    # Convert another corpus to the LSI space and index it.
+    #index = similarities.MatrixSimilarity(lsi[another_corpus])
+    
+    tfidf = models.TfidfModel(corpus)
+    
+    #index = similarities.SparseMatrixSimilarity(tfidf[corpus], num_features=12)
+    index = similarities.MatrixSimilarity(lsi[corpus])
+    print(index)
+
+
+    # Compute similarity of a query vs indexed documents.
+    query = "tree graph".split()
+    query_bow = dictionary.doc2bow(query)
+    vec_lsi = lsi[query_bow]
+    
+    print(query_bow)
+    print(tfidf[query_bow])
+    print(vec_lsi)
+    print("ok")
+    
+    #                  LdaMulticore
+    
+    lda_model = models.LdaModel(corpus=corpus,
+                         id2word=dictionary,
+                         random_state=100,
+                         num_topics=4,
+                         passes=40,
+                         chunksize=1000,
+                         #batch=False,
+                         alpha='asymmetric',
+                         decay=0.5,
+                         offset=64,
+                         eta=None,
+                         eval_every=0,
+                         iterations=100,
+                         gamma_threshold=0.001,
+                         per_word_topics=True)
+    lda_model.save('cache/lda_model.model')
+    print(lda_model.print_topics(-1))
+    print(lda_model)
+    
+    for c in lda_model[corpus]:
+        print("Document Topics      : ", c[0])      # [(Topics, Perc Contrib)]
+        print("Word id, Topics      : ", c[1][:3])  # [(Word id, [Topics])]
+        print("Phi Values (word id) : ", c[2][:2])  # [(Word id, [(Topic, Phi Value)])]
+        print("Word, Topics         : ", [(dct[wd], topic) for wd, topic in c[1][:2]])   # [(Word, [Topics])]
+        print("Phi Values (word)    : ", [(dct[wd], topic) for wd, topic in c[2][:2]])  # [(Word, [(Topic, Phi Value)])]
+        print("------------------------------------------------------\n")
+    
+    
+    sims = index[vec_lsi]
+    print("ok2")
+    print(list(enumerate(sims)))
+
+    for document_number, score in sorted(enumerate(sims), key=lambda x: x[1], reverse=True):
+        print(document_number, score)
+
+
+def nlp_sample2():
+    # load english language model
+    nlp = spacy.load('en_core_web_sm',disable=['ner','textcat'])
+
+    text = "This is a sample sentence."
+
+    # create spacy 
+    doc = nlp(text)
+
+    for token in doc:
+        print(token.text,'->',token.pos_)
+
+
+
+
+
+def one_course_enrol():
+
+    users = '96 18771 2693 5863 327'.split()
+    course = '11015'
+    the_type = 'TeacherEnrollment'                          # 'StudentEnrollment'
+    u = url + '/api/v1/courses/%s/enrollments' % course
+    
+    for user in users:
+        param = {
+            'enrollment[user_id]':user,
+            'enrollment[type]': the_type,
+            'enrollment[enrollment_state]': 'active',
+        }
+         
+        res = requests.post(u, headers = header, data=param)
+        print(res.text)
+
+
+def find_new_teachers():
+    filename = "cache/fa22_sched.json"
+    jj = json.loads(codecs.open(filename,'r','utf-8').read())
+    for J in jj:
+        print( J['teacher'])
+    
+ 
+ 
+def user_db_sync():
+    #fetch all personnel dir entries from dir_api.php.  PERSL           unique emails                                             
+    persl = fetch("http://hhh.gavilan.edu/phowell/map/dir_api.php?personnel=1")
+    persl_emails = set([x.lower() for x in funcy.pluck('email',persl)])
+    #persl_ids = set([x.lower() for x in funcy.pluck('email',persl)])
+    #
+    #fetch all staff from ilearn                        ILRN            unique emails                                             
+    ilrn = json.loads(codecs.open("cache/ilearn_staff.json","r","utf-8").read())
+    ilrn_emails = set([x.lower() for x in funcy.pluck('email',ilrn)])
+    #
+    #fetch all conf_users from dir_api.php              CONUSR          unique emails                                             
+    conusr = fetch("http://hhh.gavilan.edu/phowell/map/dir_api.php?users=1")
+    conusr_emails = set([x.lower() for x in funcy.pluck('email',conusr)])
+    
+    #fetch all gavi_personnel_ext  from dir_api.php     GPEREXT         must have column 'personnel' or 'c_users' or both.        
+    gperext = fetch("http://hhh.gavilan.edu/phowell/map/dir_api.php?personnelext=1")
+    
+    all_emails = set(persl_emails)
+    all_emails.update(ilrn_emails)
+    all_emails.update(conusr_emails)
+    
+    all_emails = list(all_emails)
+    all_emails.sort()
+    
+    fout = codecs.open('cache/db_staff_report.csv','w','utf-8')
+    fout.write('email,personnel_dir,ilearn,conf_user\n')
+    for e in all_emails:
+        
+        if e in ilrn_emails and not (e in conusr_emails) and e.endswith('@gavilan.edu'):
+            E = funcy.first(funcy.where(ilrn,email=e))
+            goo = E['login_id'][3:]
+            #print("not in conf_user: %s  \t %s \t %s" % (e,E['short_name'], E['login_id']) )
+            print("INSERT INTO conf_users (goo,email,name) VALUES ('%s', '%s', '%s');" % (goo,e,E['short_name']) )
+            
+            # goo (minus G00) email, and name go into conf_users
+        
+        fout.write(e+',')
+        if e in persl_emails:
+            fout.write('1,')
+        else:
+            fout.write('0,')
+        if e in ilrn_emails:
+            fout.write('1,')
+        else:
+            fout.write('0,')
+        if e in conusr_emails:
+            fout.write('1,')
+        else:
+            fout.write('0,')
+        fout.write('\n')
+    fout.close()
+    #
+
+    #print( json.dumps( [persl,ilrn,conusr,gperext], indent=2 ) )
+    print('done')
+
+import traceback
+
+
+def find_no_goo():
+
+    DO_DELETE_USERS = 0
+    DO_DELETE_PORTFOLIOS = 0 
+    
+    output = codecs.open('cache/no_goo_numbers.json','w','utf-8')
+    output2 = codecs.open('cache/wrong_root_acct.json','w','utf-8')
+    output3 = codecs.open('cache/wrong_sis_import_id.json','w','utf-8')
+    output4 = codecs.open('cache/bad_portfolios.json','w','utf-8')
+    #output5 = codecs.open('cache/bad_portfolios_detail.html','w','utf-8')
+    all = []
+    no_root = []
+    no_sis = []
+    port = []
+    i = 0
+    j = 0
+    k = 0
+    p = 0
+    users = json.loads(codecs.open('cache/allusers.json','r','utf-8').read())
+    for u in users: 
+        if not 'login_id' in u:
+            print(u['name'])
+            i+=1
+            all.append(u)
+            user_port = []
+            pp = fetch(url + '/api/v1/users/%s/eportfolios' % str(u['id']))
+            for p_user in pp:
+                try:
+                    user_port.append(  fetch(url+'/api/v1/eportfolios/%s' % str(p_user['id'])   ) )
+                    if DO_DELETE_PORTFOLIOS:
+                        output5.write("<br />deleted: <a href='https://ilearn.gavilan.edu/eportfolios/%s'>%s\n" % (str(p_user['id']),str(p_user['id']))  )
+                        output5.flush()
+                        del_request = requests.delete(url + "/api/v1/eportfolios/%s"    % str(p_user['id'])  ,headers=header)
+                        print(del_request.text)
+                except Exception as e:
+                    traceback.print_exc()
+            p += len(pp)
+            port.append(pp)
+            
+            if DO_DELETE_USERS:
+                print("Deleting %s..." % u['name'])
+                del_request = requests.delete(url + "/api/v1/accounts/1/users/%s"    % str(u['id'])  ,headers=header)
+                print(del_request.text)
+        if 'root_account' in u and u['root_account'] != "ilearn.gavilan.edu":
+            no_root.append(u)
+            j += 1
+        if 'sis_import_id' in u and not u['sis_import_id']:
+            no_sis.append(u)
+            k += 1
+    print("Found %i users without G numbers" % i)
+    print("Found %i users with non gav root account" % j)
+    print("Found %i users without sis id" % k)
+    print("Found %i questionable portfolios" % p)
+    output.write(  json.dumps(all,indent=2) )
+    output2.write(  json.dumps(no_root,indent=2) )
+    output3.write(  json.dumps(no_sis,indent=2) )
+    output4.write(  json.dumps(port,indent=2) )
+    
+
+def track_a_user():
+    a = input("User ID? ")
+    track_user(a)
+    
+
+
+if __name__ == "__main__":
+    print ("")
+    options = { 1: ['Fetch iLearn users with @gavilan.edu email address', teacherRolesUpdateCache],
+                2: ['Fetch all users',fetchAllUsers],
+                5: ['Download user avatars', downloadPhoto],
+                6: ['Merge photo folders', mergePhotoFolders],
+                7: ['Get all teachers logs 1 month', get_recent_views],
+                8: ['Gather teacher history, a variety of stats.', getTeachersInfoMain],
+                9: ['test rtr.', read_training_records],
+                10: ['Get a users logs', track_user],
+                11: ['test: oneYearSchedule', oneYearSchedule],
+                12: ['summarize hit activity', activity_summary],
+                13: ['Get all users logs in a class', track_users_in_class],
+                14: ['Get logs for 1 user', track_a_user ],
+                15: ['Get all users logs in a semester', track_users_in_sem],
+                16: ['Report on attendance for all classes', track_users_by_teacherclass],
+                17: ['Show all convos for a user', convos_for_user],
+                21: ['Show all pushed notifications for a user', comm_mssgs_for_user],
+                18: ['Quiz submissions', quiz_submissions],
+                19: ['NLP Sample', nlp_sample],
+                20: ['Enroll a single user into a class', one_course_enrol],
+                21: ['Teachers new this semester', find_new_teachers],
+                22: ['Sync personnel and conference user databases', user_db_sync], 
+                23: ['Find non-gnumbers', find_no_goo ],
+                #3: ['Main index, 1 year, teachers and their classes', getAllTeachersInTerm],
+                #5: ['Match names in schedule & ilearn', match_usernames],
+                #6: ['Create Dept\'s ZTC list', create_ztc_list],
+                ##7: ['Build and send ZTC emails', send_ztc_mails],
+                #14: ['investigate the logs', investigate_logs],
+                #12: ['test: match_usernames', match_usernames],
+                #13: ['test: get all names', getAllNames],
+                #13: ['x', users_with_gavilan_email], 
+    }
+    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+        resp = int(sys.argv[1])
+        print("\n\nPerforming: %s\n\n" % options[resp][0])
+    
+    else:
+        print ('')
+        for key in options:
+            print(str(key) + '.\t' + options[key][0])
+        
+        print('')
+        resp = input('Choose: ')
+    
+    # Call the function in the options dict
+    options[ int(resp)][1]() 
+
diff --git a/util.py b/util.py
new file mode 100644
index 0000000..122e929
--- /dev/null
+++ b/util.py
@@ -0,0 +1,156 @@
+
+
+
+
+import re, csv
+from collections import defaultdict
+
+def print_table(table):
+    longest_cols = [
+        (max([len(str(row[i])) for row in table]) + 3)
+        for i in range(len(table[0]))
+    ]
+    row_format = "".join(["{:>" + str(longest_col) + "}" for longest_col in longest_cols])
+    for row in table:
+        print(row_format.format(*row))
+
+def remove_nl(str):
+    return str.rstrip()
+
+def UnicodeDictReader(utf8_data, **kwargs):
+    csv_reader = csv.DictReader(utf8_data, **kwargs)
+    for row in csv_reader:
+        yield {str(key, 'utf-8'):str(value, 'utf-8') for key, value in iter(list(row.items()))}
+
+
+def minimal_string(s):
+    s = s.lower()
+    s = re.sub(r'[^a-zA-Z0-9]',' ',s)
+    s = re.sub(r'(\s+)',' ',s)
+    s = s.strip()
+    return s
+
+
+def to_file_friendly(st):
+    st = st.lower()
+    st = re.sub( r"[^a-z0-9]+","_",st)
+    return st
+    
+def clean_title(st):    
+    sq = re.sub( r"[^a-zA-Z0-9\.\-\!]"," ",st )
+    if sq: st = sq
+    if len(st)>50: return st[:50]+'...'
+    return st
+
+
+
+def match59(x):
+    if x['links']['context']==7959: return True
+    return False
+  
+
+def item_2(x): return x[2]
+
+def unix_time_millis(dt):
+    wst = pytz.timezone("US/Pacific")
+    epoch = datetime.datetime.fromtimestamp(0)
+    epoch = wst.localize(epoch)
+    return (dt - epoch).total_seconds() * 1000.0
+
+# ENGL250 returns ENGL    
+def dept_from_name(n):
+    m = re.search('^([a-zA-Z]+)\s?[\d\/]+',n)
+    if m: return m.group(1)
+    print(("Couldn't find dept from: " + n))
+    return ''
+    
+def most_common_item(li):
+    d = defaultdict(int)
+    for x in li:
+        d[x] += 1
+    s = sorted(iter(list(d.items())), key=lambda k_v: (k_v[1],k_v[0]), reverse=True)
+    #pdb.set_trace()
+    return s[0][0]
+
+def srt_times(a,b):
+    HERE = tz.tzlocal()
+    da = dateutil.parser.parse(a)
+    da = da.astimezone(HERE)
+    db = dateutil.parser.parse(b)
+    db = db.astimezone(HERE)
+    diff = da - db
+    return diff.seconds + diff.days * 24 * 3600
+            
+def how_long_ago(a):    # number of hours ago 'a' was...
+    if not a: return 9999
+    HERE = tz.tzlocal()
+    d_now = datetime.datetime.now()
+    d_now = d_now.replace(tzinfo=None)
+    #d_now = d_now.astimezone(HERE)
+    d_then = dateutil.parser.parse(a)
+    d_then = d_then.replace(tzinfo=None)
+    #d_then = d_then.astimezone(HERE)
+    diff = d_now - d_then
+    return (diff.seconds/3600) + (diff.days * 24) + 8  # add 8 hours to get back from UTC timezone
+            
+def partition(times_list):
+    # get a list of times in this format: 2017-02-14T17:01:46Z
+    # and break them into a list of sessions, [start, hits, minutes]
+    global dd
+    mm = ['x','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
+    start = ""
+    last = ""
+    hits = 0
+    minutes_till_new_session = 26
+    delta = timedelta(minutes=26)
+    HERE = tz.tzlocal()
+    sessions = []
+    
+    sorted_times_list = sorted(times_list, srt_times)
+    current_set = []
+    timeline_times = []
+   
+    for T in sorted_times_list:
+        dt_naive = dateutil.parser.parse(T)
+        dt = dt_naive.astimezone(HERE)
+        timeline_st = unix_time_millis(dt)
+        
+        timeline_et = timeline_st + (1 * 60 * 1000)  # always end 1 minute later....
+        timeline_dict = {}
+        timeline_dict['starting_time'] = timeline_st
+        timeline_dict['ending_time'] = timeline_et
+        timeline_times.append(timeline_dict)
+        
+        month = mm[ int(dt.strftime("%m"))]
+        formatted = month + " " + dt.strftime("%d %H:%M")
+        if not start:                      # start a new session
+            start = dt
+            start_f = formatted
+            last = dt
+            current_set.append(formatted)
+            hits = 1
+        else:                           # 
+            if dt > last + delta:       # too long. save sesh. start another, if hits > 2
+                minutes = (last - start)
+                minutes = (minutes.seconds / 60) + 5
+                if hits > 2: 
+                    sessions.append( [start_f, hits, minutes,current_set] )
+                start = dt
+                start_f = formatted
+                last = dt
+                hits = 1
+                current_set = [formatted]
+            else:                       # put in current session
+                last = dt
+                current_set.append(formatted)
+                hits += 1
+    # save last sesh
+    if (last):
+        minutes = (last - start)
+        minutes = (minutes.seconds / 60) + 5
+        if hits > 2:
+            sessions.append( [start_f,hits,minutes,current_set] )
+    
+    dd.write(json.dumps(timeline_times))
+    
+    return sessions