import json, codecs, requests, re, pdb, csv, textdistance, collections
import sys, csv, string, funcy, math, shutil, os
import pytz, time
import pandas as pd
import matplotlib.pyplot as plt

#from pandas import TimeGrouper
from PIL import Image
from collections import defaultdict
from pipelines import fetch, fetch_stream, header, url, FetchError, put_file
from schedules import get_semester_schedule
from courses import course_enrollment, users_in_semester
from localcache import users_this_semester_db, unwanted_req_paths, timeblock_24hr_from_dt, dt_from_24hr_timeblock
from localcache import teachers_courses_semester, course_mode, sem_schedule
from localcache2 import all_2x_sem_courses_teachers, all_sem_courses_teachers
from schedules import campus_dept_hierarchy
#from pipelines import dean, dean_names  #TODO

from util import dept_from_name, most_common_item
from os.path import exists, getmtime

from canvas_secrets import url

from dateutil import parser
from datetime import datetime as dt
from datetime import timedelta
import datetime

import queue
from threading import Thread
from os import path

# for NLP
#import spacy
from gensim import corpora, models, similarities, downloader, utils
from nltk import stem


# todo: these constants

#last_4_semesters = 'fall2020 summer2020 spring2020 fall2019'.split(' ')
#last_4_semesters_ids = [62, 60, 61, 25]
last_4_semesters = 'spring2021 fall2020 summer2020 spring2020'.split(' ')
last_4_semesters_ids = [168, 65, 64, 62]

log_default_startdate = "2021-08-23T00:00:00-07:00"
lds_stamp = parser.parse(log_default_startdate)

recvd_date = '2023-01-01T00:00:00Z'
num_threads = 25
max_log_count = 250000


##########
##########
##########    GETTING USER DATA
##########
##########

# All users to a cache file     cache/allusers.json
def fetchAllUsers():

    if exists('cache/allusers.json'):
        time = date_time = dt.fromtimestamp( getmtime('cache/allusers.json') )
        newname = 'cache/allusers_'+ time.strftime('%Y%m%d') + ".json"
        print("renaming old data file to %s" % newname)
        os.rename('cache/allusers.json', newname)



    out1 = codecs.open('cache/allusers.json','w','utf-8')
    out2 = codecs.open('cache/allusers_ids.json','w','utf-8')
    all_u = fetch_stream(url + '/api/v1/accounts/1/users?per_page=100', 1)
    
    ids = []
    main_list = []
    for this_fetch in all_u:
        for U in this_fetch:
            ids.append(U['id'])
            main_list.append(U)
    
    ids.sort()
    out2.write( json.dumps(ids, indent=2))
    out1.write( json.dumps(main_list, indent=2))
    out2.close()
    out1.close()
    return ids
    


##########
##########
##########    TEACHERS LIST AND LOCAL USERS FILE
##########
##########

# Fetch teacher users objects from local cache
def teacherRolesCache():               # I used to be load_users
    users_raw = json.load(open('cache/ilearn_staff.json','r'))
    users = {}
    users_by_id = {}
    for U in users_raw:
        users[ U['login_id'] ] = U
        users_by_id[ U['id'] ] = U
    return users, users_by_id






# Outputs:  cache/ilearn_staff.json    
# Canvas:   Fetch all people with gavilan.edu email address        
def teacherRolesUpdateCache():    # I used to be get_users
    t = fetch('/api/v1/accounts/1/users?per_page=500&search_term=%40gavilan.edu&include[]=email')
    g = open('cache/ilearn_staff.json','w')
    g.write( json.dumps(t) )
    g.close()
    #put_file('/gavilan.edu/staff/flex/2020/','cache/','ilearn_staff.json')
    print("Wrote to 'cache/ilearn_staff.json'")
    return teacherRolesCache()
    
    
# Fetch preferred email address for a given user id.   ( Canvas )
def getEmail(user_id):
    results = fetch("/api/v1/users/" + str(user_id) + "/communication_channels")
    for r in results:
        if r['type']=='email':
            return r['address']
    return ''

  
##########
##########
##########    TEACHERS AND OTHER STAFF
##########
##########
#
# Gather all my info, CRM style, in the folder teacherdata
#
# 
# Typical actions: For everyone with a teacher role:
# - What are the courses they taught for the last X semesters?
# - What's their activity level each semester?
# - Which of those courses are Online, Hybrid or Face2face?
#     + column for each semester:  OHLOHL
# - How many online classes have they taught in the past?
# - Are they brand new, or brand new online?# further...
# - what's their department?
# - what's their badges and 'tech level?'
# - 


# All teachers in a particular course
def getAllTeachers(course_id=59):  # a list 
    qry = '/api/v1/courses/' + str(course_id) + '/search_users?enrollment_type=teacher'
    t = url + qry
    while(t): t = fetch(t)
#
def classType(t):
    if t == 'lecture': return 'L'
    if t == 'online': return 'O'
    if t == 'hours': return 'R'
    if t == 'lab': return 'A'
    if t == 'hybrid': return 'H'
    else: return 'L'                   # todo: fix bug in schedule parser so non-online classes have a type field
    
def my_blank_string(): return "no data"
def my_blank_dict(): return {'name':'NoName','email':'noemail@gavilan.edu'}
def my_empty_dict(): return defaultdict(my_blank_string)

def get_email_from_rec(name,name_to_record):
    #print "Looking up: " + name
    try:
        return name_to_record[name]['email']
    except Exception as e:
        print("Missing Teacher %s" % name)
        return 'noemail@gavilan.edu'




# Pull the staff directory on the webpage. Convert to pandas dataframe
def staff_dir(get_fresh=False):
    """
    if get_fresh:
        url = "http://www.gavilan.edu/staff/dir.php"
        regex = "var\slist=(\[.*\]);"
        response = requests.get(url).text
        m = re.search(regex,response)
        if m: 
            output = '{"staff":' + m.group(1) + '}'
            of = open('cache/teacherdata/staff_dir.json','w')
            of.write(output)
            js = json.loads(output)
            df = pd.DataFrame(js['staff'])
            return df
            print("Wrote cache/teacherdata/staff_dir.json")
        else:
            print("Failed on staff directory scrape")
            return ''
    else:
        input = json.loads(open('cache/teacherdata/staff_dir.json','r').read())
        df = pd.DataFrame(input['staff'])
    return df
    """
    
    # TODO lol get fresh again... 
    
    old_dir = csv.reader(open('cache/personnel2020_04_12.csv'), delimiter=',')
    dept1_crxn = {r[0]:r[1] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
    dept2_crxn = {r[0]:r[2] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
    title_crxn = {r[0]:r[3] for r in csv.reader(open('cache/dir_corrections.csv'), delimiter=',') }
    revised_dir = [  ]
    columns = next(old_dir)
    
    for r in old_dir:
        old_dept = r[2]
        if old_dept in dept1_crxn:
            new_one = dept1_crxn[old_dept]
            if dept2_crxn[old_dept]: new_one += '/' + dept2_crxn[old_dept]
            if title_crxn[old_dept]: new_one += '/' + title_crxn[old_dept]
            r[2] =  new_one
        revised_dir.append(r)
    print(revised_dir)            
    return pd.DataFrame(revised_dir,columns=columns)    


#
#
#
#    ###
#    ###   TEACHER   CRM   FUNCTIONS
#    ###
#

def schedForTeacherOverview(long,short):
    sem = get_semester_schedule(short)
    sem['type'] = sem['type'].apply(classType)
    #sem['code'] = sem[['code','type']].apply(' '.join,axis=1)
    sem['sem'] = short
    sem = sem.drop(['time','loc','name','date','days'],axis=1)  # ,'crn'
    return sem




# Return a dataframe of the last 4 semester schedules put together
def oneYearSchedule():
    sp19 = schedForTeacherOverview('2019spring','sp19')
    su19 = schedForTeacherOverview('2019summer','su19')
    fa19 = schedForTeacherOverview('2019fall','fa19')
    sp20 = schedForTeacherOverview('2020spring','sp20')
    
    # The four-semester schedule
    a = pd.concat([sp19,su19,fa19,sp20], sort=True, ignore_index=True)
    a = a.drop(['cap','cmp','extra','rem','sec','cred','act'], axis=1)
    a.to_csv('cache/one_year_schedule.csv')
    return a

def num_sections_last_year(line):
    #if not type(line)=='str': return 0
    parts = line.split(' ')
    return len(parts)

def sec_type_stats(line):
    #print(type(line))
    #if not type(line)=='str': return {'fail':1}
    #print("in sts: " + str(line))
    parts = line.split(' ')
    output = defaultdict(int)
    for p in parts: output[p] += 1
    return output

def prct_online(line):
    d = sec_type_stats(line)
    #print(d)
    total = 0
    my_total = 0
    for k,v in d.items():
        total += v
        if k == 'O': my_total += v
    return int(100 * ((1.0)*my_total / total))

def prct_lecture(line):
    #print(line)
    d = sec_type_stats(line)
    #if 'fail' in d: return 0
    total = 0
    my_total = 0
    for k,v in d.items():
        total += v
        if k == 'L': my_total += v
    return int(100 * ((1.0)*my_total / total))


def prct_hybrid(line):
    d = sec_type_stats(line)
    #if 'fail' in d: return 0
    total = 0
    my_total = 0
    for k,v in d.items():
        total += v
        if k == 'H': my_total += v
    return int(100 * ((1.0)*my_total / total))

# Given the names of teachers in last year's schedules, fill in email, etc. from ilearn files
def teacher_basic_info(sched, from_ilearn, names):
    bi = from_ilearn    # pd.DataFrame(from_ilearn)
    bi.rename(columns={'id':'canvasid','login_id':'goo'}, inplace=True)
    # bi.drop(['name',],axis=1,inplace=True)
    
    #print(bi)
    #input('xx')
    
    sp20 = schedForTeacherOverview('2020spring','sp20')
    
    
    codes_sp20 = sp20.groupby('teacher')['code'].apply( lambda x: ' '.join(funcy.distinct(x)) )
    crns_sp20 = sp20.groupby('teacher')['crn'].apply( lambda x: ' '.join( map( str, funcy.distinct(x))) )
    codes_sp20.rename(columns={'code':'sp20code'}, inplace=True)
    codes_sp20.to_csv('cache/trash/codes_sp20.csv',header=True)
    crns_sp20.rename(columns={'crn':'sp20crn'}, inplace=True)
    crns_sp20.to_csv('cache/trash/crns_sp20.csv',header=True)

                   
    a = sched.groupby('teacher')['code'].apply( lambda x: ' '.join(funcy.distinct(x)) )
    a = pd.DataFrame(a)
    a.reset_index(inplace=True)
    a['dept'] = a.apply(guessDept,axis=1)
    print(a)
    
    def find_that_name(x):
        #print(x)
        if 'teacher' in x: return names(x['teacher'])
        #print('name not found?')
        return ''
    
    a['ilearn_name'] = a.apply( find_that_name, axis=1) 
    
    a.rename(columns={'code':'courses'}, inplace=True)
    #print(type(a))
    a.reset_index(inplace=True)
    
    a = pd.merge(a,codes_sp20.rename('sp20courses'), on='teacher')
    a = pd.merge(a,crns_sp20.rename('sp20crns'), on='teacher')
    a.to_csv('cache/trash/sched_w_sp20.csv',header=True)
    print(a)
    
    a['canvasid'] = a['teacher'].map(names)
    #print(a)
    c = pd.merge(bi, a, left_on='name', right_on='ilearn_name', how='outer')
    c.to_csv('cache/trash/basic.csv',header=True)
    #print(c)
    return c


# TODO Old and broken

# what percentage of their sections were online / hybrid /lecture ?
# Consumes:  output/semesters/fa19_sched.json and etc for 1 year 
# Outputs:   cache/teacher_by_semester.csv, 
def teacherModalityHistory(sched=[],names=[]):
    if not len(sched): 
        sched = oneYearSchedule()
        #names = match_username()
    
    # How many classes a teacher taught lect/online/hybrid/hours
    sec_type = sched.groupby(['teacher','sem'])['type'].apply(' '.join)
    sec_type.to_csv('cache/teacherdata/teacher_by_semester.csv',header=True)       
                                                                ## THIS IS THE LIST of how many
                                                                ## lecture, hybrid, online they've taught   
    
    #sec_type = pd.read_csv('cache/teacherdata/teacher_by_semester.csv')
    
    sec_grp = sec_type.groupby('teacher').aggregate( ' '.join )
    #sec_grp.to_csv('cache/trash/sec_grp_3.csv',header=True)
    
    #sec_grp = sec_grp.iloc[1:]   ## I'm seeing bad items on the first 2
    #sec_grp.drop(index='teacher')
    #sec_grp.to_csv('cache/trash/sec_grp_0.csv',header=True)
    
    #
    sec_grp = pd.DataFrame(sec_grp)
    #print(type(sec_grp))
    sec_grp['prct_online']  = sec_grp['type'].map(prct_online) 
    
    sec_grp['prct_lecture'] = sec_grp['type'].map(prct_lecture) 
    sec_grp['prct_hybrid']  = sec_grp['type'].map(prct_hybrid) 
    sec_grp['num_sections_last_year'] = sec_grp['type'].map(num_sections_last_year) 
    sec_grp.drop('type',axis=1,inplace=True)
    sec_grp.reset_index(inplace=True)
    sec_grp.to_csv('cache/teacherdata/modality_history.csv')
    return sec_grp
    
    

def teacherCourseHistory(a,names):
    pass
    # actually not using this. moved to _basic_info
    
    # YEEEAH
    sched = a.groupby(['teacher','code'])
    #for name,group in sched:
    #    print(name)
    #print(sched.count())
    return
    a['name'] = a.apply(lambda x: records_by_sname[x['teacher']]['name'],axis=1)
    a['email'] = a.apply(lambda x: records_by_sname[x['teacher']]['email'],axis=1)
    a.sort_values(by=['dept','teacher','codenum'],inplace=True)
    a = a.drop(['teacher'],axis=1)  
    a.to_csv('cache/teacherdata/courses_taught.csv')
    
    return a
    """
    d = a.groupby(['teacher'])                     # ,'dept','codenum','codeletter'
    
    out1 = open('teacherdata/courses_taught.csv','w')  
    by_dept = {}                                 # x todo: sort by dept also
    for name, group in d:
        #print name
        if re.search(r'^\d+',name) or name=='TBA':
            print("Skipping weird name: ", name)
            continue
        rec = {'email':'xx'}
        try:
            rec = records_by_sname[name]
            #print rec
        except Exception as e:
            print("Missing Teacher %s" % name)
            continue
        out1.write(name+"\t"+rec['email'])
        s = set()
        #print group
        for idx,r in group.iterrows():
            s.add( str(r[1]) + str(r[2]) + str(r[3]))
        for clas in sorted(s):
            d = dept_from_name(clas)
            if d in by_dept:
                if name in by_dept[d]:
                    by_dept[d][name].append(clas)
                else:
                    by_dept[d][name] = [ clas, ]
            else:
                by_dept[d] = { name: [ clas, ] }
                
            out1.write("\n\t"+str(clas))
        out1.write("\n")
    out1.write( json.dumps(by_dept,indent=2))"""



# Consumes:  output/semesters/fa19_sched.json and etc for 1 year 
# Outputs:   cache/course_teacher_combos.csv, 
def teacherSharedCourses(a=[]):
    if not len(a): a = oneYearSchedule()
    
    # List of classes. Group by teacher/format. Shows who has historically
    # taught a class and who teaches it most often.
    c = a.drop(['code','partofday','sem','site','type'],axis=1)   #,'dept','codeletter'
    c = c.groupby(['dept','codenum','codeletter'])  #,'teacher'
    c = c.aggregate(lambda x: set(x))
    c.to_csv('teacherdata/course_teacher_combos.csv')      ## THIS is the list of teachers who
                                                     ## share courses
    return c


# TODO: this is broken
# Consumes:  output/semesters/fa19_sched.json and etc for 1 year 
# Outputs:   cache/num_courses_per_dept.csv                 (not teacher_course_oer_deptcount)
# How many courses in each department were taught in the last year?
def departmentCountCourses(a=[]):
    if not len(a): a = oneYearSchedule()
    
    tt = a.drop(['code','partofday','sem','site','type'],axis=1)   #,'dept','codeletter'

    """records_by_sname = defaultdict(my_empty_dict, match_usernames())
    tt.drop_duplicates(keep='first',inplace=True)
    tt['name'] = tt.apply(lambda x: records_by_sname[x['teacher']]['name'],axis=1)
    tt['email'] = tt.apply(lambda x: records_by_sname[x['teacher']]['email'],axis=1)
    tt = tt.drop(['teacher'],axis=1)  
    tt.sort_values(by=['dept','name','codenum'],inplace=True)
    count = tt['dept'].value_counts()
    count.to_csv('cache/num_courses_per_dept.csv', header=True)"""


def clean_nonprint(s):
    return re.sub(f'[^{re.escape(string.printable)}]', '', s)

def read_cmte(names):
    output = []
    out2 = defaultdict(list)
    input = codecs.open('cache/teacherdata/committees_2018_2019.csv','r','utf-8')
    with input as csvfile:
        cmtereader = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in cmtereader:
            for R in row:
                R = R.strip()
                R = clean_nonprint(R)
            (fname,lname,cmtes) = row
            a = re.split(",\s*",cmtes)
            if len(a)>1:
                cmtes = a
            else:
                cmtes = a
            
            name1 = lname + ", " + fname
            name2 = fname + " " + lname
            name = name1
            realname = names(name1)
            if not realname:
                realname = names(name2)
                name = name2
            if realname:
                for cmm in cmtes:
                    output.append( [realname, cmm] )
                    out2[realname].append(cmm)
            else:
                print("committee participant name failed: %s / %s:\t%s" % (name1,name2,str(a)))
                print(type(name1))
    #print(out2)            
    return output,out2

def read_training_records():
    myinput = open('cache/teacherdata/more_2018_2019_training_attendance.txt','r').readlines()
    current_sesh = ""
    ppl_in_sesh = {}
    all_ppl = set()
    
    for L in myinput:
        L = L.strip()
        if L:
            if L.startswith('#'):
                ma = re.search(r'^\#\s(.*)$',L)
                if ma:
                    current_sesh = ma.group(1)
                else:
                    print("-- read_training_records: Couldn't find training set? " + L)
            else:
                if current_sesh in ppl_in_sesh:
                    ppl_in_sesh[current_sesh].append(L)
                else:
                    ppl_in_sesh[current_sesh] = [ L, ]
                all_ppl.add(L)
    if 0:
        print(ppl_in_sesh)
        print(all_ppl)
    
    # Want to pivot the dict, so key is a name, value is another dict, where k2 is session name, v2 is Y/N
    d_of_d = defaultdict(dict)
    
    for k,v in ppl_in_sesh.items():
        for user in v:
            d_of_d[user][k] = 'Y'
    
    return d_of_d    

# open a file and mark the people with their ids given. Return a dataframe
def read_bootcamp1(filename):
    a = pd.read_csv(filename)
    #print(a)
    b = a.loc[:, ['canvas_id','grade','last_activity']]
    b.rename(columns={'canvas_id':'bc1canvasid','grade':'bootcamp_grade','last_activity':'bootcamp_date'}, inplace=True)
    #print(b)
    return b

# open a file and mark the people with their ids given. Return a dataframe
def read_bootcamp2(filename):
    a = pd.read_csv(filename)
    #print(a)
    b = a.loc[:, ['canvas_id','grade','last_activity']]
    b.rename(columns={'canvas_id':'bc2canvasid','grade':'bootcamp_progress','last_activity':'bootcamp_date'}, inplace=True)
    #print(b)
    return b


def not_blank_or_pound(L):
    if L.startswith("#"): return False
    L = L.strip()
    if L == "": return False
    return True

def temp1(x):
    #print(x[1])
    return x[1]

def add_realnames(df,names):    # the surveys. raw name is in 2nd column
    df['ilearn_name'] = df.apply( lambda x: names(temp1(x),1), axis=1)
    return df

def compareToughNames(a,b):
    # search for a in b
    m = re.search(a, b)
    if m: return True
    return False
        

def compareNames(a,b,verbose=0):
    if a == b: return True
        
    cnDBG = 0
    try:
        parts_a = [ W.lower() for W in re.split("[\s,]", a) ]
        [ x.strip() for x in parts_a ]
        
        parts_b = [ W.lower() for W in re.split("[\s,]", b) ]
        [ x.strip() for x in parts_b ]
        
        pa2 = sorted([ parts_a[0], parts_a[-1] ])
        pb2 = sorted([ parts_b[0], parts_b[-1] ])
        
        if pa2 == pb2: 
            if cnDBG: print("->Match: %s, %s" % (a,b)) 
            return True
        if pa2[0] == pb2[0] or pa2[-1] == pb2[-1]:
            if cnDBG: print("--->Near match: %s" % b) 
            return False
    
    except Exception as e:
        #print("Problem with compareNames %s , %s" % (a,b))
        #print(e)
        return False
        
    if len(pa2[0])>3 and len(pb2[0])>3:
        if pa2[0][0] == pb2[0][0]:
            if pa2[0][1] == pb2[0][1]:
                if pa2[0][2] == pb2[0][2]:
                    if cnDBG: print("===> Near match (first 3): %s, %s, %s, %s" % (a, b, pa2[0], pb2[0])) 
                    pass
    
    b = b.lower()
    a = a.lower()
    
    #if verbose: print("searching: %s / %s" % (a,b))
    if re.search( b, a):
        #print("REGEX MATCH: %s | %s" % (a,b))
        return True
    if re.search( a, b):
        #print("REGEX MATCH: %s | %s" % (a,b))
        return True
    return False    

def find_ilearn_record(ilearn_records,manual_records, othername,verbose=0):
    # manual records are ('name':'canvas_id')
    #print(ilearn_records)
    if not othername: return ""
    if type(othername) == type(1.25): return ""
    #if math.isnan(othername): return False
        
    if othername in manual_records:
        a = funcy.first( funcy.where( ilearn_records, id=int(manual_records[othername]) ))
        if a:
            return a['name']

    for x in ilearn_records:
        #print('f_i_r')
        #print(othername)
        #print(x)
        if compareNames(othername,x['name'],verbose):
            return x['name']
    
    for k,v in manual_records.items():
        #print(k)
        #print(othername)
        #print(type(othername))
        b = re.search( k, othername)
        if b:
            a = funcy.first( funcy.where( ilearn_records, id=int(manual_records[k]) ))
            if a:
                return a['name']
    return ""


def manualNamesAndDept():
    # copied from // getTeachersInfoMain ....   
    
    schedule_one_yr = oneYearSchedule()
    from_ilearn = list(  map(  lambda y: funcy.select_keys( lambda z: z in ['name','id','email','login_id','sortable_name'], y), \
                    json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) ) )
    manual_names = manualNames()
    names_lookup = funcy.partial(find_ilearn_record, from_ilearn, manual_names)
    teacher_info = teacher_basic_info(schedule_one_yr, from_ilearn, names_lookup)
    # till here
    
    
    # the staff directory
    dr = staff_dir(False)
    print(dr)
    print(dr.columns)
    print( dr['department'].unique() )
    
    # now to reconcile and combine these....
    #
    # we want:
    #  - alternate names of academic / other depts, with one preferred
    #  - some people are PT Fac, FT Fac, Director, assistant, spec, and some titles are unknown. 
    #  - sometimes the hierarchy is of departments, and sometimes of people. try not to confuse that.
    #
    
    
    # eventually, want to get pics or other info from other sources too, o365, cranium cafe, etc
    #
    


def manualNames():
    mm = dict([ x.strip().split(',') for x in \
            open('cache/teacherdata/teacher_manual_name_lookup.csv','r').readlines()])  
    mz = {}
    for k,v in mm.items():
        mz[k] = v
        mz[k.lower()] = v
        parts = k.split(" ")
        if len(parts)==2:
            mz[  parts[1] + ", " + parts[0] ] = v
            mz[  parts[1] + "," + parts[0] ] = v
    #print(mz)
    return mz

# given a list of class codes, return the most common (academic) department
def guessDept(d_list):
    li = str(d_list.code).split(" ")
    count = defaultdict(int)
    #print(str(d_list.code))
    for i in li:
        m = re.search(r'^([A-Z]+)$',i)
        if m:
            count[m.group(1)] += 1
    mmax = 0
    max_L = ''
    for k,v in count.items():
        #print("  %s:%i,   " % (k,v), end='')
        if v > mmax: 
            mmax = v
            max_L = k
    print("")
    return max_L

"""
# Faculty Info Plans



bootcamp_active.csv				Started bootcamp. Remind them to finish it?

bootcamp_passed.csv				Badge'd for BC. Online and Hybrid teachers not on this list need reminding. 

courses_taught.csv				x

course_teacher_combos.csv		Teachers who share the teaching of a course. Courses in common. 

emails_deans+chairs.txt         Just a email list

FA2017 Faculty Survey.csv       Look at answers for video, helpful formats, and comments

faculty_main_info.csv           Has percentage mix of a teachers' online/hybrid/lecture history

historical_shells_used.json     x

SP2019 Faculty Survey.csv       Look at rate tech skills, topics interested in, would add video, and comments

committees 2018 2019.csv       Committees people serve on.



Not so useful:

teacher_by_semester.csv         precursor to faculty_main_info. Has semesters separated.   

"""
#
#
#
# Call all the teacher info / CRM gathering stuff
# Make one big csv file of everything I know about a teacher    
def getTeachersInfoMain():

    schedule_one_yr = oneYearSchedule()
    #print(schedule_one_yr)
    #if input('q to quit ')=='q': return
    
    # comes from  teacherRolesUpdateCache  ...  search for  @gavilan.edu in email address
    from_ilearn = list(  map(  lambda y: funcy.select_keys( lambda z: z in ['name','id','email','login_id','sortable_name'], y), \
                    json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) ) )
    #names_from_ilearn = list( [x.lower() for x in map( str, sorted(list(funcy.pluck('name',from_ilearn)))) ] )
    from_ilearn_df = pd.DataFrame(from_ilearn)
    
    
    manual_names = manualNames()
    names_lookup = funcy.partial(find_ilearn_record, from_ilearn, manual_names)
    #print(from_ilearn_df)
    #if input('q to quit ')=='q': return
    
    
    #print(schedule_one_yr)
    #print("This is one year schedule.")
    #input('\npress enter to continue')
    
    teacher_info = teacher_basic_info(schedule_one_yr, from_ilearn_df, names_lookup)
    #print(teacher_info)
    #input('\nThis is teacher info.\npress enter to continue')

    modality_history = teacherModalityHistory(schedule_one_yr,names_lookup)
    print(modality_history)
    #print("This is teacher modality history.")
    #input('\npress enter to continue')

    
    master = pd.merge( modality_history, teacher_info, on='teacher', how='outer')
    print(master)
    master.to_csv('cache/trash/joined1.csv')
    print(master.columns)
    #input('\nThis is Joined 1.\npress enter to continue')
    
    wp = read_bootcamp1('cache/teacherdata/bootcamp_passed.csv')
    #print(wp)
    master2 = pd.merge( master, wp, left_on='canvasid_x', right_on='bc1canvasid', how='outer')
    master2.to_csv('cache/trash/joined2.csv')
    print(master2)
    print(master2.columns)
    #input('\nThis is Joined 2.\npress enter to continue')
    
    
    wp = read_bootcamp2('cache/teacherdata/bootcamp_active.csv')
    master3 = pd.merge( master2, wp, left_on='canvasid_x', right_on='bc2canvasid', how='outer')
    master3.to_csv('cache/trash/joined3.csv')
    print(master3)
    print(master3.columns)
    #input('\nThis is Joined 3.\npress enter to continue')
    
    
    # THE VIEWS / HISTORY. UPDATE with    get_recent_views()   .... check it for appropriate dates....
    views = json.loads( codecs.open('cache/teacherdata/activitysummary.json','r','utf-8').read() )
    vdf = pd.DataFrame.from_dict(views,orient='index',columns=['cid','cname','views','goo','dates','dateviews'])
    print(vdf)
    #input('k')
    
    #master3.set_index('canvasid_x')
    master3 = pd.merge(master3, vdf, left_on='canvasid_x', right_on='cid',how='outer')
    
    dir_records = pd.DataFrame(staff_dir())
    dir_records['email'] = dir_records['email'].str.lower()
    master3['email'] = master3['email'].str.lower()
    
    print(dir_records)
    master3 = pd.merge(master3, dir_records, on='email',how='outer')
    print(master3)
    #if input('q to quit ')=='q': return
    
    #master3.fillna(0, inplace=True)
    #master3['views'] = master3['views'].astype(int)
    #master3['num_sections_last_year'] = master3['num_sections_last_year'].astype(int)
    
    
    #cmte = pd.read_csv('cache/teacherdata/committees_2018_2019.csv')
    cmte,cmte_by_name = read_cmte(names_lookup)
    cmte_str_by_name = {}
    for k in cmte_by_name.keys():
        #print(k)
        #print(cmte_by_name[k])
        cmte_str_by_name[k] = ",".join(cmte_by_name[k])
    cc = pd.DataFrame.from_dict(cmte_str_by_name,orient='index',columns=['committees'])   # 'teacher',
    cc.reset_index(inplace=True)
    master4 = pd.merge(master3, cc, left_on='name', right_on='index', how='outer')
    master4.to_csv('cache/trash/joined4.csv')
    
    master4.drop(['teacher','ilearn_name','canvasid_y','bc1canvasid','bc2canvasid','cid','cname','index_y'],axis=1,inplace=True)
    
    # Exclude surveys for now
    """
    survey_2017 = pd.read_csv('cache/teacherdata/FA2017 Faculty Survey.csv')
    survey_2017 = add_realnames(survey_2017,names_lookup)
    survey_2017.to_csv('cache/trash/survey1.csv')
    master5 = pd.merge(master4, survey_2017, left_on='name', right_on='ilearn_name', how='left')
    master5.to_csv('cache/trash/joined5.csv')
    
    survey_2019 = pd.read_csv('cache/teacherdata/SP2019 Faculty Survey.csv')
    survey_2019 = add_realnames(survey_2019,names_lookup)
    master6 = pd.merge(master5, survey_2019, left_on='name', right_on='ilearn_name', how='left')
    master6.to_csv('cache/trash/joined6.csv')


    newnames = [ x.strip() for x in open('cache/poll_question_names.txt','r').readlines() ]
    namedict = {}
    for i,n in enumerate(newnames):
        if i%3==1: newname = n
        if i%3==2: namedict[oldname] = newname
        if i%3==0: oldname = n
    master6 = master6.rename(columns=namedict)
    master6.to_csv('cache/teacherdata/staff_main_table.csv')
    master6.to_csv('cache/teacherdata/staff_main_table.csv')
    """
    
    
    master4.to_csv('cache/teacherdata/staff_main_table.csv')
    master4.to_csv('gui/public/staff_main_table.csv')    

    other_training_records = read_training_records()
    #print(json.dumps(other_training_records,indent=2))
    #print("This is misc workshops.")
    tt = pd.DataFrame.from_dict(other_training_records,orient='index')
    tt = tt.fillna("")
    #print(tt)
    #input('\npress enter to continue')
    
    
   
    #teacherSharedCourses(schedule_one_yr)
    #getAllTeachersInTerm()
    

# TODO - broken

def enroll_staff_shell():
    pass
    """staff = users_with_gavilan_email()
    for i,s in staff.iterrows():
        print(s['canvasid'],s['name'])
        u = url + '/api/v1/courses/8528/enrollments'
        param = {
            'enrollment[user_id]':s['canvasid'],
            'enrollment[type]': 'StudentEnrollment',
            'enrollment[enrollment_state]': 'active',
        }
         
        res = requests.post(u, headers = header, data=param)
        print(res.text)
    """


#"Jun 28 2018 at 7:40AM" -> "%b %d %Y at %I:%M%p"
#"September 18, 2017, 22:19:55" -> "%B %d, %Y, %H:%M:%S"
#"Sun,05/12/99,12:30PM" -> "%a,%d/%m/%y,%I:%M%p"
#"Mon, 21 March, 2015" -> "%a, %d %B, %Y"
#"2018-03-12T10:12:45Z" -> "%Y-%m-%dT%H:%M:%SZ"


# take a list of raw hits. 
def activity_summary(hits):
    #infile = "cache/teacherdata/activity/G00101483.json"
    #data = json.loads(open(infile,'r').read())
    #hits = data['raw']
    if not hits:
        return [ [], [], ]
    dt_list = []
    
    one_week = datetime.timedelta(days=14)                       # actually two....
    today = dt.now().replace(tzinfo=pytz.timezone('UTC'))
    
    target = today - one_week
    
    for h in hits:
        the_stamp = parser.parse(h['created_at'])
        if the_stamp > target:
            dt_list.append(the_stamp)
    df = pd.DataFrame(dt_list,  columns=['date',])
    df.set_index('date', drop=False, inplace=True)
    df.rename(columns={'date':'hits'}, inplace=True)
    #df.resample('1D').count().plot(kind='bar')
    #return df.resample('1D').count().to_json(date_format='iso')
    #print(hits)
    #print(df)
    if not df.size:
        return [ [], [], ]
    bins = df.resample('1D').count().reset_index()
    bins['date'] = bins['date'].apply(str)
    #print(bins)
    return [bins['date'].to_list(), bins['hits'].to_list()]
    
    #plt.show()
    
    #df = df.groupby([df['date'].dt.to_period('D')]).count().unstack()
    #df.groupby(TimeGrouper(freq='10Min')).count().plot(kind='bar')
    #df.plot(kind='bar')



# next step
# 1. save timestamp of the fetch
#
# 2. parse it and only fetch since then. afterwards, pull out non-hits. Summarize day/week/month stats. 
#
# 2a. merge old and new records, and re-summarize.
#
# 3. Next improvements in GUI. hook up to python server backend.
#
# Get views counts on current teachers. todo: month is hardcoded here
def get_recent_views(id=1):
    dt_format = "%Y-%m-%dT%H:%M:%SZ"
    default_start_time = dt.strptime("2020-08-14T00:00:00Z", dt_format)
    default_start_time = default_start_time.replace(tzinfo=pytz.timezone('UTC'))
    end_time = dt.now(pytz.utc)
    print("End time is: %s" % str(end_time))
    myheaders = "x,teacher,prct_online,prct_lecture,prct_hybrid,num_sections_last_year,canvasid_x,name,sortable_name,goo,email,index_x,courses,dept,ilearn_name_x,canvasid_y,canvasid_x,bootcamp_grade,bootcamp_date_x,canvasid_y,bootcamp_progress,bootcamp_date_y,index_y,committees".split(",")
    
    teachers = [row for row in csv.reader(open('cache/teacherdata/staff_main_table.csv','r'))][1:]
    
    #tt = teachers[6:10]
    
    summary = {}
    
    for t in teachers:
        name = t[1]
        if name=="" or name=="TBA": continue
        if not t[6]: continue
        the_id = int(float(t[6]))
        if the_id == 290: continue   # STAFF STAFF
        goo = t[9]
        print(goo)
        
        # read log of this person:
        try:
            prev_logf = codecs.open('cache/teacherdata/activity/%s.json' % goo,'r','utf-8')
            prev_log = json.loads(prev_logf.read())
            prev_logf.close()
        except:
            print("Exception happened on reading previous temp logs.")
            prev_log = ''
        
        if type(prev_log) == dict:
            lastfetch = dt.strptime(prev_log['meta']['lastfetch'], dt_format)
            lastfetch = lastfetch.replace(tzinfo=pytz.timezone('UTC'))
            print("last fetch is: " + str(lastfetch))
            print("Hits BEFORE was: %i" % len(prev_log['raw']))
        else:
            lastfetch = default_start_time
            prev_log = { "raw":[], }
        
        end_time = dt.now(pytz.utc)
        u = url + "/api/v1/users/%s/page_views?start_time=%s&end_time=%s&per_page=100" % (str(the_id),lastfetch.strftime(dt_format), end_time.strftime(dt_format))
        #print(u)
        #input('getting this url')
        
        print(name + "\t",end='\n')
        if 1:    # get fresh data?
            r = fetch(u)
            prev_log['raw'].extend( r )
        summ = activity_summary(prev_log['raw'])
        mydata = {'meta':{'lastfetch':end_time.strftime(dt_format)},'summary':summ,'raw':prev_log['raw']}
        codecs.open('cache/teacherdata/activity/%s.json' % goo,'w','utf-8').write( json.dumps(mydata,indent=2))
        summary[the_id] = [the_id, name, len(prev_log['raw']),goo, summ ,mydata['meta']]
        print("Hits AFTER is: %i" % len(prev_log['raw']))
    codecs.open('cache/teacherdata/activitysummary.json','w','utf-8').write( json.dumps(summary,indent=2) )
    codecs.open('gui/public/activitysummary.json','w','utf-8').write( json.dumps(summary,indent=2) )


# TODO broken?
# Have they taught online or hybrid classes?        
"""
def categorize_user(u):
    global role_table, term_courses
    their_courses = get_enrlmts_for_user(u, role_table)
    num_s = 0
    num_t = 0
    type = 's'
    online_only = 1
    is_online = []
    #print their_courses
    for x in their_courses.iterrows():
        if len(x):
            ttype = x[1]['type']
            if ttype=='StudentEnrollment': num_s += 1
            if ttype=='TeacherEnrollment': num_t += 1
            cid = x[1]['course_id']
            current_term = term_courses[lambda x: x['id']==cid]
            if not current_term.empty:
                is_online.append(current_term['is_online'].values[0])
            else: online_only = 0
        else: online_only = 0
    if num_t > num_s: type='t'
    if len(is_online)==0: online_only = 0
    
    for i in is_online: 
        if i==0: online_only = 0
    #print "Type: " + type + " All online: " + str(online_only) + " Number courses this term: " + str(len(is_online))
    return (u[0],type, online_only, len(is_online))
"""

        
##########
##########
##########    PHOTOS
##########
##########    # todo: threaded

# Doest the account have a photo loaded?
def checkForAvatar(id=2):
    try:
        t = url + '/api/v1/users/%s?include[]=last_login' % str(id)
        r2 = requests.get(t, headers = header)
        result = json.loads(r2.text)
        codecs.open('cache/users/%s.txt' % str(id),'w','utf-8').write( json.dumps(result,indent=2) )
        
        if 'avatar_url' in result:
            if re.search(r'avatar\-50',result['avatar_url']): return 0 
            else: return (result['login_id'], result['avatar_url'], result['name'])
    except Exception as e:
        print("Looking for an avatar / profile pic had a problem: %s" % str(e))
    return 0

# Grab em. Change the first if when continuing after problems....
def downloadPhoto():
    import imghdr
    pix_dir = 'cache/picsCanvas2022/'
    # Update the list of all ilearn users?
    i_last_ix = '-1'
    photo_log_f = ''
    if 0:                           ##  CHANGE TO 0 IF CRASHED / RESUMING....
        ii = fetchAllUsers()
        photo_log_f = open("cache/fotolog.txt", "w")
    else:
        ii = json.loads(codecs.open('cache/allusers_ids.json','r').read())
        photo_log_f = open("cache/fotolog.txt", "r+")
        i_last_ix = -1
        try:
            ab = photo_log_f.read()
            print(ab)
            ac = ab.split("\n")
            print(ac)
            i_last_ix = ac[-2]
            print(i_last_ix)
        except:
            i_last_ix = -1
        i_last_ix = int(i_last_ix)

    
    print("Last user index checked was: %s, which is id: %s" %  \
            (i_last_ix, ii[i_last_ix] ))

    print("Max index is: %i" % len(ii))


    i_last_ix += 1
    for index in range(i_last_ix, len(ii)):
        i = ii[index]
        photo_log_f.write("\n%i" % i )

        a = checkForAvatar(i)
        if a:
            print(str(i) + ":\t" + str(a[0]) + "\t" + str(a[2]) )
            
            try:
                r = requests.get(a[1], stream=True)
                if r.status_code == 200:
                    r.raw.decode_content = True
                    h=r.raw
                    with open(pix_dir + a[0].lower(), 'wb') as f:
                        shutil.copyfileobj(h, f) 
                    # rename to right file extension          
                    # TODO: Change imghdr to PILLOW (PIL)
                    #img = Image.open(filename)
                    #img_type = img.format  # 'JPEG'
                    img_type = imghdr.what(pix_dir + a[0].lower())
                    if img_type == 'jpeg': img_type = 'jpg'
                    try:
                        shutil.move(pix_dir + a[0].lower(),pix_dir + a[0].lower()+'.'+img_type)
                    except Exception as e:
                        print("   \tCouldn't rewrite file")
                else:
                    print(str(i) + ":\t didn't get expected photo")
            except Exception as e:
                print("   \tProblem with download " + str(e))
        else:
            print(str(i) + ":\tno user or no photo")
            pass
            

def mergePhotoFolders():
    
    staff = [ row for row in csv.reader(  open('cache/teacherdata/staff_main_table.csv','r') ) ]
    
    headers = staff[0]
    staff = staff[1:]
    
    activestaff = []
    
    for i,h in enumerate(headers):
        #print("%i. %s" % (i,h) )
        pass
    
    for S in staff:
        if S[7] and S[15]:   #   if teacher (name present) and sp20crns (taught in sp20)
            activestaff.append(S[9].lower())
    activestaffset=set(activestaff)
    
    #return
    
    a = 'cache/picsCanvas'
    b = 'gui/public/picsCanvas2018'
    c = 'gui/public/picsCanvasAll'
    
    
    # I want a big list of who has an avatar pic. 
    
    # and i want to know how many updated since last DL, and how many are in only one or the other. 
    
    
    old = os.listdir(b)
    count = defaultdict(int)
    
    oldset = set()
    newset = set()
    
    for O in old:
        if O.endswith('.jpg') or O.endswith('.png'):
            g = O.split(r'.')[0]
            oldset.add(g)
    
    for N in os.listdir(a):
        if N.endswith('.jpg') or N.endswith('.png'):
            g = N.split(r'.')[0]
            newset.add(g)
    
    """print("Active SP20 Teachers")
    print(activestaffset)

    print("Old Avatars")
    print(oldset)

    print("New Avatars")
    print(newset)"""
    
    updated_set = oldset.union(newset)
    
    tch_set = updated_set.intersection(activestaffset)
    
    only_old = oldset.difference(newset)
    
    only_new = newset.difference(oldset)
    
    print("Tch: %i  Old: %i New: %i" % (len(activestaffset),len(oldset),len(newset)))
    
    print("All avatars: %i  Teachers: %i   Only in old: %i   Only in new: %i" % (  len(updated_set), len(tch_set), len(only_old), len(only_new)))

    allpics = os.listdir(c)
    
    haveapic = {}
    for A in allpics:
        if A.endswith('.jpg') or A.endswith('.png'):
            g = (A.split(r'.')[0]).upper()
            
            haveapic[g] = A
    outie = codecs.open('gui/public/pics.json','w').write( json.dumps( haveapic,indent=2))
            
        
def mergePhotoFolders2():
    
    staff = [ row for row in csv.reader(  open('cache/teacherdata/staff_main_table.csv','r') ) ]
    
    headers = staff[0]
    staff = staff[1:]
    
    activestaff = []
    
    for i,h in enumerate(headers):
        #print("%i. %s" % (i,h) )
        pass
    
    for S in staff:
        if S[5]:
            activestaff.append(S[9].lower())
    
    a = 'cache/picsCanvas'
    b = 'gui/public/picsCanvas2018'
    c = 'gui/public/picsCanvasAll'
    
    old = os.listdir(b)
    count = defaultdict(int)
    for N in os.listdir(a):
        if N.endswith('.jpg') or N.endswith('.png'):
            g = N.split(r'.')[0]
            if g in activestaff:
                count['s'] += 1
            if N in old:
                #print( "Y - %s" % N)
                count['y'] += 1
            else:
                #print( "N - %s" %N )
                count['n'] += 1
        else:
            #print("x - %s" % N)
            count['x'] += 1
    print("Of the 2020 avatars, %i are in the 2018 folder, and %i are new." % (count['y'],count['n']))
    print("Of %i active teachers, %i have avatars." % (len(activestaff),count['s']))
    #print(json.dumps(count,indent=2))
        
    

# Go through my local profile pics, upload any that are missing.
def uploadPhoto():
    files = os.listdir('pics2017')
    #print json.dumps(files)
    pics_i_have = {}
    #goo = "g00188606"
    canvas_users = json.loads(open('canvas/users.json','r').read())
    t = url +  '/api/v1/users/self/files'
    i = 0
    j = 0
    pics_dir = 'pics2017/'
    
    for x in canvas_users:
        j += 1
        if x['login_id'].lower() + '.jpg' in files:
            #print x['login_id'] + " " + x['name']
            i += 1
            pics_i_have[x['id']] = x
    
    print('Canvas users: ' + str(j))
    print('Pic matches: ' + str(i))
    account_count = 0
    ids_i_uploaded = []
    
    for id, target in list(pics_i_have.items()):
        #if account_count > 50:
        #    print 'Stopping after 5.'
        #    break
            
        print('trying ' + target['name'] + '(' + str(id) + ')')
        if checkForAvatar(id):
            print("Seems to have avatar loaded.")
            continue
        
        goo = target['login_id'].lower()
        local_img = pics_dir + goo + '.jpg'
        inform_parameters = {
            'name':goo + '.jpg',
            'size':os.path.getsize(local_img), # read the filesize
            'content_type':'image/jpeg',
            'parent_folder_path':'profile pictures',
            'as_user_id':'{0}'.format(id)
        }
         
        res = requests.post(t, headers = header, data=inform_parameters)
        print("Done prepping Canvas for upload, now sending the data...")
        json_res = json.loads(res.text,object_pairs_hook=collections.OrderedDict)
        files = {'file':open(local_img,'rb').read()}
      
        _data = list(json_res.items())
        _data[1] = ('upload_params',list(_data[1][1].items()))
        print("Yes! Done sending pre-emptive 'here comes data' data, now uploading the file...")
        upload_file_response = requests.post(json_res['upload_url'],data=_data[1][1],files=files,allow_redirects=False)
        # Step 3: Confirm upload
        print("Done uploading the file, now confirming the upload...")
        confirmation = requests.post(upload_file_response.headers['location'],headers=header)
        if 'id' in confirmation.json():
            file_id = confirmation.json()['id'] 
        else:
            print('no id here')
        #print(confirmation.json())
        print("upload confirmed...nicely done!")
        
        time.sleep(1)
        # Make api call to set avatar image to the token of the uploaded imaged (file_id)
        params = { 'as_user_id':'{0}'.format(id)}
        avatar_options = requests.get(url + "/api/v1/users/%s/avatars"% '{0}'.format(id),headers=header,params=params)
        #print "\nAvatar options: " 
        #print avatar_options.json()
        for ao in avatar_options.json():
            #print ao.keys()
            if ao.get('display_name')==goo + '.jpg':
                #print("avatar option found...")
                #print((ao.get('display_name'),ao.get('token'), ao.get('url')))
                params['user[avatar][token]'] = ao.get('token')
                set_avatar_user = requests.put(url + "/api/v1/users/%s"% '{0}'.format(id),headers=header,params=params)
                if set_avatar_user.status_code == 200:
                    print(('success uploading user avatar for {0}'.format(id)))
                    account_count += 1
                    ids_i_uploaded.append(id)
                else:
                    print('some problem setting avatar')
            else:
                pass #print 'didnt get right display name?'
    print("Uploaded these guys: " + json.dumps(ids_i_uploaded))
    



##########
##########
##########    EMAILING PEOPLE
##########
##########



#def test_email():
#    send_z_email("Peter Howell", "Peter", "phowell@gavilan.edu", ['CSIS85','CSIS42'])
    

def create_ztc_list():
    course_combos = pd.read_csv('cache/teacher_course_oer_email_list.csv')
    course_combos.fillna('',inplace=True)
    
    # read this file and make it a dict (in one line!)
    dept_counts = { x[0]:x[1].strip() for x in [ y.split(',') for y in open('cache/teacher_course_oer_deptcount.csv','r').readlines() ][1:] }
    
    
    course_template = "<a href='%s'>%s</a> &nbsp; &nbsp;"
    url_template = "https://docs.google.com/forms/d/e/1FAIpQLSfZLQp6wHFEdqsmpZ7jz2Y8HtKLo8XTAhrE2fyvTDOEgquBDQ/viewform?usp=pp_url&entry.783353363=%s&entry.1130271051=%s"    #   % (FULLNAME, COURSE1)



    # list depts
    mydepts = sorted(list(set(course_combos['dept'] )))
    i = 0
    outp = open("output/oer_email_list.csv","w")
    outp.write("fullname,firstname,email,link,courses\n")
    
    ones_i_did = [ int(x) for x in "40 38 31 21 7 12 24 25 1 13 18 22 44 55 56 51 20 16 2 3 4 5 6 8 9 10 11 14 15 17 23 53 52 50 30 48 39 37 54 49 47 46 45 43 42 41 33 32 29 28 27 26".split(" ") ]
    
    for D in mydepts:
        i += 1
        extra = ''
        if D in dept_counts:
            extra = " (%s)" % dept_counts[D]
        extra2 = ''
        if i in ones_i_did: 
            extra2 = "xxxx "
        print("%s  %i. %s %s" % (extra2,i,D,extra))
    choice_list = input("Which department? (for multiple, separate with spaces) ").split(' ')
    
    all_people_df = []
    
    for choice in choice_list:
        is_cs = course_combos['dept']==mydepts[int(choice)-1] 
        filtered =  pd.DataFrame(course_combos[is_cs])
        if len(all_people_df): all_people_df = pd.concat([filtered,all_people_df])
        else: all_people_df = filtered
        print(mydepts[int(choice)-1])
        print(all_people_df)
        print(' ')
    all_people_df.sort_values(by=['name'],inplace=True)    
    print(all_people_df)
    
    b = all_people_df.groupby(['name'])
    for name,group in b:
        if name == 'no data': continue
        nameparts = name.split(', ')
        fullname = nameparts[1] + ' ' + nameparts[0]
        firstname = nameparts[1]
        
        outp.write(fullname + ',' + firstname + ',')
        email = ''
        link = ''
        courses = []
        flag = 1
        for i in group.iterrows():
            g = i[1]   # wtf is this shi.....
            this_course = g.dept + ' ' + str(g.codenum) + g.codeletter
            courses.append( this_course ) #print(g)
            email = g.email
            if flag:
                link = url_template % (fullname, this_course)
            flag = 0
        
        outp.write(email + ',' + link + "," + "    ".join(courses) + "\n")
        
    outp.close()
    

##########
##########
##########    FORENSICS TYPE STUFF
##########
##########

# better name for this standard fetch. so they stay together in alpha order too....

def get_user_info(id):
    u = fetch( '/api/v1/users/%i' % id )
    ff = codecs.open('cache/users/%i.txt' % id, 'w', 'utf-8')
    ff.write( json.dumps(u, indent=2))
    return u


# these are any messages that get pushed out to their email
def comm_mssgs_for_user(uid=0):
    if not uid:
        uid = input('Canvas id of the user? ')
    u = url + '/api/v1/comm_messages?user_id=%s&start_time=%s&end_time=%s' % (uid,'2021-01-01T01:01:01Z','2021-08-01T01:01:01Z')     # &filter[]=user_%s' % uid
    convos = fetch(u,1)
    
    oo = codecs.open('cache/comms_push_user_%s.txt' % str(uid), 'w')
    oo.write('USER %s\n' % uid)
    oo.write(json.dumps(convos, indent=2))
    
    print(convos)


#
def convos_for_user(uid=0):
    if not uid:
        uid = input('Canvas id of the user? ')
    u = url + '/api/v1/conversations?include_all_conversation_ids=true&as_user_id=%s' % uid     # &filter[]=user_%s' % uid
    convos = fetch(u,1)
    
    oo = codecs.open('cache/convo_user_%s.txt' % str(uid), 'w')
    oo.write('USER %s\n' % uid)
    oo.write(json.dumps(convos, indent=2))
    
    convo_ids_list = convos["conversation_ids"]
    print(convo_ids_list)
    
    u2 = url + '/api/v1/conversations?include_all_conversation_ids=true&scope=archived&as_user_id=%s' % uid     # &filter[]=user_%s' % uid
    archived_convos = fetch(u2,1)
    try:
        aconvo_ids_list = archived_convos["conversations_ids"]
        print(aconvo_ids_list)
    except:
        print("didnt seem to be any archived.")
        aconvo_ids_list = []
    
    u3 = url + '/api/v1/conversations?include_all_conversation_ids=true&scope=sent&as_user_id=%s' % uid     # &filter[]=user_%s' % uid
    sent_convos = fetch(u3,1)
    try:
        sconvo_ids_list = sent_convos["conversations_ids"]
        print(sconvo_ids_list)
    except:
        print("didnt seem to be any sent.")
        sconvo_ids_list = []
    
    convo_ids_list.extend(aconvo_ids_list)
    convo_ids_list.extend(sconvo_ids_list)
    
    
    ##
    ## Now get all the messages in each of these conversations
    ##
    
    for cid in convo_ids_list:
        print("Fetching conversation id: %s" % cid)
        oo.write("\n\n----------------\nconversation id: %s\n\n" % cid)
        
        u4 = url + '/api/v1/conversations/%s?as_user_id=%s' % (cid,uid)       #     ' % (cid, uid
        coverstn = fetch(u4,1)
        oo.write("\n%s\n\n" % json.dumps(coverstn,indent=2))
        
        
    
    
    
    """
    for c in convos:
        c['participants'] = ", ".join([ x['name'] for x in c['participants'] ])
    includes = tuple("last_message subject last_message_at participants".split(" "))
    convos = list( \
            reversed([ funcy.project(x, includes) for x in convos ]))
    """
    
    #
    
    #print(json.dumps(convos, indent=2))


# single q sub
def quiz_get_sub(courseid, quizid, subid=0):
    u = url + "/api/v1/courses/%s/quizzes/%s/submissions/%s" % ( str(courseid), str(quizid), str(subid) )

    u = url + "/api/v1/courses/%s/quizzes/%s/questions?quiz_submission_id=%s" % \
            ( str(courseid), str(quizid), str(subid) )

    u = url + "/api/v1/courses/%s/assignments/%s/submissions/%s?include[]=submission_history" % \
            ( str(courseid), str(quizid), str(subid) )

    u = url + "/api/v1/courses/%s/students/submissions?student_ids[]=all&include=submission_history&grouped=true&workflow_state=submitted" % str(courseid)
    return fetch(u)

    #?quiz_submission_id=%s"  

# quiz submissions for quiz id x, in course id y
def quiz_submissions(courseid=9768, quizid=32580):
    #subs = quiz_get_sub(courseid, quizid)
    #print( json.dumps( subs, indent=2 ) )

    if 1:
        # POST
        data = { "quiz_report[includes_all_versions]": "true", "quiz_report[report_type]": "student_analysis" }

        u = url + "/api/v1/courses/%s/quizzes/%s/reports?" % ( str(courseid), str(quizid) )
        res = requests.post(u, headers = header, data=data)
        print(res.content)

    #u2 = url + "/api/v1/courses/%s/quizzes/%s/reports" % ( str(courseid), str(quizid) )
    #res2 = fetch(u2)
    #print( json.dumps(res2.content, indent=2))

    jres2 = json.loads( res.content )
    print(jres2)
    if jres2['file'] and jres2['file']['url']:
        u3 = jres2['file']['url']
        r = requests.get(u3, headers=header, allow_redirects=True)
        open('cache/quizreport.txt', 'wb').write(r.content)
    return

    for R in res2:
        if R['id'] == 7124:
            u3 = R['url']
            r = requests.get(u3, headers=header, allow_redirects=True)
            open('cache/quizreport.txt', 'wb').write(r.content)
    return

    u3 = url + "/api/v1/courses/%s/quizzes/%s/reports/%s" % ( str(courseid), str(quizid), res2[''] )

    oo = codecs.open('cache/submissions.json','w', 'utf-8')
    oo.write('[\n')
    for s in subs:
        if len(s['submissions']):
            j = json.dumps(s, indent=2) 
            print(j)
            oo.write(j)
            oo.write('\n')

    oo.write('\n]\n')
    return 0


    #u = url + "/api/v1/courses/%s/quizzes/%s/submissions?include[]=submission" % (str(courseid), str(quizid))
    u = url + "/api/v1/courses/%s/quizzes/%s/submissions" % (str(courseid), str(quizid))
    subs = fetch(u, 0)
    print( json.dumps( subs, indent=1 ) )

    for S in subs['quiz_submissions']:
        print(json.dumps(S))
        submis = quiz_get_sub(courseid, quizid, S['id'])
        print(json.dumps(submis, indent=2))



# return (timeblock, course, read=0,write=1)
def requests_line(line,i=0):
    try:
        L = line  # strip?
        if type(L) == type(b'abc'): L = line.decode('utf-8')
        for pattern in unwanted_req_paths:
            if pattern in L: 
                return 0
        i = 0
        line_parts = list(csv.reader( [L] ))[0]
        #for p in line_parts:
        #    print("%i\t%s" % (i, p))
        #    i += 1

        d = parser.parse(line_parts[7])
        d = d.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
        d = timeblock_24hr_from_dt(d)

        #r = re.search('context\'\:\s(\d+)', line_parts[22])
        #c = 0
        #if r:
        #    c = r.groups(1)
        str1 = line_parts[20]
        str2 = str1.replace("'",'"')
        str2 = str2.replace("None",'""')
        #print(str2)
        j = json.loads(str2  )
        c = j['context']
        a = line_parts[5]
        #print( str( (d, c, a) ))
        return (d, str(c), a)
    except Exception as e:
        #print("Exception: " + str(e))
        return 0


# 
def report_logs(id=0):
    if not id:
        L = ['10531', ]
    else:
        L = [ id, ]
    report = []
    for id in L:
        emt_by_id = course_enrollment(id)
        for U in emt_by_id.values():
            user_d = defaultdict( int )
            print( "Lookin at user: %s" % U['user']['name'] )
            report.append( "User: %s\n" % U['user']['name'] )
            log_file_name = 'cache/users/logs/%i.csv' % U['user']['id']
            if path.exists(log_file_name):
                print("Log file %s exists" % log_file_name)
                temp = open(log_file_name, 'r').readlines()
                for T in temp[1:]:
                    #print(T)
                    result = requests_line(T)
                    if result:
                        (d, c, a)  = result
                        if c == id:
                            user_d[d] += 1
            print(json.dumps(user_d, indent=2))
            for V in sorted(user_d.keys()):
                report.append( "\t%s: %i\n" % ( dt_from_24hr_timeblock(V), user_d[V]) )
        report.append("\n\n")
    return report


def track_users_in_sem():
    L = users_this_semester_db()
    sL = list(L)
    sL.sort(reverse=True)
    fetch_queue = queue.Queue()
    
    for i in range(num_threads):
        worker = Thread(target=track_user_q, args=(i,fetch_queue))
        worker.setDaemon(True)
        worker.start()

    for U in sL:
        print( "adding %s to the queue" % U )
        fetch_queue.put( U )

    fetch_queue.join()
    print("Done.")


def track_users_in_class(L=[]):
    if len(L)==0:
        #id = '10531'
        ids = input("Course ids, separated with comma: ")
        L = [x for x in ids.split(',')]
    print("Getting users in: " + str(L))

    fetch_queue = queue.Queue()
    
    for i in range(num_threads):
        worker = Thread(target=track_user_q, args=(i,fetch_queue))
        worker.setDaemon(True)
        worker.start()
    

    users_set = set()
    for id in L:
        emt_by_id = course_enrollment(id)
        print(emt_by_id)
        for U in emt_by_id.values():
            if not U['user_id'] in users_set:
                print(U)
                print( "adding %s to the queue" % U['user']['name'] )
                fetch_queue.put( U['user_id'] )
                users_set.add(U['user_id'])

    all_reports = []
    fetch_queue.join()
    print("Done with %i users in these courses." % len(users_set))
    for id in L:
        rpt = report_logs(id)
        all_reports.append(rpt)
        outp = codecs.open('cache/courses/report_%s.txt' % id, 'w', 'utf-8')
        outp.write(''.join(rpt))
        outp.close()
    return all_reports

def track_user_q(id, q):
    while True:
        user = q.get()
        print("Thread %i: Going to download user %s" % (id, str(user)))
        try:
            track_user(user, id)
        except FetchError as e:
            pass
        q.task_done()


# honestly it doesn't make much sense to get full histories this way if they're
# already in the canvas data tables....

# just the most recent hits or a short period
# 
# Live data would be better.

# Maintain local logs. Look to see if we have some, download logs since then for a user.
def track_user(id=0,qid=0):
    global recvd_date 
    L = [id,]
    if not id:
        ids = input("User ids (1 or more separated by comma): ")
        L = [int(x) for x in ids.split(',')]
    print("Getting users: " + json.dumps(L))


    for id in L:
        id = int(id)
        # Open info file if it exists, check for last day retrived
        try:
            infofile = open("cache/users/%i.txt" % id, 'r')
            info = json.loads( infofile.read() )

            # TODO: set up this info file if it isn't there. check any changes too. it
            # was written where?....
            infofile.close()
        except Exception as e:
            print("failed to open info file for user id %i" % id)

            info = get_user_info(id)

        print("(%i) Student %i Info: " % (qid,id))
        #print( json.dumps(info, indent=2))

        url_addition = ""
        
        if 1:    # hard code dates
            start_date = "2024-01-01T00:00:00-07:00"
            end_date   = dt.now().strftime("%Y-%m-%dT%H:%M:%S-07:00")                               # "2024-10-01T00:00:00-07:00"
            url_addition = f"?start_time={start_date}&end_time={end_date}"
        elif 'last_days_log' in info:
            print("There's existing log data for %s (%s)" % (info['name'] , info['sis_user_id']))
            print("Last day logged was: %s" % info['last_days_log'])
            url_addition = "?start_time=%s" % info['last_days_log']
            the_stamp = parser.parse(info['last_days_log']) 
            the_stamp = the_stamp.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
            now = dt.now()
            now = now.replace(tzinfo=pytz.timezone('UTC')).astimezone(pytz.timezone('US/Pacific'))
            dif = now - the_stamp
            print("It was %s ago" % str(dif))
            if the_stamp < lds_stamp:
                print("Too long, taking default")
                url_addition = "?start_time=%s" % log_default_startdate
            
            #lds_stamp = parser.parse(log_default_startdate)

##########
        else:
            url_addition = "?start_time=%s" % log_default_startdate
            #if dif.days > 1:

        url = "/api/v1/users/%i/page_views%s" % (id, url_addition)
        print(url)

        try:

            api_gen = fetch_stream(url,0)
            
            log_file_name = 'cache/users/logs/%i.csv' % id
            if path.exists(log_file_name):
                print("Log file %s exists" % log_file_name)
                temp = open(log_file_name, 'a', newline='')
                csv_writer = csv.writer(temp) 
            else:
                print("Creating new log file: %s" % log_file_name)
                temp = open(log_file_name, 'w', newline='')    ### TODO  
                csv_writer = csv.writer(temp) 
            
            
            count = 0
            for result in api_gen:
                if count == 0 and len(result):
                    header = result[0].keys() 
                    csv_writer.writerow(header) 
                    # results come in newest first....
                    recvd_date = result[0]['updated_at']
                    print("(%i) Most recent hit is %s" % (qid,recvd_date))

                count += len(result)
                indent = "    " * qid
                #print("(%i) Got %i records, %i so far" % (qid,len(result),count))
                print("(%s - %i) %s %i" % (qid, id, indent, count))
                if count > max_log_count:
                    print("Too many logs, bailing. sorry.")
                    break
           
                for R in result:
                    csv_writer.writerow(R.values()) 

            latest = parser.parse(recvd_date)
            #last_full_day = (latest - timedelta(days=1)).isoformat()
            info['last_days_log'] = recvd_date    #last_full_day

            infofile = open("cache/users/%i.txt" % id, 'w')
            infofile.write(json.dumps( info, indent=2 ))
            infofile.close()
              
            print("(%i) Output to 'cache/users/log/%i.csv'" % (qid,id))
        except FetchError as e:
            print("Getting a 502 error.")
            raise FetchError()
        except Exception as e2:
            print("Got an error receiving logs: %s" % str(e2))

#
def track_users_by_teacherclass():
    all_teachers = teachers_courses_semester()

    skip_to = "Punit Kamrah"
    skipping = 1

    grouped = funcy.group_by( lambda x: x[4], all_teachers )
    g2 = {}
    for k,v in grouped.items():
        print(k)
        if skipping and skip_to != k:
            print("skipping")
            continue
        skipping = 0

        g2[k] = list(funcy.distinct( v, 1 ))
        print("\n\n\n\n\n")
        print(k)
        print("\n\n\n\n\n")

        teacherfile = codecs.open('cache/teacherdata/reports/%s.txt' % k.replace(" ","_"),'w','utf-8')
        class_ids = funcy.lpluck(1,v)
        class_names = funcy.lpluck(2,v)
        print(class_ids)
        print(class_names)
        
        rpts = track_users_in_class(class_ids)

        for i, R in enumerate(rpts):
            teacherfile.write('\n\n\n---\n\n%s \n\n' % class_names[i])
            teacherfile.write(''.join(R))
            teacherfile.flush()
        teacherfile.close()



    print(json.dumps(g2, indent=2))
 

def nlp_sample():
    # Stream a training corpus directly from S3.
    #corpus = corpora.MmCorpus("s3://path/to/corpus")
    
    stemmer = stem.porter.PorterStemmer()
    
    strings = [
    "Human machine interface for lab abc computer applications",
    "A survey of user opinion of computer system response time",
    "The EPS user interface management system",
    "System and human system engineering testing of EPS",
    "Relation of user perceived response time to error measurement",
    "The generation of random binary unordered trees",
    "The intersection graph of paths in trees",
    "Graph minors IV Widths of trees and well quasi ordering",
    "Graph minors A survey",
]
    processed = [ [ stemmer.stem(y) for y in utils.simple_preprocess(x, min_len=4)] for x in strings]
    print(processed)
    dictionary = corpora.Dictionary( processed )
    dct = dictionary
    print(dictionary)
    
    corpus = [dictionary.doc2bow(text) for text in processed]
    
    print(corpus)
    
    # Train Latent Semantic Indexing with 200D vectors.
    lsi = models.LsiModel(corpus, num_topics=4)
    print(lsi.print_topics(-1))

    # Convert another corpus to the LSI space and index it.
    #index = similarities.MatrixSimilarity(lsi[another_corpus])
    
    tfidf = models.TfidfModel(corpus)
    
    #index = similarities.SparseMatrixSimilarity(tfidf[corpus], num_features=12)
    index = similarities.MatrixSimilarity(lsi[corpus])
    print(index)


    # Compute similarity of a query vs indexed documents.
    query = "tree graph".split()
    query_bow = dictionary.doc2bow(query)
    vec_lsi = lsi[query_bow]
    
    print(query_bow)
    print(tfidf[query_bow])
    print(vec_lsi)
    print("ok")
    
    #                  LdaMulticore
    
    lda_model = models.LdaModel(corpus=corpus,
                         id2word=dictionary,
                         random_state=100,
                         num_topics=4,
                         passes=40,
                         chunksize=1000,
                         #batch=False,
                         alpha='asymmetric',
                         decay=0.5,
                         offset=64,
                         eta=None,
                         eval_every=0,
                         iterations=100,
                         gamma_threshold=0.001,
                         per_word_topics=True)
    lda_model.save('cache/lda_model.model')
    print(lda_model.print_topics(-1))
    print(lda_model)
    
    for c in lda_model[corpus]:
        print("Document Topics      : ", c[0])      # [(Topics, Perc Contrib)]
        print("Word id, Topics      : ", c[1][:3])  # [(Word id, [Topics])]
        print("Phi Values (word id) : ", c[2][:2])  # [(Word id, [(Topic, Phi Value)])]
        print("Word, Topics         : ", [(dct[wd], topic) for wd, topic in c[1][:2]])   # [(Word, [Topics])]
        print("Phi Values (word)    : ", [(dct[wd], topic) for wd, topic in c[2][:2]])  # [(Word, [(Topic, Phi Value)])]
        print("------------------------------------------------------\n")
    
    
    sims = index[vec_lsi]
    print("ok2")
    print(list(enumerate(sims)))

    for document_number, score in sorted(enumerate(sims), key=lambda x: x[1], reverse=True):
        print(document_number, score)

'''
def nlp_sample2():
    # load english language model
    nlp = spacy.load('en_core_web_sm',disable=['ner','textcat'])

    text = "This is a sample sentence."

    # create spacy 
    doc = nlp(text)

    for token in doc:
        print(token.text,'->',token.pos_)
'''



def one_course_enrol():

    users = input("user,ids,separated,by,commas> ").split(',')
    course = input("course id> ")
    my_types = {1:'StudentEnrollment', 2:'TeacherEnrollment'}
    print(json.dumps(my_types,indent=2))
    the_type = my_types[ int(input("enrollment type> ")) ]
    # the_type = 'StudentEnrollment'  
    # the_type = 'TeacherEnrollment'
    u = f"{url}/api/v1/courses/{course}/enrollments"
    
    for user in users:
        param = {
            'enrollment[user_id]':user,
            'enrollment[type]': the_type,
            'enrollment[enrollment_state]': 'active',
        }
         
        res = requests.post(u, headers = header, data=param)
        print(res.text)


def find_new_teachers():
    filename = "cache/fa22_sched.json"
    jj = json.loads(codecs.open(filename,'r','utf-8').read())
    for J in jj:
        print( J['teacher'])
    

import traceback


def find_no_goo():

    DO_DELETE_USERS = 0
    DO_DELETE_PORTFOLIOS = 0 
    
    output = codecs.open('cache/no_goo_numbers.json','w','utf-8')
    output2 = codecs.open('cache/wrong_root_acct.json','w','utf-8')
    output3 = codecs.open('cache/wrong_sis_import_id.json','w','utf-8')
    output4 = codecs.open('cache/bad_portfolios.json','w','utf-8')
    #output5 = codecs.open('cache/bad_portfolios_detail.html','w','utf-8')
    all = []
    no_root = []
    no_sis = []
    port = []
    i = 0
    j = 0
    k = 0
    p = 0
    users = json.loads(codecs.open('cache/allusers.json','r','utf-8').read())
    for u in users: 
        if not 'login_id' in u:
            print(u['name'])
            i+=1
            all.append(u)
            user_port = []
            pp = fetch(url + '/api/v1/users/%s/eportfolios' % str(u['id']))
            for p_user in pp:
                try:
                    user_port.append(  fetch(url+'/api/v1/eportfolios/%s' % str(p_user['id'])   ) )
                    if DO_DELETE_PORTFOLIOS:
                        #output5.write("<br />deleted: <a href='https://ilearn.gavilan.edu/eportfolios/%s'>%s\n" % (str(p_user['id']),str(p_user['id']))  )
                        #output5.flush()
                        del_request = requests.delete(url + "/api/v1/eportfolios/%s"    % str(p_user['id'])  ,headers=header)
                        print(del_request.text)
                except Exception as e:
                    traceback.print_exc()
            p += len(pp)
            port.append(pp)
            
            if DO_DELETE_USERS:
                print("Deleting %s..." % u['name'])
                del_request = requests.delete(url + "/api/v1/accounts/1/users/%s"    % str(u['id'])  ,headers=header)
                print(del_request.text)
        if 'root_account' in u and u['root_account'] != "ilearn.gavilan.edu":
            no_root.append(u)
            j += 1
        if 'sis_import_id' in u and not u['sis_import_id']:
            no_sis.append(u)
            k += 1
    print("Found %i users without G numbers" % i)
    print("Found %i users with non gav root account" % j)
    print("Found %i users without sis id" % k)
    print("Found %i questionable portfolios" % p)
    output.write(  json.dumps(all,indent=2) )
    output2.write(  json.dumps(no_root,indent=2) )
    output3.write(  json.dumps(no_sis,indent=2) )
    output4.write(  json.dumps(port,indent=2) )
    

def track_a_user():
    a = input("User ID? ")
    track_user(a)
    
def compare_db_tables():
    import requests

    # Download JSON files
    url_a = 'http://www.gavilan.edu/staff/tlc/db.php?a=dir'
    url_b = 'http://www.gavilan.edu/staff/tlc/db.php?a=confusers'
    response_a = requests.get(url_a)
    print('got 1')
    response_b = requests.get(url_b)
    print('got 2')

    # Parse JSON data
    data_a = response_a.json()
    data_b = response_b.json()

    by_email_conf = {}
    for item in data_a:
        by_email_conf[item['email']] = item

    # Extract email addresses from each file
    emails_a = {item['email'] for item in data_a}
    emails_b = {item['email'] for item in data_b}

    emails_a = {item for item in emails_a if item is not None}
    emails_b = {item for item in emails_b if item is not None}

    emails_a = {item.lower() for item in emails_a}
    emails_b = {item.lower() for item in emails_b}

    # Find common emails
    common_emails = emails_a.intersection(emails_b)

    # Find distinct emails for each file
    distinct_emails_a = emails_a.difference(emails_b)
    distinct_emails_b = emails_b.difference(emails_a)

    # Print the results
    print("Common Emails:")
    for email in sorted(list(common_emails)):
        print(email)

    print("\nDistinct Emails in Staff directory:")
    for email in sorted(list(distinct_emails_a)):
        print(email)

    print("\nDistinct Emails in conf users table:")
    for email in sorted(list(distinct_emails_b)):
        print(email)
    
    out = codecs.open('cache/users_fix.sql','w','utf-8')
    for e in common_emails:
        out.write(f"update `conf_users` set `p2id`='{by_email_conf[e]['id']}' where lower(`email`)='{e}';\n")


def training_find_goos():
    from openpyxl import Workbook, load_workbook
    from openpyxl.chart import BarChart, Series, Reference
    from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
    wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
    print(wb.sheetnames)

    all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())

    records = {}
    sheets = ['GOTT1', 'GOTT2', 'GOTT4', 'GOTT5', 'GOTT6', 'HUM.STEM', 'POCR Reviewed','BOOT CAMP','TITLE V GOTT ACADEMY', 'Other Certifications']
    for sname in sheets:
        print(f"{sname}:")
        sheet = wb[sname]
        records[sname] = {}
        for row in sheet.iter_rows():
            try:
                name = row[0].value
                if row[0].value == 'G Number': continue
                lastname = name.split(' ')[-1].lower()
                goo = row[1].value
                if not goo:
                    print(f"   Missing ID: {row[0].value}")
                    for t in all_teachers:
                        if re.search(lastname, t['name'].lower()):
                            print(f"      {t['name']}: {t['sis_user_id']}")
            except Exception as e:
                pass
        print()

def cross_ref_training():
    from openpyxl import Workbook, load_workbook
    from openpyxl.chart import BarChart, Series, Reference
    from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
    wb = load_workbook("C:/Users/phowell/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
    print(wb.sheetnames)

    # Fetch from Canvas DB. Make sure its recently updated.
    # Also relies on schedule being in database. Run localcache2.courses_to_sched()
    courses = all_2x_sem_courses_teachers('202550', '202570')        # 
    #courses = all_sem_courses_teachers('202470')   


    # report for email
    report = codecs.open('cache/gott_report.txt','w','utf-8')

    # update local list of teachers from ilearn?
    RELOAD_TEACHERS = 0
    ask = input('download new list of teachers? (y/n) ')
    if ask.strip()=='y': RELOAD_TEACHERS = 1

    if RELOAD_TEACHERS:
        teacherRolesUpdateCache()
    
    # TODO inefficient but just read it again
    all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())

    course_to_gp, course_to_area, areacode_to_area, area_to_dean, course_to_dean, dean_code_to_name = campus_dept_hierarchy()

    records = {}
    sheets = ['GOTT1', 'GOTT2', 'GOTT4', 'GOTT5', 'GOTT6', 'HUM.STEM', 'POCR Reviewed', 'SU21 Workshop', 'BOOT CAMP', 'GOTT ABC', 'TITLE V GOTT ACADEMY', 'Other Certifications']
    for sname in sheets:
        print(f"{sname}:")
        sheet = wb[sname]
        records[sname] = {}
        for row in sheet.iter_rows():
            if row[1].value == 'ID': continue
            goo = row[1].value
            rowvals = [str(v.value) for v in row]
            records[sname][goo] = rowvals
            print("  " + " ".join(rowvals))
        print()
    #print(json.dumps(records,indent=2))
    teachers = defaultdict(list)

    teachers_bydept = defaultdict(set)
    alldepts = set()

    # reconfigure to key on goo
    by_goo = defaultdict(dict)

    for course,coursedict in records.items():
        print(course)
        for goo,record in coursedict.items():
            if goo=='ID': continue
            if record[0]=='Name': continue
            print(f"    {goo}")
            try:
                if record[4]=="None":
                    by_goo[goo][course] = "ok"
                else:
                    by_goo[goo][course] = record[4]   # record the end date
            except:
                print(f"  -- problem with this record {json.dumps(record)}")
                by_goo[goo][course] = 'ok'

    bg_file = codecs.open('cache/gott_by_goo.json','w','utf-8')
    bg_file.write(json.dumps(by_goo,indent=2))


    ##
    ## Start looking at the courses to cross reference
    ##
    for c in courses:
        print(c)
        try:
            goo = c[8]
            crn = c[4]
            name = c[1]        # full course name
            teacher = c[6]     # last, first
            ctype = c[3]
            dept1 = re.search(r'([A-Z]+)(\d+)',c[2].split(' ')[0]).group(1)
            alldepts.add(dept1)
            d = list(c)
            #d.append(ctype)
            if not ctype:
                print(f"not finding mode for {name}")
                continue
            if ctype=='in-person': continue
            teachers[teacher].append(d)
            teachers_bydept[dept1].add(teacher)
        except: 
            print(f"Problem with {c}, skipping")

    alldepts = list(alldepts)
    alldepts.sort()

    sheet = wb.create_sheet("New Summary")
    r = 1

    deptfont = Font(bold=True)
    flagfont = PatternFill("solid", fgColor="00FFFFCC")

    for thedean in ['et','nl','ss','jn', 'de']:
        sheet.cell(row=r, column=1).value = dean_code_to_name[thedean]
        sheet.cell(row=r, column=1).font = deptfont
        r += 2

        report.write(f"Dean: {dean_code_to_name[thedean]}\n")

        for D in alldepts:
            if not D in course_to_dean:
                print(f"MISSING DEAN for dept: {D}")
            if course_to_dean[D] == thedean:
                if len(teachers_bydept[D]) == 0: continue
                print(f"\n------------\n{D}")
                sheet.cell(row=r, column=1).value = D
                sheet.cell(row=r, column=1).font = deptfont
                r += 1

                for t in teachers_bydept[D]:
                    if t == 'STAFF, STAFF': continue
                    completed = 0
                    waived = 0
                    sects = teachers[t]
                    print(f"Sections for {t}: {sects}")
                    goo = sects[0][8]
                    course_mode = sects[0][3]
                    print(t)
                    sheet.cell(row=r, column=1).value = f"{t}"
                    sheet.cell(row=r, column=2).value = f"{goo}"
                    r += 1
                    if goo in records['GOTT1']:
                        sheet.cell(row=r, column=2).value = f"✓ GOTT 1 Trained"
                        print(f"   GOTT 1")
                        completed =1
                        r += 1
                    if goo in records['Other Certifications']:
                        sheet.cell(row=r, column=2).value = f"✓ GOTT Waived - Outside Training"
                        print(f"   GOTT outside training")
                        completed = 1
                        waived = 1
                        r += 1
                    if goo in records['GOTT2']:
                        sheet.cell(row=r, column=2).value = f"✓ GOTT 2 Trained"
                        print(f"   GOTT 2")
                        completed = 1
                        waived = 1
                        r += 1
                    if goo in records['POCR Reviewed']:
                        sheet.cell(row=r, column=2).value = f"✓ POCR Reviewed"
                        print(f"   POCR")
                        completed = 1
                        waived = 1
                        r += 1
                    if goo in records['TITLE V GOTT ACADEMY']:
                        sheet.cell(row=r, column=2).value = f"✓ TITLE V GOTT ACADEMY 2014"
                        print(f"   GOTT Academy")
                        completed = 1
                        waived = 1
                        r += 1
                    if not completed:
                        if waived:
                            sheet.cell(row=r, column=2).value = f"✓ GOTT 1 Waived"
                        else:
                            sheet.cell(row=r, column=2).value = f"- MISSING GOTT 1"
                            sheet.cell(row=r-1,column=1).fill = flagfont
                            report.write(f"\tMISSING GOTT 1: {t} {goo}\n")
                        r += 1
                    if goo in records['GOTT4']:
                        sheet.cell(row=r, column=2).value = f"✓ GOTT 4 Trained"
                        print(f"   GOTT 4")
                        r += 1
                    if goo in records['GOTT5']:
                        sheet.cell(row=r, column=2).value = f"✓ GOTT 5 Trained"
                        print(f"   GOTT 5")
                        r += 1
                    if goo in records['GOTT6']:
                        sheet.cell(row=r, column=2).value = f"✓ GOTT 6 Trained"
                        print(f"   GOTT ")
                        r += 1
                    if goo in records['SU21 Workshop']:
                        sheet.cell(row=r, column=2).value = f"✓ SU21 Workshop"
                        print(f"   summer 21 workshop")
                        r += 1
                    if goo in records['HUM.STEM']:
                        sheet.cell(row=r, column=2).value = f"✓ Humanizing Stem"
                        print(f"   humanizing stem")
                        r += 1
                    if goo in records['BOOT CAMP']:
                        sheet.cell(row=r, column=2).value = f"✓ Boot Camp Self Paced"
                        print(f"   bc self paced")
                        r += 1
                    if goo in records['GOTT ABC']:
                        sheet.cell(row=r, column=2).value = f"✓ {records['GOTT ABC'][goo][2]} Self Paced"
                        print(f"   GOTT abc self paced")
                        r += 1
                    for s in sects:
                        sheet.cell(row=r, column=2).value = f"{s[3]}"
                        sheet.cell(row=r, column=3).value = f"{s[1]}"
                        r += 1

                        if (not completed) and (not waived):
                            report.write(f"\t\t{s[3]}\t{s[1]}\t{s[7]}\n")
                    if (not completed) and (not waived):
                        report.write(f"\n")
    
    #for c in sheet.columns:
    #    print(c)
    #    print(f"{c} width: {sheet.column_dimensions[c].value}")
    sheet.column_dimensions['A'].width = 20
    sheet.column_dimensions['B'].width = 30
    sheet.column_dimensions['C'].width = 75
    formatted_date = datetime.datetime.now().strftime('%Y%m%d')
    wb.save(f"C:/Users/phowell/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx")

def cross_ref_training_withcsv():
    from localcache2 import all_sem_courses_teachers
    from localcache import course_mode
    from localcache import sem_schedule

    gott1 = {}
    g1read = csv.reader(codecs.open('cache/GOTT_1.csv','r','utf-8'))
    i = 0
    for row in g1read:
        if i == 0:
            headers = row
            #print(f"Headers: {headers}")
            i += 1
        else:
            goo = row[1]
            gott1[goo] = row

    gott2 = {}
    g2read = csv.reader(codecs.open('cache/GOTT_2.csv','r','utf-8'))
    i = 0
    for row in g2read:
        if i == 0:
            headers = row
            #print(f"Headers: {headers}")
            i += 1
        else:
            goo = row[1]
            gott2[goo] = row

    gott4 = {}
    g4read = csv.reader(codecs.open('cache/GOTT_4.csv','r','utf-8'))
    i = 0
    for row in g4read:
        if i == 0:
            headers = row
            #print(f"Headers: {headers}")
            i += 1
        else:
            goo = row[1]
            gott4[goo] = row

    gott5 = {}
    g5read = csv.reader(codecs.open('cache/GOTT_5.csv','r','utf-8'))
    i = 0
    for row in g5read:
        if i == 0:
            headers = row
            #print(f"Headers: {headers}")
            i += 1
        else:
            goo = row[1]
            gott5[goo] = row

    gott6 = {}
    g6read = csv.reader(codecs.open('cache/GOTT_6.csv','r','utf-8'))
    i = 0
    for row in g6read:
        if i == 0:
            headers = row
            #print(f"Headers: {headers}")
            i += 1
        else:
            goo = row[1]
            gott6[goo] = row

    gott9 = {}
    g9read = csv.reader(codecs.open('cache/GOTT_others.csv','r','utf-8'))
    i = 0
    for row in g9read:
        if i == 0:
            headers = row
            #print(f"Headers: {headers}")
            i += 1
        else:
            goo = row[1]
            gott9[goo] = row


    #report = codecs.open('cache/training_report.csv','w','utf-8')
    #out = csv.writer(report)
    #out.writerow(['crn','course','mode','teacher','goo','training','training_date'])

    #sections = sem_schedule('sp24')
    teachers = defaultdict(list)

    teachers_bydept = defaultdict(set)
    alldepts = set()


    courses = all_sem_courses_teachers()
    for c in courses:
        goo = c[7]
        crn = c[2].split(' ')[-1].split('/')[0]
        name = c[2]
        teacher = c[4]
        ctype = course_mode(crn,'sp24')

        dept1 = re.search(r'([A-Z]+)(\d+)',c[2].split(' ')[0]).group(1)
        alldepts.add(dept1)
        d = list(c)
        d.append(ctype)
        if ctype=='in-person': continue
        teachers[teacher].append(d)

        teachers_bydept[dept1].add(teacher)


        #print(f"{crn} {ctype} {name}  ")
        #if goo in gott1:
        #    out.writerow([c[1], c[2], ctype, c[4], goo, "GOTT 1", "/".join(gott1[goo][2:])])
        #
        #else:
        #    out.writerow([c[1], c[2], ctype, c[4], goo, "GOTT 1 MISSING", ''])
    
    alldepts = list(alldepts)
    alldepts.sort()

    for D in alldepts:
        print(f"\n------------\n{D}")
        for t in teachers_bydept[D]:
            if t == 'STAFF STAFF': continue
            sects = teachers[t]
            print(t)
            goo = sects[0][7]
            if goo in gott1:
                print("   + GOTT 1 Trained")
            else:
                print("   - MISSING GOTT 1")
            if goo in gott2:
                print("   + GOTT 2 Trained")
            if goo in gott4:
                print("   + GOTT 4 Trained")
            if goo in gott5:
                print("   + GOTT 5 Trained")
            if goo in gott6:
                print("   + GOTT 6 Trained")
            if goo in gott9:
                print("   + GOTT Waived - Outside Training")
            for s in sects:
                print(f"   {s[8]}   {s[2]}")
            print()

def get_portfolios(id=0):
    if not id:
        id = int( input( "what user id? "))

    p = fetch( f"{url}/api/v1/users/{id}/eportfolios" )

    print(json.dumps(p, indent=2))


def get_port_pages(id=0):
    if not id:
        id = int( input("what portfolio id? "))

    p = fetch(f"{url}/api/v1/eportfolios/{id}/pages")
    print(json.dumps(p, indent=2))



if __name__ == "__main__":
    print ("")
    options = { 1: ['Fetch iLearn users with @gavilan.edu email address', teacherRolesUpdateCache],
                2: ['Fetch all users',fetchAllUsers],
                5: ['Download user avatars', downloadPhoto],
                6: ['Merge photo folders', mergePhotoFolders],
                7: ['Get all teachers logs 1 month', get_recent_views],
                8: ['Gather teacher history, a variety of stats.', getTeachersInfoMain],
                9: ['test rtr.', read_training_records],
                10: ['Get a users logs', track_user],
                11: ['test: oneYearSchedule', oneYearSchedule],
                12: ['summarize hit activity', activity_summary],
                13: ['Get all users logs in a class', track_users_in_class],
                14: ['Get logs for 1 user', track_a_user ],
                15: ['Get all users logs in a semester', track_users_in_sem],
                16: ['Report on attendance for all classes', track_users_by_teacherclass],
                17: ['Show all convos for a user', convos_for_user],
                21: ['Show all pushed notifications for a user', comm_mssgs_for_user],
                18: ['Quiz submissions', quiz_submissions],
                19: ['NLP Sample', nlp_sample],
                20: ['Enroll a single user into a class', one_course_enrol],
                21: ['Teachers new this semester', find_new_teachers],
                #22: ['Sync personnel and conference user databases', user_db_sync], 
                23: ['Find non-gnumbers', find_no_goo ],
                24: ['compare user tables', compare_db_tables],
                
                25: ['cross ref training', cross_ref_training],
                26: ['find goo numbers in training spreadsheet', training_find_goos],

                30: ['get portfolios for user id', get_portfolios],
                31: ['get portfolio pages for portfolio id', get_port_pages],

                #3: ['Main index, 1 year, teachers and their classes', getAllTeachersInTerm],
                #5: ['Match names in schedule & ilearn', match_usernames],
                #6: ['Create Dept\'s ZTC list', create_ztc_list],
                ##7: ['Build and send ZTC emails', send_ztc_mails],
                #14: ['investigate the logs', investigate_logs],
                #12: ['test: match_usernames', match_usernames],
                #13: ['test: get all names', getAllNames],
                #13: ['x', users_with_gavilan_email], 
    }
    if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
        resp = int(sys.argv[1])
        print("\n\nPerforming: %s\n\n" % options[resp][0])
    
    else:
        print ('')
        for key in options:
            print(str(key) + '.\t' + options[key][0])
        
        print('')
        resp = input('Choose: ')
    
    # Call the function in the options dict
    options[ int(resp)][1]()