diff --git a/apphelp.py b/apphelp.py
index 0d117af..ac61694 100644
--- a/apphelp.py
+++ b/apphelp.py
@@ -1,1619 +1,1977 @@
-
-
-import os,re
-
-output = ''
-todos = 0
-
-todos_d = {}
-
-# TODO: Make a second pass, and look for fxn calls inside of
-# each function. Draw a graphviz.
-#
-
-def fxns(fname):
- global output, todos
- lines = open(fname).readlines()
- prev_L = ""
- for L in lines:
-
- # is it a todo
- a = re.search(r'(TODO|todo)\s*:?\s*(.*)$',L)
- if a:
- output += "\t\ttodo: " + a.group(2) + "\n"
- todos += 1
- if fname in todos_d:
- todos_d[fname] += 1
- else:
- todos_d[fname] = 1
-
- # is it a function def?
- if re.search('^\s*def\s',L):
- output += "\n"
- if re.search('^#',prev_L): output += "\t"+prev_L
- output += "\t"+L+""
- prev_L = L
-
-files = os.listdir('.')
-files.sort()
-
-
-for F in files:
- if F=='apphelp.py': continue
- if re.search('\.py$',F):
- output += "\n" + F + "\n"
- fxns(F)
-
-prog_in = open('apphelp.py','r')
-prog_out = ''
-td = '# Total TODOs remaining: %i' % todos
-
-td2 = '# TODOs per file: \n#\n'
-for k in sorted(todos_d.keys()):
- td2 += "#\t%i - %s \n" % (todos_d[k],k)
-
-
-for R in prog_in.readlines():
- if re.search('^##\sF',R):
- prog_out += "## Functions\n#\n%s\n#\n%s\n#\n" % (td,td2)
- break
- prog_out += R
-
-prog_out += '\n"""\n\n' + output + '\n\n"""\n\n'
-prog_in.close()
-prog_write = open('apphelp.py','w')
-prog_write.write(prog_out)
-prog_write.close()
-
-
-
-
-
-## Functions
-#
-# Total TODOs remaining: 57
-#
-# TODOs per file:
-#
-# 1 - checker.py
-# 1 - content.py
-# 6 - courses.py
-# 3 - curriculum.py
-# 5 - depricated.py
-# 6 - localcache.py
-# 2 - outcomes.py
-# 20 - pipelines.py
-# 2 - server.py
-# 2 - tasks.py
-# 1 - tempget.py
-# 8 - users.py
-
-#
-
-"""
-
-
-__init__.py
-
-checker.py
- todo: make this sweet
-
- def safe_html(html):
-
- def _attr_name_whitelisted(attr_name):
-
- def safe_css(attr, css):
-
- def plaintext(input):
-
- def _unescape(text):
-
- def fixup(m):
-
- def check_folder(fname,path):
-
- def check_class(folder):
-
- def check_all():
-
-content.py
-
- def d(s):
-
- def stripper(s):
-
- def mycleaner(s):
-
- def freshdesk():
-
- # Build a master file with the entire class content
- def accessible_check(id=""):
- todo: include linked pages even if they aren't in module
-
- def pan_testing():
-
- # Given course, page url, and new content, upload the new revision of a page
- def create_page(course_num,new_title,new_content):
-
- def md_to_course():
-
- # DL pages only
- def grab_course_pages(course_num=-1):
-
- # Appears to not be used
- def put_course_pages():
-
- # Also not used
- def put_revised_pages():
-
- # Download, clean html, and reupload page
- def update_page():
-
- # Given course, page url, and new content, upload the new revision of a page
- def upload_page(course_num,pageurl,new_content):
-
- # Use template to build html page with homegrown subtitles
- def build_srt_embed_php(data):
-
- def yt_title(code):
-
- def swap_youtube_subtitles():
-
- def test_swap():
-
- def multiple_downloads():
-
-courses.py
- todo:
-
- def int_or_zero(x):
-
- def float_or_zero(x):
-
- # Gott 1 Bootcamp - report on who completed it.
- def get_gott1_passers():
-
- # Plagiarism Module - report on who completed it.
- def get_plague_passers():
-
- # Who, in a class, passed?
- def get_course_passers(course, min_passing, passers_filename, still_active_filename):
-
- # Change courses to show 2 announcements
- def change_course_ann_homepage(id="10458"):
-
- def scrape_bookstore():
- todo: where does the most recent schedule come from?
-
- # Input: xxxx_sched.json. Output: xxxx_latestarts.txt
- def list_latestarts():
-
- # All students enrolled in a class in the given semester. Simpler verson of below. Return SET of course_ids.
- def users_in_semester():
- todo:
-
- # All students in STEM (or any list of depts.. match the course_code). Return SET of canvas ids.
- def users_in_depts_live(depts=[], termid='171'):
-
- def course_enrollment(id=''):
-
- def askForTerms():
-
- # Return a list of term names and IDs. Also store in cache/courses/terms.txt
- def getTerms(printme=1, ask=1):
- todo: unsafe overwrite
-
- def getCourses(): # a dict
-
- # Relevant stuff trying to see if its even being used or not
- def course_term_summary():
-
- # Fetch all courses in a given term
- def getCoursesInTerm(term=0,show=1,active=0): # a list
-
- def getCoursesTermSearch(term=0,search='',v=0):
-
- def courseLineSummary(c,sections={}):
-
- def xlistLineSummary(c,sections={}):
-
- def eslCrosslister():
-
- def xlist(parasite='', host=''): # section id , new course id
- todo: need to get the section id from each course:
-
- def unenroll_student(courseid,enrolid):
-
- def enroll_stem_students_live():
-
- def enroll_orientation_students():
-
- def summarize_proportion_online_classes(u):
-
- def summarize_num_term_classes(u):
-
- def make_ztc_list(sem='sp20'):
-
- def course_search_by_sis():
-
- def add_evals(section=0):
- todo: wanted: group shell for each GP (guided pathway) as a basic student services gateway....
-
-curriculum.py
- todo: These secrets
-
- def another_request(url,startat):
-
- def fetch_all_classes():
-
- def fetch_all_programs():
-
- def sortable_class(li):
-
- def c_name(c):
-
- def show_classes(createoutput=1):
-
- def clean_d_name(d):
-
- def show_programs():
-
- def dd(): return defaultdict(dd)
-
- def organize_courses():
-
- def check_de():
-
- def clean_programs():
-
- def course_lil_format(s):
-
- def header_lil_format(s):
-
- def organize_programs():
-
- def divide_courses_list(li,rwd,online):
-
- def organize_programs2():
-
- # sorting by order key of dict
- def cmp_2(a):
-
- def cmp_order(a,b):
-
- # decipher the grouped up courses line
- def split_course(st):
-
- # Any number gets an X (checked). Blank or zero gets no check.
- def units_to_x(u):
-
- def p_block_rule(r,printme,doc,out=0):
-
- def p_cert_header(type,doc,r='',out=0):
-
- def p_block_header(r,doc,out=0):
-
- def p_cert_course_missing(cd,doc,out=0):
-
- def p_cert_course(cd,history,doc,out=0):
-
- def p_end_block(out=0):
-
- def p_end_cert(bigdoc, out=0):
-
- def ask_for_rule(r):
-
- def action_to_english(a):
-
- # Return True if the courses satisfy the rule
- def check_a_block(b, courses, verbose=False):
-
- def read_block_english_to_code():
-
- def read_section_online_history():
- todo: this file depends on other fxns. which?
-
- # This is the 3rd attempt.
- def simple_find_online_programs():
- todo: courses with a HYPHEN in NAME get parsed wrong.
-
- def check_a_block_a(b,verbose=False):
-
- def smart_find_online_programs():
-
- def show_contained_class(c):
-
- def show_block(c):
-
- def show_block(c):
-
- def is_online(c):
-
- def is_online(c):
-
- def is_online_inblock(c):
-
- def is_online_inblock(c):
-
- # of all the programs, what can be accomplished online?
- def find_online_programs():
-
- # take a string of all the types of classes offered, return a vector of [tot,lec,hyb,onl]
- def string_to_types(st):
-
- def my_default_counter():
-
- # Of the recent schedules, what was actually offered online?
- def summarize_online_sections():
-
- def fibonacci(n):
-
- def test_pampy():
-
- def cq_parse_experiment(root=0, indent=''):
-
- def cq_start():
-
- def cq_pattern_backup1(root=0, indent=''):
-
- def found(*x):
-
- def lookForMatch(rules,item):
-
- def cq_pattern(root=0, indent=''):
-
- def myprinter(item, indent=''):
-
- def cq_pattern_start():
-
- def baby_int(j):
-
- def find_deg_in_cluster( clusters, deg ):
-
- def try_match_deg_programs():
-
- def dict_generator(indict, pre=None):
-
- def print_dict(v, prefix='',indent=''):
-
- def walk_file():
-
- def tag(x,y): return "<%s>%s" % (x,y,x)
-
- def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
-
- def a(t,h): return '%s' % (h,t)
-
- def server_save(key,value):
-
- def flask_thread(q):
-
- def home():
-
- def s(key,val):
-
- def hello():
-
- def sd():
-
- def serve():
-
- def attempt_match8020(rules,item):
-
- def clever_printer(item, indent=''):
-
- def print_return(x):
-
- def cq_8020(root=0, indent=''):
-
- def cq_8020_start():
-
-curriculum2020.py
-
- def to_md(s):
-
- def print_return(x):
-
- def cq_8020(root,indent=0):
-
- def cq_8021(root,indent=0):
-
- def cq_8021_start():
-
- def cq_8022(root,indent=0):
-
- def cq_8022_start():
-
- def sortable_class(li):
-
- def c_name(c):
-
- def show_classes2020():
-
- def show_classes2020_start():
-
-curriculum_patterns.py
-
- def div1(a,b):
-
- def d2(a,b):
-
- def d3(a,b):
-
- def pp1(a,b):
-
- def pp2(a,b):
-
- def pp3(a,b,c,d,e,f,g):
-
- def pp4(a,b):
-
- def pp5(a,b,c):
-
- def pp6(a,b):
-
- def pp7(a,b):
-
- def pp8(a,b):
-
- def jj1(a,b,c,d,e):
-
- def jj2(a,b,c,d,e,f):
-
- def jj3(a,b,c,d,e):
-
- def jj4(a,b,c,d):
-
- def jj5(a,b,c,d,e,f):
-
- def jj6(a,b,c,d):
-
- def jj2(a,b,c,d):
-
- def jj2(a,b,c,d):
-
-depricated.py
-
- # Don't know
- def demo():
-
- def stats():
-
- def dict_generator(indict, pre=None):
-
- def print_dict(v, prefix='',indent=''):
-
- def walk_file():
-
- def tag(x,y): return "<%s>%s" % (x,y,x)
-
- def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
-
- def a(t,h): return '%s' % (h,t)
-
- def server_save(key,value):
-
- def flask_thread(q):
-
- def home():
-
- def s(key,val):
-
- def hello():
-
- def sd():
-
- def serve():
- todo: this duplicates courses.py ??
-
- # Prompt for course id, return list of user dicts. TODO this duplicates courses.py ??
- def getUsersInCourse(id=0): # returns list
- todo:
-
- # NO LONGER USED - SEE COURSES
- def enroll_stem_students():
-
- # unused?
- def getAllTeachersInTerm(): # a list
- todo: hits in courses by teachers https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
-
- def teacherActivityLog(uid=1): ### Next: save results in a hash and return that....
-
- def summarize_student_teacher_role(u):
-
- def user_roles2():
-
- def req_to_db(fname_list):
-
- def has_online(series):
-
- def has_lecture(series):
-
- def has_hybrid(series):
-
- # Wrapper to get 2 schedules at once
- def dl_sched():
- todo: these semesters
-
- # Send a personalized email regarding ZTC
- def send_z_email(fullname, firstname, addr, courses_list):
-
- def getInactiveTeachersInTerm(t=23): # a list
-
- def course_location(course):
-
- def course_time(course):
-
- def course_teacher(course):
-
- def reg_nums():
-
- # In the schedule, is this a class or a continuation of the class above?
- def categorize():
- todo: must we open all these files?
-
- # Deprecated. call perl.
- def constructSchedule():
-
- def fetch_dict(target,params={}):
-
- def get_schedule(term='201870', sem='fall'):
-
-interactive.py
-
- def dict_generator(indict, pre=None):
-
- def print_dict(v, prefix='',indent=''):
-
- def walk_file():
-
- def tag(x,y): return "<%s>%s" % (x,y,x)
-
- def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
-
- def a(t,h): return '%s' % (h,t)
-
- def server_save(key,value):
-
- def flask_thread(q):
-
- def before_request():
-
- def save_post():
-
- def restart():
-
- def dispatch3(func,arg,arrg):
-
- def dispatch2(func,arg):
-
- def dispatch(func):
-
- def dispatch3j(func,arg,arrg):
-
- def dispatch2j(func,arg):
-
- def dispatch1j(func):
-
- def home():
-
- def send_jslib(path):
-
- def send_cachedata(path):
-
- def send_js(path):
-
- def s(key,val):
-
- def do_sample():
-
- def media(file_id):
-
- def podcast():
-
- def weblec():
-
- def hello():
-
- def sd():
-
- def test_message(message):
-
- def serve():
-
- def make_teacher_rel(self, tchr, clss):
-
- def __init__(self, uri, user, password):
-
- def close(self):
-
- def print_greeting(self, message):
-
- def _create_and_return_greeting(tx, message):
-
- def make_teacher_rel(g, tchr, clss):
-
- def testgraph():
-
- def Memoize( func):
-
- def wrapper(*args):
-
- def startup(self, outfile):
-
- def set_my_dict(self,d):
-
- def cycle_color(self, s):
-
- def ascii_art(self, text):
-
- def close_window(self, ):
-
- def suggest(self, word):
-
- def curses_print_word(self, word,color_pair_code):
-
- def curses_print_line(self, line,color_pair_code):
-
- def redraw(self, start_y,end_y,fallback_y,fallback_x):
-
- def scroll_down(self, noredraw,fallback_y,fallback_x):
-
- def clear_upside(self, n,y,x):
-
- def display_suggest(self, y,x,word):
-
- def inputloop(self, ):
-
- def setup_command(self,outfile):
-
- def cleanup_command(self):
-
- def handle_command(self, cmd):
-
- def repl_staff():
-
- def repl_degs():
-
- def repl():
-
- def repl():
-
-interactivex.py
-
- def dict_generator(indict, pre=None):
-
- def print_dict(v, prefix='',indent=''):
-
- def walk_file():
-
- def tag(x,y): return "<%s>%s" % (x,y,x)
-
- def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
-
- def a(t,h): return '%s' % (h,t)
-
- def server_save(key,value):
-
- def flask_thread(q):
-
- def before_request():
-
- def restart():
-
- def dispatch3(func,arg,arrg):
-
- def dispatch2(func,arg):
-
- def dispatch(func):
-
- def dispatch3j(func,arg,arrg):
-
- def dispatch2j(func,arg):
-
- def dispatch1j(func):
-
- def home():
-
- def send_jslib(path):
-
- def send_cachedata(path):
-
- def send_js(path):
-
- def s(key,val):
-
- def do_sample():
-
- def hello():
-
- def sd():
-
- def serve():
-
- def make_teacher_rel(self, tchr, clss):
-
- def __init__(self, uri, user, password):
-
- def close(self):
-
- def print_greeting(self, message):
-
- def _create_and_return_greeting(tx, message):
-
- def make_teacher_rel(g, tchr, clss):
-
- def Memoize( func):
-
- def wrapper(*args):
-
- def startup(self, outfile):
-
- def set_my_dict(self,d):
-
- def cycle_color(self, s):
-
- def ascii_art(self, text):
-
- def close_window(self, ):
-
- def suggest(self, word):
-
- def curses_print_word(self, word,color_pair_code):
-
- def curses_print_line(self, line,color_pair_code):
-
- def redraw(self, start_y,end_y,fallback_y,fallback_x):
-
- def scroll_down(self, noredraw,fallback_y,fallback_x):
-
- def clear_upside(self, n,y,x):
-
- def display_suggest(self, y,x,word):
-
- def inputloop(self, ):
-
- def setup_command(self,outfile):
-
- def cleanup_command(self):
-
- def handle_command(self, cmd):
-
- def repl_staff():
-
- def repl_degs():
-
- def repl():
-
- def repl():
-
-ipython_log.py
-
-localcache.py
-
- def db():
-
- def setup_table(table='requests'):
-
- # Help the next function to upload new users directly to conf database on gavilan.
- def employees_refresh_flex(data):
-
- # Everyone in iLearn DB with an xyz@gavilan.edu email address.
- def all_gav_employees():
-
- #
- def teachers_courses_semester():
-
- #
- def teachers_by_term():
-
- # Report for AEC
- def aec_su20_report():
-
- # Return the most up do date version of the given file. Useful for 'dimensions'.
- def most_recent_file_of( target ):
-
- def finder(st):
-
- # Given a table schema, parse log file, return a list of dicts. Optionally remove some columns.
- def parse_file_with( file, format, with_gid=0 ):
-
- # I return a list of the read lines if the log dates in the file are within dates (top of this file), or FALSE
- def is_requestfile_interesting(fname):
- todo: more robust here
- todo: - investigate pearson, developer key: 170000000000376 and their ridiculous amounts of hits.
-
- # Return a 'timeblock'. An integer number of 15 minute blocks from my epoch. Expects a datetime object in PST timezone.
- def timeblock_from_dt(dt_obj):
-
- # Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
- def dt_from_timeblock(tb):
-
- # Twenty Four hour timeblocks
- def timeblock_24hr_from_dt(dt_obj):
-
- # Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
- def dt_from_24hr_timeblock(tb):
-
- # Four hour timeblocks
- def timeblock_4hr_from_dt(dt_obj):
-
- # Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
- def dt_from_4hr_timeblock(tb):
-
- # I make the line into a dict, erase keys with no data, make a DT field called date, make a time_block (int) field.
- def requests_line(line,i=0):
-
- # Bulk insert of requests logs. Too much data to be useful.
- def requests_file(fname_list):
- todo: select if timeblock exists
-
- # Insert or update a request line.
- def upsert_request(line, vals):
-
- # Generic insert of a dict into a table. Keys of dict must match table columns.
- def dict_to_insert(thisline,table): # a dict
-
- # This now does tallying by timeblock.
- def merge_requests():
-
- def merge_comm_channel():
-
- def merge_pseudonym():
-
- def merge_users():
-
- def merge_courses():
-
- def merge_enrollment():
-
- def merge_term():
-
- def merge_roles():
-
- def merge_convos():
-
- # For returning sqlite results as dicts
- def dict_factory(cursor, row):
- todo: ... approaches to all this data... list requests in order descending time, unique users, and just
-
- # Attempt to do tallying
- def make_views_summarys():
-
- # original without time_blocks info.
- def make_views_summarys_v1():
-
- # Setup my basic db stats base from scratch
- def full_reload():
-
- def guess_dept(t):
-
- # Main view of all class / all user overview...
- def dept_with_studentviews(dept="", sem=''):
-
- def f(x):
-
- # get student count and teacher name from local db
- def course_quick_stats(canvasid):
-
- # What a student has taken / teacher has taught
- def user_enrolled_in(userid):
-
- # All students in this semester ...
- def users_this_semester_db(sem=''):
-
- # Everyone whose first semester is .....
- def users_new_this_semester(sem=''):
-
- # All student users in STEM - from local db
- def user_in_stem():
-
- # Get all the classes in one dept
- def dept_classes(dept,sem=''):
- todo:
-
- def depts_with_classcounts(sem=''):
- todo:
-
- def f(x):
-
- def name_with_count(name,li):
-
- def arrange_data_for_web(dept='', sem=''):
-
- def f(x):
-
- # Get enrollments. (Best to freshly run pipelines/get_rosters) and put them into DB
- def build_tables(headers,name):
-
- def load_tables(table,headers,row,verbose=0):
-
- def semester_enrollments(verbose=0):
-
- def qstrip(txt): return txt.strip('"')
-
- def more_unused_xreferencing():
-
- def user_role_and_online():
-
- def comm_channel_file():
-
- def pseudonym_file():
-
- def users_p_file():
-
- def com_channel_dim():
-
- def abcd():
-
- def crns_to_teachers():
-
-main.py
-
-outcomes.py
-
- def outcome_overview(term=21):
-
- def create_acct_lvl_outcomes(src,dept,makefolder='',folder=0):
-
- def connect_acct_oc_to_course(course_id,oc_group_id):
-
- def outcome_groups():
-
- def outcome_groups_backup():
-
- def x_ref_dept_names():
-
- def create_course_group(short,parent):
-
- def create_dept_group(short):
-
- def outcomes_attached_to_courses(term=65,limitdept=''):
- todo: Handle this: CSIS/DM85 WEB DESIGN 40823/24
-
- def summarize_course_online_slo(outcome_list):
-
- def fetch_outcome_details(id):
-
- # Report on the actual evaluation data?
- def outcome_report1():
- todo:
-
- # For the given course, get all outcome measurements, and display scores and stats.
- def outcome_report2():
-
- def fix_joined_class(str):
-
- def split_slo_name(str):
-
- def outcome_report3():
-
- def read_slo_source():
-
- def slo_source_by_dept():
-
-patterns_8020.py
-
-patterns_topdown.py
-
- def pp0(a,b,c,d,e):
-
- def pp1(a,b,c,d,e,f):
-
- def pp2(a,b,c,d,e):
-
- def pp3(a,b,c,d):
-
- def pp4(a,b,c):
-
- def pp5(a,b,c):
-
- def pp6(a,b,c):
-
- def div1(a,b):
-
- def d2(a,b):
-
- def d3(a,b):
-
- def pp1(a,b):
-
- def pp2(a,b):
-
- def pp3(a,b,c,d,e,f,g):
-
- def pp4(a,b):
-
- def pp5(a,b,c):
-
- def pp6(a,b):
-
- def pp7(a,b):
-
- def pp8(a,b):
-
- def jj3(a,b,c,d,e):
-
- def jj5(a,b,c,d,e,f):
-
- def jj2(a,b,c,d):
-
- def jj2(a,b,c,d):
-
-pipelines.py
- todo: secrets
- todo: all these constants for SSB -- line 1008
- todo: secrets
-
- def d(s):
-
- # Main canvas querying fxn
- def fetch(target,verbose=0):
-
- # Main canvas querying fxn - stream version - don't die on big requests
- def fetch_stream(target,verbose=0):
-
- # paging makes problems... example: enrollment_terms
- def fetch_collapse(target,collapse='',verbose=0):
-
- # Teacher name format changed. Remove commas and switch first to last
- def fix_t_name(str):
-
- # Separate dept and code
- def split_class_dept(c):
-
- def split_class_code(c):
-
- def split_class_code_letter(c):
-
- # go from sp20 to 2020spring
- def shortToLongSem(s):
-
- # Go to the semesters folder and read the schedule. Return dataframe
- def getSemesterSchedule(short='sp21'): # I used to be current_schedule
- todo: Some semesters have a different format.... partofday type site xxx i just dL'd them again
-
- def prep_online_courses_df():
-
- def course_is_online(crn):
-
- def get_crn_from_name(name):
-
- def get_enrlmts_for_user(user,enrollments):
-
- # Get something from Canvas Data
- def do_request(path):
-
- # Canvas data, download all new files
- def sync_non_interactive():
-
- # list files in canvas_data (online) and choose one or some to download.
- def interactive():
-
- def todays_date_filename(): # helper
-
- def nowAsStr(): # possible duplicate
-
- def row_has_data(r): # helper
-
- def row_text(r): # helper
-
- # Take banner's html and make a csv(?) file
- def ssb_to_csv(src):
-
- def clean_funny(str):
-
- def clean_funny2(str):
-
- def clean_funny3(str):
-
- ### course is a list of 1-3 lists, each one being a line in the schedule's output. First one has section
- def course_start(course):
- todo: use this to make a early/late/short field and store semester dates w/ other constants
-
- def time_to_partofday(t):
- todo: account for multiple sites/rows
-
- # Deduce a 'site' field, based on room name and known offsite locations
- def room_to_site(room,verbose=0):
- todo: account for multiple sites/rows
- todo: better way to store these offsite labels
-
- # take text lines and condense them to one dict per section
- def to_section_list(input_text,verbose=0):
- todo: no output files
- todo: if extra line is different type?
-
- # Log the history of enrollments per course during registration
- def log_section_filling(current_sched_list):
-
- # Same as above, but compressed, act only
- def log_section_filling2(current_sched_list):
-
- # Use Firefox and log in to ssb and get full schedule. Only works where selenium is installed
- def scrape_schedule():
- todo: my data here.... secret
- todo:
-
- # recreate schedule json files with most current online schedule format.
- def recent_schedules():
- todo: sems is a global in this file. Is that the right thing to do?
- todo: the pipeline is disorganized. Organize it to have
- todo: where does this belong in the pipeline? compare with recent_schedules()
-
- # Take the generically named rosters uploads files and move them to a semester folder and give them a date.
- def move_to_folder(sem,year,folder):
-
- # This relates to enrollment files, not schedule.
- def convert_roster_files(semester="",year="",folder=""):
-
- # From instructure sftp site
- def fetch_current_rosters():
- todo: secret
-
- def fetch_current_rosters_auto():
-
- # read schedule file with an eye toward watching what's filling up
- def schedule_filling():
- todo: hardcoded
-
- # Upload a json file to www
- def put_file(remotepath,localpath, localfile,prompt=1):
- todo: remove this secret
- todo: these paths
-
- def sec(t): return ""+t+"
\n"
-
- def para(t): return ""+t+"
\n"
-
- def ul(t): return "\n"
-
- def li(t): return ""+t+"\n"
-
- def question(t,bracket=1):
-
- def answer(t):
-
- def read_paragraph_element(element,type="NORMAL_TEXT"):
-
- def get_doc(docid, bracket=1, verbose=0):
- todo: x link, x bold, list, image.
-
- def read_paragraph_element_2(element,type="NORMAL_TEXT"):
-
- # t is a string that begins with "Icons: " ... and contains comma(space) separated list
- def handle_icons(t):
-
- # t is a string that begins with "Tags: " ... and contains comma(space) separated list
- def handle_tags(t):
-
- def handle_question(t,bracket=1):
-
- def handle_answer(t):
-
- def handle_sec(t): return ('section',t)
-
- def handle_para(t): return ('paragraph',t)
-
- def handle_ul(t): return ('unorderdedlist',t)
-
- def handle_li(t): return ('listitem',t)
-
- def fetch_doc_image(k,value):
-
- def get_doc_generic(docid, bracket=1, verbose=0):
-
- def scrape_schedule_py():
-
-server.py
-
- def tag(x,y): return "<%s>%s" % (x,y,x)
-
- def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
-
- def a(t,h): return '%s' % (h,t)
-
- def homepage():
-
- def orgline(L):
- todo: \s\[\#A\](.*)$', L)
-
- def editor(src):
-
- def in_form(txt,path):
-
- def mytime(fname):
-
- def index():
-
- def writing(fname):
-
- def dashboard():
-
- def dash():
-
- def mycalendar():
-
- def most_recent_file_of( target, folder ):
-
- def finder(st):
-
- def news():
-
- def randPic():
-
- def sample():
-
- def sample2(a=""):
-
- # Filter a stream of loglines for those that match a course's url / id
- def has_course(stream,courseid):
-
- def js(s):
-
- def sem_from_array_crn(crn):
-
- def user_courses(uid):
-
- def user_course_history_summary(usr_id):
-
- def roster(crn):
-
- def user_course_hits(usr,courseid):
-
- def profiles(id=1,b=2,c=3):
-
- # Departments, classes in each, and students (with hits) in each of those.
- def enrollment(a):
-
- # All the classes in this dept, w/ all the students in each, with count of their views.
- def dept(d=''):
-
- def user(canvas_id=None):
-
- def lectures():
-
- def web_lectures():
- todo: update: dept, title, any of the other fields.
-
- # update a value: dept id of a personnel id
- def update_pers_title(pid, tid):
-
- # update a value: dept id of a personnel id
- def update_pers_dept(pid, did):
-
- def user_edit(canvas_id='2'):
-
- def staff_dir(search=''):
-
- def server_save(key,value):
-
- def server_dispatch_json(function_name,arg='', arg2=''):
-
- def server_dispatch(function_name,arg='', arg2=''):
-
-stats.py
-
- def grades_rundown():
-
- def class_logs():
-
- def user_logs():
-
- def recent_logins():
-
- def userHitsThisSemester(uid=2):
-
- def getCurrentActivity(): # a dict
-
- def externaltool(): # a list
-
-tasks.py
-
- def survey_answer(q=0):
-
- def survey_organize():
-
- def build_quiz(filename=""):
-
- # Send an email
- def send_email(fullname, firstname, addr, subj, content):
-
- def convert_to_pdf(name1, name2):
-
- # Build (docx/pdf) certificates for gott graduates
- def certificates_gott_build():
-
- # Email experiment
- def mail_test():
-
- # Change LTI Settings. Experimental
- def modify_x_tool():
-
- # Upload with sftp to www website folder: student/online/srt/classfoldername
- def put_file(classfoldername):
- todo: ',cnopts=cnopts) as sftp:
-
- # Switch everyone in a class to a teacher
- def switch_enrol():
-
- # Change dates & term of a class to unrestrict enrollment
- def unrestrict_course():
-
- # Bulk enroll users into a course
- def enroll_accred():
-
- # Calculate attendance stats based on enrollment/participation at 20% of term progressed, then 60% of term progressed.
- def twenty_sixty_stats(li):
-
- # Older positive attendance hours calculation.
- def hours_calc():
-
- def course_2060_dates(crn=""):
- todo:
-
- def course_update_all_users_locallogs(course_id=''):
-
- def hours_calc_pulldata(course_id=''):
-
- def xlist_cwe():
-
- def pos_atten():
-
-temp.py
-
-tempget.py
-
- # Use Firefox and log in to ssb and get full schedule
- def login():
- todo: my data here.... secret
-
- def filename_friendly(str):
-
- def otter():
-
-templates.py
-
- def item_to_masonry(item):
-
- def try_untemplate():
-
- def php_remove(m):
-
- def php_add(m):
-
- def do_template(temp,source,side):
-
- def remove_filetype(f):
-
- def make():
-
- def txt_2_table():
-
- def studenttech_faq():
-
- # https://docs.google.com/document/d/1tI_b-q75Lzu25HcA0GCx9bGfUt9ccM8m2YrrioDFZcA/edit?usp=sharing
- def de_faq():
-
- def degwork_faq():
-
- def vrc_faq():
-
- def counseling_faq():
-
- def finaid_faq():
-
- def coun_loc():
-
- def tutor_faq():
-
- def test_repl():
-
-timer.py
-
- def func(a, b):
-
-users.py
- todo: these constants
-
- # All users to a cache file cache/allusers.json
- def fetchAllUsers():
-
- # Fetch teacher users objects from local cache
- def teacherRolesCache(): # I used to be load_users
-
- # Canvas: Fetch all people with gavilan.edu email address
- def teacherRolesUpdateCache(): # I used to be get_users
-
- # Fetch preferred email address for a given user id. ( Canvas )
- def getEmail(user_id):
-
- # All teachers in a particular course
- def getAllTeachers(course_id=59): # a list
-
- #
- def classType(t):
- todo: fix bug in schedule parser so non-online classes have a type field
-
- def my_blank_string(): return "no data"
-
- def my_blank_dict(): return {'name':'NoName','email':'noemail@gavilan.edu'}
-
- def my_empty_dict(): return defaultdict(my_blank_string)
-
- def get_email_from_rec(name,name_to_record):
-
- # Pull the staff directory on the webpage. Convert to pandas dataframe
- def staff_dir(get_fresh=False):
- todo: lol get fresh again...
-
- def schedForTeacherOverview(long,short):
-
- # Return a dataframe of the last 4 semester schedules put together
- def oneYearSchedule():
-
- def num_sections_last_year(line):
-
- def sec_type_stats(line):
-
- def prct_online(line):
-
- def prct_lecture(line):
-
- def prct_hybrid(line):
-
- # Given the names of teachers in last year's schedules, fill in email, etc. from ilearn files
- def teacher_basic_info(sched, from_ilearn, names):
-
- def find_that_name(x):
-
- # Outputs: cache/teacher_by_semester.csv,
- def teacherModalityHistory(sched=[],names=[]):
-
- def teacherCourseHistory(a,names):
- todo: sort by dept also
-
- # Outputs: cache/course_teacher_combos.csv,
- def teacherSharedCourses(a=[]):
-
- # How many courses in each department were taught in the last year?
- def departmentCountCourses(a=[]):
-
- def clean_nonprint(s):
-
- def read_cmte(names):
-
- def read_training_records():
-
- # open a file and mark the people with their ids given. Return a dataframe
- def read_bootcamp1(filename):
-
- # open a file and mark the people with their ids given. Return a dataframe
- def read_bootcamp2(filename):
-
- def not_blank_or_pound(L):
-
- def temp1(x):
-
- def add_realnames(df,names): # the surveys. raw name is in 2nd column
-
- def compareToughNames(a,b):
-
- def compareNames(a,b,verbose=0):
-
- def find_ilearn_record(ilearn_records,manual_records, othername,verbose=0):
-
- def manualNamesAndDept():
-
- def manualNames():
-
- # given a list of class codes, return the most common (academic) department
- def guessDept(d_list):
-
- # Make one big csv file of everything I know about a teacher
- def getTeachersInfoMain():
-
- def enroll_staff_shell():
-
- # take a list of raw hits.
- def activity_summary(hits):
- todo: month is hardcoded here
-
- # Get views counts on current teachers. todo: month is hardcoded here
- def get_recent_views(id=1):
-
- # Have they taught online or hybrid classes?
- def categorize_user(u):
- todo: threaded
-
- # Doest the account have a photo loaded?
- def checkForAvatar(id=2):
-
- # Grab em. Change the first if when continuing after problems....
- def downloadPhoto():
-
- def mergePhotoFolders():
-
- def mergePhotoFolders2():
-
- # Go through my local profile pics, upload any that are missing.
- def uploadPhoto():
-
- def test_email():
-
- def create_ztc_list():
-
- def get_user_info(id):
-
- # these are any messages that get pushed out to their email
- def comm_mssgs_for_user(uid=0):
-
- #
- def convos_for_user(uid=0):
-
- # single q sub
- def quiz_get_sub(courseid, quizid, subid=0):
-
- # quiz submissions for quiz id x, in course id y
- def quiz_submissions(courseid=9768, quizid=32580):
-
- # return (timeblock, course, read=0,write=1)
- def requests_line(line,i=0):
-
- #
- def report_logs(id=0):
-
- def track_users_in_sem():
-
- def track_users_in_class(L=[]):
-
- def track_user_q(id, q):
-
- # Maintain local logs. Look to see if we have some, download logs since then for a user.
- def track_user(id=0,qid=0):
- todo: set up this info file if it isn't there. check any changes too. it
- todo:
-
- #
- def track_users_by_teacherclass():
-
- def nlp_sample():
-
- def nlp_sample2():
-
- def one_course_enrol():
-
-util.py
-
- def print_table(table):
-
- def remove_nl(str):
-
- def UnicodeDictReader(utf8_data, **kwargs):
-
- def minimal_string(s):
-
- def to_file_friendly(st):
-
- def clean_title(st):
-
- def match59(x):
-
- def item_2(x): return x[2]
-
- def unix_time_millis(dt):
-
- # ENGL250 returns ENGL
- def dept_from_name(n):
-
- def most_common_item(li):
-
- def srt_times(a,b):
-
- def how_long_ago(a): # number of hours ago 'a' was...
-
- def partition(times_list):
-
-
-"""
-
+
+
+import os,re
+
+output = ''
+todos = 0
+
+todos_d = {}
+
+# TODO: Make a second pass, and look for fxn calls inside of
+# each function. Draw a graphviz.
+#
+
+def fxns(fname):
+ global output, todos
+ lines = open(fname).readlines()
+ prev_L = ""
+ for L in lines:
+
+ # is it a todo
+ a = re.search(r'(TODO|todo)\s*:?\s*(.*)$',L)
+ if a:
+ output += "\t\ttodo: " + a.group(2) + "\n"
+ todos += 1
+ if fname in todos_d:
+ todos_d[fname] += 1
+ else:
+ todos_d[fname] = 1
+
+ # is it a function def?
+ if re.search('^\s*def\s',L):
+ output += "\n"
+ if re.search('^#',prev_L): output += "\t"+prev_L
+ output += "\t"+L+""
+ prev_L = L
+
+files = os.listdir('.')
+files.sort()
+
+
+for F in files:
+ if F=='apphelp.py': continue
+ if re.search('\.py$',F):
+ output += "\n" + F + "\n"
+ fxns(F)
+
+prog_in = open('apphelp.py','r')
+prog_out = ''
+td = '# Total TODOs remaining: %i' % todos
+
+td2 = '# TODOs per file: \n#\n'
+for k in sorted(todos_d.keys()):
+ td2 += "#\t%i - %s \n" % (todos_d[k],k)
+
+
+for R in prog_in.readlines():
+ if re.search('^##\sF',R):
+ prog_out += "## Functions\n#\n%s\n#\n%s\n#\n" % (td,td2)
+ break
+ prog_out += R
+
+prog_out += '\n"""\n\n' + output + '\n\n"""\n\n'
+prog_in.close()
+prog_write = open('apphelp.py','w')
+prog_write.write(prog_out)
+prog_write.close()
+
+
+
+
+
+## Functions
+#
+# Total TODOs remaining: 67
+#
+# TODOs per file:
+#
+# 6 - content.py
+# 6 - courses.py
+# 3 - curriculum.py
+# 6 - depricated.py
+# 7 - localcache.py
+# 2 - outcomes.py
+# 17 - pipelines.py
+# 2 - server.py
+# 5 - tasks.py
+# 1 - tempget.py
+# 12 - users.py
+
+#
+
+"""
+
+
+__init__.py
+
+canvas_secrets.py
+
+checker.py
+
+ def safe_html(html):
+
+ def _attr_name_whitelisted(attr_name):
+
+ def safe_css(attr, css):
+
+ def plaintext(input):
+
+ def _unescape(text):
+
+ def fixup(m):
+
+ def check_folder(fname,path):
+
+ def check_class(folder):
+
+ def check_all():
+
+content.py
+
+ def d(s):
+
+ # Build a master file with the entire class content
+ def accessible_check(id=""):
+ todo: include linked pages even if they aren't in module
+
+ def pan_testing():
+
+ # Given course, page url, and new content, upload the new revision of a page
+ def create_page(course_num,new_title,new_content):
+
+ def md_to_course():
+
+ # DL pages only
+ def grab_course_pages(course_num=-1):
+
+ # Download, clean html, and reupload page
+ def update_page():
+
+ # Given course, page url, and new content, upload the new revision of a page
+ def upload_page(course_num,pageurl,new_content):
+
+ # Use template to build html page with homegrown subtitles
+ def build_srt_embed_php(data):
+
+ def yt_title(code):
+
+ def swap_youtube_subtitles():
+
+ def test_swap():
+
+ def multiple_downloads():
+
+ def demo_vector_search():
+
+ def is_complete_sentence(text):
+ todo: site scraper
+ todo: find package that extracts text from web page
+ todo: master list of what to index.
+ todo: PDFs and DOCXs
+ todo: fix urls w/ anchors
+
+ def clean_fn(s):
+
+ def format_html(html):
+
+ def visit(self, link, source=None):
+
+ def fail(self, link):
+
+ def crawl():
+
+ def txt_clean_index():
+
+ def samples():
+
+courses.py
+ todo:
+
+ # Gott 1 Bootcamp - report on who completed it.
+ def get_gott1_passers():
+
+ # Plagiarism Module - report on who completed it.
+ def get_plague_passers():
+
+ # Who, in a class, passed?
+ def get_course_passers(course, min_passing, passers_filename, still_active_filename):
+
+ # Who, in a class and a quiz, passed?
+ def get_quiz_passers():
+
+ # Change courses to show 2 announcements
+ def change_course_ann_homepage(id="10458"):
+
+ # All students enrolled in a class in the given semester. Simpler verson of below. Return SET of course_ids.
+ def users_in_semester():
+ todo:
+
+ # All students in STEM (or any list of depts.. match the course_code). Return SET of canvas ids.
+ def users_in_depts_live(depts=[], termid='171'):
+
+ def course_enrollment(id=''):
+
+ def askForTerms():
+
+ # Return a list of term names and IDs. Also store in cache/courses/terms.txt
+ def getTerms(printme=1, ask=1):
+ todo: unsafe overwrite
+
+ def getCourses(x=0): # a dict
+
+ def update_course_conclude(courseid="13590",enddate='2021-12-23T01:00Z'):
+
+ # Relevant stuff trying to see if its even being used or not
+ def course_term_summary_local(term="176",term_label="FA22"):
+
+ # Relevant stuff trying to see if its even being used or not
+ def course_term_summary(term="176",term_label="FA22"):
+
+ # Fetch all courses in a given term
+ def getCoursesInTerm(term=0,get_fresh=1,show=0,active=0): # a list
+
+ def getCoursesTermSearch(term=0,search='',v=0):
+
+ def courseLineSummary(c,sections={}):
+
+ def xlistLineSummary(c,sections={}):
+
+ def numbers_in_common(L):
+
+ def combined_name(nic,L):
+
+ def semester_cross_lister():
+
+ def xlist_ii(parasite_id,host_id,new_name,new_code):
+
+ def all_semester_course_sanity_check():
+
+ def eslCrosslister():
+
+ def xlist(parasite='', host=''): # section id , new course id
+ todo: need to get the section id from each course:
+
+ def unenroll_student(courseid,enrolid):
+
+ def enroll_stem_students_live():
+
+ def enroll_bulk_students_bydept(course_id, depts, the_term="172", cautious=1): # a string, a list of strings
+ todo: not done here
+
+ def enroll_art_students_live():
+
+ def enroll_orientation_students():
+
+ def enroll_o_s_students():
+
+ def make_ztc_list(sem='sp20'):
+
+ def course_search_by_sis():
+
+ def mod_eval_visibility( shell_id, visible=True ):
+
+ def instructor_list_to_activate_evals():
+
+ def add_evals(section=0):
+
+ def course_dates_terms(section=0):
+
+ def remove_n_analytics(section=0):
+
+ def create_sandboxes():
+
+ def course_term_summary_2():
+
+ def get_ext_tools():
+
+ def set_ext_tools():
+ todo: wanted: group shell for each GP (guided pathway) as a basic student services gateway....
+
+cq_demo.py
+
+ def fetch(target):
+
+curric2022.py
+
+ def fetch_all_programs():
+
+ def nothing(x=0):
+
+ def clean(st):
+
+ def recur_matcher(item, depth=0):
+
+ def single_course_parse(c):
+
+ def match_style_test():
+
+ def single_program_path_parse(c):
+
+ def path_style_prog():
+
+ def term_txt_to_code(t):
+
+ def all_outcomes():
+
+ def ddl():
+
+ def splitclassline(cl, id=''):
+
+ def path_style_2_html():
+
+ def course_path_style_2_html():
+
+ def another_request(url,startat):
+
+ def fetch_all_classes():
+
+ def recur_path_matcher(item, path=[]):
+
+ def x2_path_update(x,y,z):
+
+ def pathstyle(theclass):
+
+ def single_course_path_parse(c):
+
+ def path_style_test():
+
+ def make_sl():
+
+ def course_rank():
+
+curriculum.py
+
+ def dbg(x):
+
+ def another_request(url,startat):
+
+ def fetch_all_classes():
+
+ def fetch_all_programs():
+
+ def sortable_class(li):
+
+ def c_name(c):
+
+ def show_classes(createoutput=1):
+
+ def clean_d_name(d):
+
+ def show_programs():
+
+ def dd(): return defaultdict(dd)
+
+ def organize_courses():
+
+ def check_de():
+
+ def clean_programs():
+
+ def course_lil_format(s):
+
+ def header_lil_format(s):
+
+ def organize_programs():
+
+ def divide_courses_list(li,rwd,online):
+
+ def organize_programs2():
+
+ # sorting by order key of dict
+ def cmp_2(a):
+
+ def cmp_order(a,b):
+
+ # decipher the grouped up courses line
+ def split_course(st):
+
+ # Any number gets an X (checked). Blank or zero gets no check.
+ def units_to_x(u):
+
+ def p_block_rule(r,printme,doc,out=0):
+
+ def p_cert_header(type,doc,r='',out=0):
+
+ def p_block_header(r,doc,out=0):
+
+ def p_cert_course_missing(cd,doc,out=0):
+
+ def p_cert_course(cd,history,doc,out=0):
+
+ def p_end_block(out=0):
+
+ def p_end_cert(bigdoc, out=0):
+
+ def ask_for_rule(r):
+
+ def action_to_english(a):
+
+ # Return True if the courses satisfy the rule
+ def check_a_block(b, courses, verbose=False):
+
+ def read_block_english_to_code():
+
+ def read_section_online_history():
+ todo: this file depends on other fxns. which?
+
+ # This is the 3rd attempt.
+ def simple_find_online_programs():
+ todo: courses with a HYPHEN in NAME get parsed wrong.
+
+ def check_a_block_a(b,verbose=False):
+
+ def smart_find_online_programs():
+
+ def show_contained_class(c):
+
+ def show_block(c):
+
+ def show_block(c):
+
+ def is_online(c):
+
+ def is_online(c):
+
+ def is_online_inblock(c):
+
+ def is_online_inblock(c):
+
+ # 9/2021 clean programs to good json
+ def organize_programs_stage2():
+
+ # of all the programs, what can be accomplished online?
+ def find_online_programs():
+
+ # take a string of all the types of classes offered, return a vector of [tot,lec,hyb,onl]
+ def string_to_types(st):
+
+ def my_default_counter():
+
+ # Of the recent schedules, what was actually offered online?
+ def summarize_online_sections():
+
+ def fibonacci(n):
+
+ def test_pampy():
+
+ def cq_parse_experiment(root=0, indent=''):
+
+ def cq_start():
+
+ def cq_pattern_backup1(root=0, indent=''):
+
+ def found(*x):
+
+ def lookForMatch(rules,item):
+
+ def cq_pattern(root=0, indent=''):
+
+ def myprinter(item, indent=''):
+
+ def cq_pattern_start():
+
+ def baby_int(j):
+
+ def find_deg_in_cluster( clusters, deg ):
+
+ def try_match_deg_programs():
+
+ def dict_generator(indict, pre=None):
+
+ def print_dict(v, prefix='',indent=''):
+
+ def walk_file():
+
+ def tag(x,y): return "<%s>%s" % (x,y,x)
+
+ def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
+
+ def a(t,h): return '%s' % (h,t)
+
+ def server_save(key,value):
+
+ def flask_thread(q):
+
+ def home():
+
+ def s(key,val):
+
+ def hello():
+
+ def sd():
+
+ def serve():
+
+ def attempt_match8020(rules,item):
+
+ def clever_printer(item, indent=''):
+
+ def print_return(x):
+
+ def cq_8020(root=0, indent=''):
+
+ def cq_8020_start():
+
+ def recurse3(sec,path=''):
+
+ def get_id_sortorder(sec):
+
+ def include_exclude(str,inc,exc=[]):
+
+ def pbd3(str):
+
+ def handleField(f):
+
+ def boolToStr(b):
+
+ # Almost final formatting
+ def prog_info_to_entry(c):
+
+ def cbd_to_entry(c):
+
+ def pc5(str):
+
+ def remove_prefix(str,i):
+
+ def course_to_entry(c,order="0"):
+
+ def courseline_to_pretty(line):
+
+ # restarted oct 2019 and try to simplify
+ def prog_take_4(program):
+ todo:
+
+curriculum2020.py
+
+ def to_md(s):
+
+ def print_return(x):
+
+ def cq_8020(root,indent=0):
+
+ def cq_8021(root,indent=0):
+
+ def cq_8021_start():
+
+ def cq_8022(root,indent=0):
+
+ def cq_8022_start():
+
+ def sortable_class(li):
+
+ def c_name(c):
+
+ def show_classes2020():
+
+ def show_classes2020_start():
+
+curriculum_patterns.py
+
+ def div1(a,b):
+
+ def d2(a,b):
+
+ def d3(a,b):
+
+ def pp1(a,b):
+
+ def pp2(a,b):
+
+ def pp3(a,b,c,d,e,f,g):
+
+ def pp4(a,b):
+
+ def pp5(a,b,c):
+
+ def pp6(a,b):
+
+ def pp7(a,b):
+
+ def pp8(a,b):
+
+ def jj1(a,b,c,d,e):
+
+ def jj2(a,b,c,d,e,f):
+
+ def jj3(a,b,c,d,e):
+
+ def jj4(a,b,c,d):
+
+ def jj5(a,b,c,d,e,f):
+
+ def jj6(a,b,c,d):
+
+ def jj2(a,b,c,d):
+
+ def jj2(a,b,c,d):
+
+depricated.py
+ todo: where does the most recent schedule come from?
+
+ # Input: xxxx_sched.json. Output: xxxx_latestarts.txt
+ def list_latestarts():
+
+ def prep_online_courses_df():
+
+ def course_is_online(crn):
+
+ def get_crn_from_name(name):
+
+ def get_enrlmts_for_user(user,enrollments):
+
+ # Don't know
+ def demo():
+
+ def stats():
+
+ def dict_generator(indict, pre=None):
+
+ def print_dict(v, prefix='',indent=''):
+
+ def walk_file():
+
+ def tag(x,y): return "<%s>%s" % (x,y,x)
+
+ def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
+
+ def a(t,h): return '%s' % (h,t)
+
+ def server_save(key,value):
+
+ def flask_thread(q):
+
+ def home():
+
+ def s(key,val):
+
+ def hello():
+
+ def sd():
+
+ def serve():
+
+ def summarize_proportion_online_classes(u):
+
+ def summarize_num_term_classes(u):
+ todo: this duplicates courses.py ??
+
+ # Prompt for course id, return list of user dicts. TODO this duplicates courses.py ??
+ def getUsersInCourse(id=0): # returns list
+
+ def recur_look_for_leafs(item,indent=0,show=1):
+
+ def am_i_a_leaf(item):
+
+ def sampleclass():
+
+ def matchstyle():
+
+ def user_role_and_online():
+
+ def more_unused_xreferencing():
+
+ def users_p_file():
+
+ def com_channel_dim():
+ todo:
+
+ # NO LONGER USED - SEE COURSES
+ def enroll_stem_students():
+
+ # unused?
+ def getAllTeachersInTerm(): # a list
+ todo: hits in courses by teachers https://gavilan.instructure.com:443/api/v1/users/2/page_views?end_time=Dec%2010%2C%202018
+
+ def teacherActivityLog(uid=1): ### Next: save results in a hash and return that....
+
+ def summarize_student_teacher_role(u):
+
+ def user_roles2():
+
+ def req_to_db(fname_list):
+
+ def has_online(series):
+
+ def has_lecture(series):
+
+ def has_hybrid(series):
+
+ # Wrapper to get 2 schedules at once
+ def dl_sched():
+ todo: these semesters
+
+ # Send a personalized email regarding ZTC
+ def send_z_email(fullname, firstname, addr, courses_list):
+
+ def getInactiveTeachersInTerm(t=23): # a list
+
+ def course_location(course):
+
+ def course_time(course):
+
+ def course_teacher(course):
+
+ def reg_nums():
+
+ # In the schedule, is this a class or a continuation of the class above?
+ def categorize():
+ todo: must we open all these files?
+
+ # Deprecated. call perl.
+ def constructSchedule():
+
+ def fetch_dict(target,params={}):
+
+ def get_schedule(term='201870', sem='fall'):
+
+ def dates(s):
+
+ def parse_www_csv_sched():
+
+ def parse_json_test_sched():
+
+ def put_revised_pages():
+
+ def put_course_pages():
+
+ def freshdesk():
+
+gpt.py
+
+graphics.py
+
+interactive.py
+
+ def dict_generator(indict, pre=None):
+
+ def print_dict(v, prefix='',indent=''):
+
+ def walk_file():
+
+ def tag(x,y): return "<%s>%s" % (x,y,x)
+
+ def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
+
+ def a(t,h): return '%s' % (h,t)
+
+ def server_save(key,value):
+
+ def flask_thread(q):
+
+ def before_request():
+
+ def clears():
+
+ def dpi():
+
+ def dpi2():
+
+ def screenoff_a():
+
+ def light():
+
+ def do_image(filename):
+
+ def do_image_crop(filename,x,y,w,h,newname):
+
+ def save_post():
+
+ def writing_img(fname):
+
+ def restart():
+
+ def dispatch3(func,arg,arrg):
+
+ def dispatch2(func,arg):
+
+ def dispatch(func):
+
+ def dispatch3j(func,arg,arrg):
+
+ def dispatch2j(func,arg):
+
+ def dispatch1j(func):
+
+ def home():
+
+ def send_cachedata(path):
+
+ def send_js(path):
+
+ def send_jslib(path):
+
+ def s(key,val):
+
+ def do_sample():
+
+ def media(file_id):
+
+ def podcast():
+
+ def weblec():
+
+ def hello():
+
+ def sd():
+
+ def test_message(message):
+
+ def serve():
+
+ def make_teacher_rel(self, tchr, clss):
+
+ def __init__(self, uri, user, password):
+
+ def close(self):
+
+ def print_greeting(self, message):
+
+ def _create_and_return_greeting(tx, message):
+
+ def make_teacher_rel(g, tchr, clss):
+
+ def testgraph():
+
+ def Memoize( func):
+
+ def wrapper(*args):
+
+ def startup(self, outfile):
+
+ def set_my_dict(self,d):
+
+ def cycle_color(self, s):
+
+ def ascii_art(self, text):
+
+ def close_window(self, ):
+
+ def suggest(self, word):
+
+ def curses_print_word(self, word,color_pair_code):
+
+ def curses_print_line(self, line,color_pair_code):
+
+ def redraw(self, start_y,end_y,fallback_y,fallback_x):
+
+ def scroll_down(self, noredraw,fallback_y,fallback_x):
+
+ def clear_upside(self, n,y,x):
+
+ def display_suggest(self, y,x,word):
+
+ def inputloop(self, ):
+
+ def setup_command(self,outfile):
+
+ def cleanup_command(self):
+
+ def handle_command(self, cmd):
+
+ def repl_staff():
+
+ def repl_degs():
+
+ def repl():
+
+ def repl():
+
+interactivex.py
+
+ def dict_generator(indict, pre=None):
+
+ def print_dict(v, prefix='',indent=''):
+
+ def walk_file():
+
+ def tag(x,y): return "<%s>%s" % (x,y,x)
+
+ def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
+
+ def a(t,h): return '%s' % (h,t)
+
+ def server_save(key,value):
+
+ def flask_thread(q):
+
+ def before_request():
+
+ def restart():
+
+ def dispatch3(func,arg,arrg):
+
+ def dispatch2(func,arg):
+
+ def dispatch(func):
+
+ def dispatch3j(func,arg,arrg):
+
+ def dispatch2j(func,arg):
+
+ def dispatch1j(func):
+
+ def home():
+
+ def send_jslib(path):
+
+ def send_cachedata(path):
+
+ def send_js(path):
+
+ def s(key,val):
+
+ def do_sample():
+
+ def hello():
+
+ def sd():
+
+ def serve():
+
+ def make_teacher_rel(self, tchr, clss):
+
+ def __init__(self, uri, user, password):
+
+ def close(self):
+
+ def print_greeting(self, message):
+
+ def _create_and_return_greeting(tx, message):
+
+ def make_teacher_rel(g, tchr, clss):
+
+ def Memoize( func):
+
+ def wrapper(*args):
+
+ def startup(self, outfile):
+
+ def set_my_dict(self,d):
+
+ def cycle_color(self, s):
+
+ def ascii_art(self, text):
+
+ def close_window(self, ):
+
+ def suggest(self, word):
+
+ def curses_print_word(self, word,color_pair_code):
+
+ def curses_print_line(self, line,color_pair_code):
+
+ def redraw(self, start_y,end_y,fallback_y,fallback_x):
+
+ def scroll_down(self, noredraw,fallback_y,fallback_x):
+
+ def clear_upside(self, n,y,x):
+
+ def display_suggest(self, y,x,word):
+
+ def inputloop(self, ):
+
+ def setup_command(self,outfile):
+
+ def cleanup_command(self):
+
+ def handle_command(self, cmd):
+
+ def repl_staff():
+
+ def repl_degs():
+
+ def repl():
+
+ def repl():
+
+localcache.py
+
+ def db():
+
+ def setup_table(table='requests'):
+
+ # Help the next function to upload new users directly to conf database on gavilan.
+ def employees_refresh_flex(data):
+
+ # Everyone in iLearn DB with an xyz@gavilan.edu email address.
+ def all_gav_employees():
+
+ #
+ def teachers_courses_semester():
+
+ #
+ def teachers_by_term():
+
+ # Report for AEC
+ def aec_su20_report():
+
+ # Return the most up do date version of the given file. Useful for 'dimensions'.
+ def most_recent_file_of( target ):
+
+ def finder(st):
+
+ # Given a table schema, parse log file, return a list of dicts. Optionally remove some columns.
+ def parse_file_with( file, format, with_gid=0 ):
+
+ # I return a list of the read lines if the log dates in the file are within dates (top of this file), or FALSE
+ def is_requestfile_interesting(fname):
+ todo: more robust here
+ todo: - investigate pearson, developer key: 170000000000376 and their ridiculous amounts of hits.
+
+ # Return a 'timeblock'. An integer number of 15 minute blocks from my epoch. Expects a datetime object in PST timezone.
+ def timeblock_from_dt(dt_obj):
+
+ # Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+ def dt_from_timeblock(tb):
+
+ # Twenty Four hour timeblocks
+ def timeblock_24hr_from_dt(dt_obj):
+
+ # Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+ def dt_from_24hr_timeblock(tb):
+
+ # Four hour timeblocks
+ def timeblock_4hr_from_dt(dt_obj):
+
+ # Returns a time in PST, given a 'timeblock'. Will be used in translating back to human time
+ def dt_from_4hr_timeblock(tb):
+
+ # I make the line into a dict, erase keys with no data, make a DT field called date, make a time_block (int) field.
+ def requests_line(line,i=0):
+
+ # Bulk insert of requests logs. Too much data to be useful.
+ def requests_file(fname_list):
+ todo: select if timeblock exists
+
+ # Insert or update a request line.
+ def upsert_request(line, vals):
+
+ # Generic insert of a dict into a table. Keys of dict must match table columns.
+ def dict_to_insert(thisline,table): # a dict
+
+ # This now does tallying by timeblock.
+ def merge_requests():
+
+ def merge_comm_channel():
+
+ def merge_pseudonym():
+
+ def merge_users():
+
+ def merge_courses():
+
+ def merge_enrollment():
+
+ def merge_term():
+
+ def merge_roles():
+
+ def merge_convos():
+
+ # For returning sqlite results as dicts
+ def dict_factory(cursor, row):
+ todo: ... approaches to all this data... list requests in order descending time, unique users, and just
+
+ # Attempt to do tallying
+ def make_views_summarys():
+
+ # original without time_blocks info.
+ def make_views_summarys_v1():
+
+ # Setup my basic db stats base from scratch
+ def full_reload():
+
+ def guess_dept(t):
+
+ # Main view of all class / all user overview...
+ def dept_with_studentviews(dept="", sem=''):
+
+ def f(x):
+
+ def get_courses_in_term_local(term="172"):
+
+ # get student count
+ def course_student_stats(canvasid):
+
+ # get teacher name from local db
+ def course_quick_stats(canvasid):
+
+ # What a student has taken / teacher has taught
+ def user_enrolled_in(userid):
+
+ # All students in this semester ...
+ def users_this_semester_db(sem=''):
+
+ # Everyone whose first semester is .....
+ def users_new_this_semester(sem=''):
+
+ # All student users in STEM - from local db
+ def user_in_stem():
+
+ # Get all the classes in one dept
+ def dept_classes(dept,sem=''):
+ todo:
+
+ def depts_with_classcounts(sem=''):
+ todo:
+
+ def f(x):
+
+ def name_with_count(name,li):
+
+ def arrange_data_for_web(dept='', sem=''):
+
+ def f(x):
+
+ # Get enrollments. (Best to freshly run pipelines/get_rosters) and put them into DB
+ def build_tables(headers,name):
+
+ def load_tables(table,headers,row,verbose=0):
+
+ def semester_enrollments(verbose=0):
+
+ def qstrip(txt): return txt.strip('"')
+
+ def comm_channel_file():
+
+ def pseudonym_file():
+
+ def abcd():
+
+ def crns_to_teachers():
+
+ def all_sem_courses_teachers():
+
+ def to_sis_sem(s):
+
+ def build_db_schedule():
+
+ def finder(st):
+
+ def process_enrollment_data():
+
+ def sem_to_idx(s):
+ todo:
+
+ def do_encoding():
+
+main.py
+
+myconsole.py
+
+ def handler(signum, frame):
+
+ def mainloop():
+
+outcomes.py
+
+ def outcome_overview(term=21):
+
+ def create_acct_lvl_outcomes(src,dept,makefolder='',folder=0):
+
+ def connect_acct_oc_to_course(course_id,oc_group_id):
+
+ def outcome_groups_dump():
+
+ def outcome_groups_backup():
+
+ def create_course_group(short,parent):
+
+ def create_dept_group(short):
+
+ def outcomes_attached_to_courses(term=65,limitdept=''):
+ todo: Handle this: CSIS/DM85 WEB DESIGN 40823/24
+
+ def summarize_course_online_slo(outcome_list):
+
+ def fetch_outcome_details(id):
+
+ # Report on the actual evaluation data?
+ def outcome_report1():
+ todo:
+
+ # For the given course, get all outcome measurements, and display scores and stats.
+ def outcome_report2():
+
+ def fix_joined_class(str):
+
+ def split_slo_name(str):
+
+ def outcome_report3():
+
+ def read_slo_source():
+
+ def slo_source_by_dept():
+
+ def printj(j):
+
+ def writej(o,j):
+
+ # Get root outcome group
+ def root_og():
+
+ def recur_og():
+
+ def recur_main(out,g_url=""):
+
+ def recur2(out,og={}):
+
+ def all_og():
+
+ def course_slo_getter(q):
+
+ def threaded_getter():
+
+ def demo_o_fetch():
+
+ def outcome_groups_2021():
+
+ def x_ref_dept_names():
+
+ def all_outcome_results_in_term(termid=''):
+
+ def all_outcome_results_in_term_sub(termid=''):
+
+ def all_linked_outcomes_in_term(termid=''):
+
+ def all_linked_outcomes_in_term_sub(termid=''):
+
+ def assemblerow(g,parent=''):
+
+ def recur_full_fetch(out,g,parent=""):
+
+ # return the appropriate cq course version.
+ def find_cq_course_version(code):
+
+ def outcome_groups():
+
+ def summary_string(s):
+
+ def add_outcomes_course_id(canvas_id):
+
+ def add_outcomes_course_code():
+
+ def add_outcomes_course_code_sub(target_code='AJ184',term=178,fresh=0):
+
+ def add_csis_sp22():
+
+ def quick_add_course_outcomes(ilearn_course_id, cq_outcome_id_list):
+
+ def stringpad(s,n):
+
+ def code_from_ilearn_name(n,verbose=0):
+
+ def parse_ilearn_course_names_ALLSEMESTERS():
+
+ def parse_ilearn_course_names(term='178',fresh=1,log=0):
+
+outcomes2022.py
+
+ def course_slo_getter(q):
+
+ def ilearn_shell_slo_to_csv(shell_slos):
+
+patterns_8020.py
+
+patterns_topdown.py
+
+ def pp0(a,b,c,d,e):
+
+ def pp1(a,b,c,d,e,f):
+
+ def pp2(a,b,c,d,e):
+
+ def pp3(a,b,c,d):
+
+ def pp4(a,b,c):
+
+ def pp5(a,b,c):
+
+ def pp6(a,b,c):
+
+ def div1(a,b):
+
+ def d2(a,b):
+
+ def d3(a,b):
+
+ def pp1(a,b):
+
+ def pp2(a,b):
+
+ def pp3(a,b,c,d,e,f,g):
+
+ def pp4(a,b):
+
+ def pp5(a,b,c):
+
+ def pp6(a,b):
+
+ def pp7(a,b):
+
+ def pp8(a,b):
+
+ def jj3(a,b,c,d,e):
+
+ def jj5(a,b,c,d,e,f):
+
+ def jj2(a,b,c,d):
+
+ def jj2(a,b,c,d):
+
+pipelines.py
+ todo: all these constants for SSB -- line 1008
+ todo: https://stackoverflow.com/questions/42656247/how-can-i-use-canvas-data-rest-api-using-python
+
+ def d(s,end=''):
+
+ # Main canvas querying fxn
+ def fetch(target,verbose=0):
+
+ # Main canvas querying fxn - stream version - don't die on big requests
+ def fetch_stream(target,verbose=0):
+
+ # paging makes problems... example: enrollment_terms
+ def fetch_collapse(target,collapse='',verbose=0):
+
+ # Teacher name format changed. Remove commas and switch first to last
+ def fix_t_name(str):
+
+ # Separate dept and code
+ def split_class_dept(c):
+
+ def split_class_code(c):
+
+ def split_class_code_letter(c):
+
+ # go from sp20 to 2020spring
+ def shortToLongSem(s):
+
+ # Go to the semesters folder and read the schedule. Return dataframe
+ def getSemesterSchedule(short='sp21'): # I used to be current_schedule
+ todo: Some semesters have a different format.... partofday type site xxx i just dL'd them again
+
+ def get_enrlmts_for_user(user,enrollments):
+
+ # Get something from Canvas Data
+ def do_request(path):
+
+ # Canvas data, download all new files
+ def sync_non_interactive():
+
+ # list files in canvas_data (online) and choose one or some to download.
+ def interactive():
+
+ def todays_date_filename(): # helper
+
+ def nowAsStr(): # possible duplicate
+
+ def row_has_data(r): # helper
+
+ def row_text(r): # helper
+
+ # Take banner's html and make a csv(?) file
+ def ssb_to_csv(src):
+
+ def clean_funny(str):
+
+ def clean_funny2(str):
+
+ def clean_funny3(str):
+
+ ### course is a list of 1-3 lists, each one being a line in the schedule's output. First one has section
+ def course_start(course):
+ todo: use this to make a early/late/short field and store semester dates w/ other constants
+ todo: do these years matter?
+
+ def time_to_partofday(t):
+ todo: account for multiple sites/rows
+
+ # Deduce a 'site' field, based on room name and known offsite locations
+ def room_to_site(room,verbose=0):
+ todo: account for multiple sites/rows
+ todo: better way to store these offsite labels
+
+ # take text lines and condense them to one dict per section
+ def to_section_list(input_text,verbose=0):
+ todo: no output files
+ todo: if extra line is different type?
+
+ # Log the history of enrollments per course during registration
+ def log_section_filling(current_sched_list):
+
+ # Same as above, but compressed, act only
+ def log_section_filling2(current_sched_list):
+
+ # Use Firefox and log in to ssb and get full schedule. Only works where selenium is installed
+ def scrape_schedule():
+ todo:
+
+ def dza_sched():
+
+ # recreate schedule json files with most current online schedule format.
+ def recent_schedules():
+ todo: sems is a global in this file. Is that the right thing to do?
+ todo: the pipeline is disorganized. Organize it to have
+ todo: where does this belong in the pipeline? compare with recent_schedules()
+
+ # Take the generically named rosters uploads files and move them to a semester folder and give them a date.
+ def move_to_folder(sem,year,folder):
+
+ # This relates to enrollment files, not schedule.
+ def convert_roster_files(semester="",year="",folder=""):
+
+ # From instructure sftp site
+ def fetch_current_rosters():
+
+ def fetch_current_rosters_auto():
+
+ # read schedule file with an eye toward watching what's filling up
+ def schedule_filling():
+ todo: hardcoded
+
+ # Upload a json file to www
+ def put_file(remotepath,localpath, localfile,prompt=1):
+ todo: these paths
+
+ def sec(t): return ""+t+"
\n"
+
+ def para(t): return ""+t+"
\n"
+
+ def ul(t): return "\n"
+
+ def li(t): return ""+t+"\n"
+
+ def question(t,bracket=1):
+
+ def answer(t):
+
+ def read_paragraph_element(element,type="NORMAL_TEXT"):
+
+ def get_doc(docid, bracket=1, verbose=0):
+ todo: x link, x bold, list, image.
+
+ def read_paragraph_element_2(element,type="NORMAL_TEXT"):
+
+ # t is a string that begins with "Icons: " ... and contains comma(space) separated list
+ def handle_icons(t):
+
+ # t is a string that begins with "Tags: " ... and contains comma(space) separated list
+ def handle_tags(t):
+
+ def handle_question(t,bracket=1):
+
+ def handle_answer(t):
+
+ def handle_sec(t): return ('section',t)
+
+ def handle_para(t): return ('paragraph',t)
+
+ def handle_ul(t): return ('unorderdedlist',t)
+
+ def handle_li(t): return ('listitem',t)
+
+ def fetch_doc_image(k,value):
+
+ def get_doc_generic(docid, bracket=1, verbose=0):
+
+ def scrape_schedule_py():
+
+ def scrape_schedule_multi():
+
+ def scrape_for_db():
+
+ def argos_data():
+
+ def days_times(s):
+
+ def remove_year(s):
+
+ def argos_data_from_cvc():
+
+ def expand_old_semesters():
+
+ # Input: xxxx_sched.json. Output: xxxx_latestarts.txt
+ def list_latestarts(term="su23"):
+
+server.py
+
+ def mqtt_loop():
+
+ # called when MQTT server connects
+ def on_connect(client, userdata, flags, rc):
+
+ # The callback for when a PUBLISH message is received from the server.
+ def on_message(client, userdata, msg):
+
+ def displaypi_on():
+
+ def displaypi_off():
+
+ def desklight():
+
+ def clearscreens():
+
+ def screenoff():
+
+ def tag(x,y): return "<%s>%s" % (x,y,x)
+
+ def tagc(x,c,y): return '<%s class="%s">%s' % (x,c,y,x)
+
+ def a(t,h): return '%s' % (h,t)
+
+ def homepage():
+
+ def orgline(L):
+ todo: \s\[\#A\](.*)$', L)
+
+ def editor(src):
+
+ def in_form(txt,path):
+
+ def mytime(fname):
+
+ def index():
+
+ def writing(fname):
+
+ def dashboard():
+
+ def dash():
+
+ def mycalendar():
+
+ def most_recent_file_of( target, folder ):
+
+ def finder(st):
+
+ def news():
+
+ def randPic():
+
+ def do_img_crop(im):
+
+ def sample():
+
+ def sample2(a=""):
+
+ # Filter a stream of loglines for those that match a course's url / id
+ def has_course(stream,courseid):
+
+ def js(s):
+
+ def sem_from_array_crn(crn):
+
+ def user_courses(uid):
+
+ def user_course_history_summary(usr_id):
+
+ def roster(crn):
+
+ def user_course_hits(usr,courseid):
+
+ def profiles(id=1,b=2,c=3):
+
+ # Departments, classes in each, and students (with hits) in each of those.
+ def enrollment(a):
+
+ # All the classes in this dept, w/ all the students in each, with count of their views.
+ def dept(d=''):
+
+ def user(canvas_id=None):
+
+ def lectures():
+
+ def web_lectures():
+ todo: update: dept, title, any of the other fields.
+
+ # update a value: dept id of a personnel id
+ def update_pers_title(pid, tid):
+
+ # update a value: dept id of a personnel id
+ def update_pers_dept(pid, did):
+
+ def user_edit(canvas_id='2'):
+
+ def staff_dir(search=''):
+
+ def find_goo(n):
+
+ def byname(x):
+
+ def fn_to_struct( n, staff ):
+
+ def image_edit(filename=''):
+
+ def image_crop(filename,x,y,w,h,newname=''):
+
+ def server_save(key,value):
+
+ def server_dispatch_json(function_name,arg='', arg2=''):
+
+ def server_dispatch(function_name,arg='', arg2=''):
+
+stats.py
+
+ def grades_rundown():
+
+ def class_logs():
+
+ def user_logs():
+
+ def recent_logins():
+
+ def userHitsThisSemester(uid=2):
+
+ def getCurrentActivity(): # a dict
+
+ def externaltool(): # a list
+
+tasks.py
+
+ def scrape_bookstore():
+
+ def survey_answer(q=0):
+
+ def survey_organize():
+
+ def build_quiz(filename=""):
+
+ # Send an email
+ def send_email(fullname, firstname, addr, subj, content):
+
+ def convert_to_pdf(name1, name2):
+
+ # Build (docx/pdf) certificates for gott graduates
+ def certificates_gott_build():
+
+ # Email experiment
+ def mail_test():
+
+ # Change LTI Settings. Experimental
+ def modify_x_tool():
+
+ # Upload with sftp to www website folder: student/online/srt/classfoldername
+ def put_file(classfoldername):
+ todo: ',cnopts=cnopts) as sftp:
+
+ # Switch everyone in a class to a teacher
+ def switch_enrol():
+
+ # Change dates & term of a class to unrestrict enrollment
+ def unrestrict_course():
+
+ # Bulk enroll users into a course
+ def enroll_accred():
+
+ # Calculate attendance stats based on enrollment/participation at 20% of term progressed, then 60% of term progressed.
+ def twenty_sixty_stats(li):
+
+ # Older positive attendance hours calculation.
+ def hours_calc():
+
+ def course_2060_dates(crn=""):
+ todo:
+
+ def course_update_all_users_locallogs(course_id=''):
+
+ def hours_calc_pulldata(course_id=''):
+
+ def xlist_cwe():
+
+ def pos_atten():
+
+ def lname(x):
+
+ def l_initial(x):
+
+ def job_titles2():
+
+ def job_titles():
+
+ # an early version, before tearing up...
+ def job_titles3():
+
+ def index_pics():
+
+ def cmtes():
+
+ def strip(x): return x.strip()
+
+ def esc_comma(x): return re.sub(',','[CMA]',x)
+
+ def by_sem(x): return x['sem']
+
+ def parse_schedule():
+ todo: check if i need to update it
+ todo: some weird hour offset issue w/ these activities
+
+ def cal():
+ todo: >
+
+ def file_renamer():
+
+temp.py
+
+tempget.py
+
+ # Use Firefox and log in to ssb and get full schedule
+ def login():
+ todo: my data here.... secret
+
+ def filename_friendly(str):
+
+ def otter():
+
+templates.py
+
+ def item_to_masonry(item):
+
+ def try_untemplate():
+
+ def php_remove(m):
+
+ def php_add(m):
+
+ def do_template(temp,source,side):
+
+ def remove_filetype(f):
+
+ def make():
+
+ def txt_2_table():
+
+ def studenttech_faq():
+
+ # https://docs.google.com/document/d/1tI_b-q75Lzu25HcA0GCx9bGfUt9ccM8m2YrrioDFZcA/edit?usp=sharing
+ def de_faq():
+
+ def degwork_faq():
+
+ def vrc_faq():
+
+ def counseling_faq():
+
+ def finaid_faq():
+
+ def coun_loc():
+
+ def tutor_faq():
+
+ def test_repl():
+
+timer.py
+
+ def func(a, b):
+
+users.py
+ todo: these constants
+
+ # All users to a cache file cache/allusers.json
+ def fetchAllUsers():
+
+ # Fetch teacher users objects from local cache
+ def teacherRolesCache(): # I used to be load_users
+
+ # Canvas: Fetch all people with gavilan.edu email address
+ def teacherRolesUpdateCache(): # I used to be get_users
+
+ # Fetch preferred email address for a given user id. ( Canvas )
+ def getEmail(user_id):
+
+ # All teachers in a particular course
+ def getAllTeachers(course_id=59): # a list
+
+ #
+ def classType(t):
+ todo: fix bug in schedule parser so non-online classes have a type field
+
+ def my_blank_string(): return "no data"
+
+ def my_blank_dict(): return {'name':'NoName','email':'noemail@gavilan.edu'}
+
+ def my_empty_dict(): return defaultdict(my_blank_string)
+
+ def get_email_from_rec(name,name_to_record):
+
+ # Pull the staff directory on the webpage. Convert to pandas dataframe
+ def staff_dir(get_fresh=False):
+ todo: lol get fresh again...
+
+ def schedForTeacherOverview(long,short):
+
+ # Return a dataframe of the last 4 semester schedules put together
+ def oneYearSchedule():
+
+ def num_sections_last_year(line):
+
+ def sec_type_stats(line):
+
+ def prct_online(line):
+
+ def prct_lecture(line):
+
+ def prct_hybrid(line):
+
+ # Given the names of teachers in last year's schedules, fill in email, etc. from ilearn files
+ def teacher_basic_info(sched, from_ilearn, names):
+
+ def find_that_name(x):
+ todo: Old and broken
+
+ # Outputs: cache/teacher_by_semester.csv,
+ def teacherModalityHistory(sched=[],names=[]):
+
+ def teacherCourseHistory(a,names):
+ todo: sort by dept also
+
+ # Outputs: cache/course_teacher_combos.csv,
+ def teacherSharedCourses(a=[]):
+ todo: this is broken
+
+ # How many courses in each department were taught in the last year?
+ def departmentCountCourses(a=[]):
+
+ def clean_nonprint(s):
+
+ def read_cmte(names):
+
+ def read_training_records():
+
+ # open a file and mark the people with their ids given. Return a dataframe
+ def read_bootcamp1(filename):
+
+ # open a file and mark the people with their ids given. Return a dataframe
+ def read_bootcamp2(filename):
+
+ def not_blank_or_pound(L):
+
+ def temp1(x):
+
+ def add_realnames(df,names): # the surveys. raw name is in 2nd column
+
+ def compareToughNames(a,b):
+
+ def compareNames(a,b,verbose=0):
+
+ def find_ilearn_record(ilearn_records,manual_records, othername,verbose=0):
+
+ def manualNamesAndDept():
+
+ def manualNames():
+
+ # given a list of class codes, return the most common (academic) department
+ def guessDept(d_list):
+
+ # Make one big csv file of everything I know about a teacher
+ def getTeachersInfoMain():
+ todo: - broken
+
+ def enroll_staff_shell():
+
+ # take a list of raw hits.
+ def activity_summary(hits):
+ todo: month is hardcoded here
+
+ # Get views counts on current teachers. todo: month is hardcoded here
+ def get_recent_views(id=1):
+ todo: broken?
+
+ def categorize_user(u):
+ todo: threaded
+
+ # Doest the account have a photo loaded?
+ def checkForAvatar(id=2):
+
+ # Grab em. Change the first if when continuing after problems....
+ def downloadPhoto():
+
+ def mergePhotoFolders():
+
+ def mergePhotoFolders2():
+
+ # Go through my local profile pics, upload any that are missing.
+ def uploadPhoto():
+
+ def create_ztc_list():
+
+ def get_user_info(id):
+
+ # these are any messages that get pushed out to their email
+ def comm_mssgs_for_user(uid=0):
+
+ #
+ def convos_for_user(uid=0):
+
+ # single q sub
+ def quiz_get_sub(courseid, quizid, subid=0):
+
+ # quiz submissions for quiz id x, in course id y
+ def quiz_submissions(courseid=9768, quizid=32580):
+
+ # return (timeblock, course, read=0,write=1)
+ def requests_line(line,i=0):
+
+ #
+ def report_logs(id=0):
+
+ def track_users_in_sem():
+
+ def track_users_in_class(L=[]):
+
+ def track_user_q(id, q):
+
+ # Maintain local logs. Look to see if we have some, download logs since then for a user.
+ def track_user(id=0,qid=0):
+ todo: set up this info file if it isn't there. check any changes too. it
+ todo:
+
+ #
+ def track_users_by_teacherclass():
+
+ def nlp_sample():
+
+ def nlp_sample2():
+
+ def one_course_enrol():
+
+ def find_new_teachers():
+
+ def user_db_sync():
+
+ def find_no_goo():
+
+ def track_a_user():
+
+util.py
+
+ def stripper(s):
+
+ def mycleaner(s):
+
+ def print_table(table):
+
+ def remove_nl(str):
+
+ def UnicodeDictReader(utf8_data, **kwargs):
+
+ def minimal_string(s):
+
+ def to_file_friendly(st):
+
+ def clean_title(st):
+
+ def int_or_zero(x):
+
+ def float_or_zero(x):
+
+ def match59(x):
+
+ def item_2(x): return x[2]
+
+ def unix_time_millis(dt):
+
+ # ENGL250 returns ENGL
+ def dept_from_name(n):
+
+ def most_common_item(li):
+
+ def srt_times(a,b):
+
+ def how_long_ago(a): # number of hours ago 'a' was...
+
+ def partition(times_list):
+
+
+"""
+
diff --git a/content.py b/content.py
index a855d93..6bc1972 100644
--- a/content.py
+++ b/content.py
@@ -1,25 +1,24 @@
#saved_titles = json.loads( codecs.open('cache/saved_youtube_titles.json','r','utf-8').read() )
-import requests, codecs, os, re, json
+import requests, codecs, os, re, json, sys, pypandoc
+import webbrowser, bs4, trafilatura, pickle, tomd, checker
+import html2markdown as h2m
from pipelines import header, fetch, url, put_file
from util import clean_title, to_file_friendly, minimal_string, stripper, mycleaner
from bs4 import BeautifulSoup as bs
from html.parser import HTMLParser
-import tomd, checker
-import html2markdown as h2m
-import pypandoc
-import webbrowser
-h = HTMLParser()
+from collections import defaultdict
+from pdfminer.high_level import extract_text
+from sentence_transformers import SentenceTransformer, util
+h = HTMLParser()
DBG = 1
def d(s):
global DBG
if DBG: print(s)
-
-
@@ -828,63 +827,265 @@ Schedule an In-Person, Phone or Zoom Appointment"""
print(f"Vector for the word '{example_word}': {vector}")
+
+def makedir():
+ files = os.listdir('cache/crawl')
+ #print(files)
+ files.sort()
+ for f in files:
+ m = re.match(r'https?..www\.gavilan\.edu\+(.*)\.\w\w\w\w?\.txt$',f)
+ if m:
+ name = m.groups()[0]
+ parts = name.split('+')
+ print(parts)
+
+def manual_index():
+ files = os.listdir('cache/crawl')
+ #print(files)
+ ii = codecs.open('cache/crawl/index.html','w','utf-8')
+ ii.write('Site index
\n')
+ files.sort()
+ for f in files:
+ m = re.match(r'https?..www\.gavilan\.edu\+(.*)\.\w\w\w\w?\.txt$',f)
+ if m:
+ name = m.groups()[0]
+ parts = name.split('+')
+ ii.write('
'+f+'\n')
+
+def my_site():
+ files = os.listdir('cache/crawl')
+ output = []
+ files.sort()
+ for f in files:
+ m = re.match(r'https?..www\.gavilan\.edu\+(.*)\.\w\w\w\w?\.txt$',f)
+ if m:
+ name = m.groups()[0]
+ parts = name.split('+')
+ output.append(parts)
+ return output
+
+
## TODO site scraper
-
-## TODO finde package that extracts text from web page
-
+## TODO find package that extracts text from web page
### TODO master list of what to index.
-from pattern.web import URL, plaintext, extension
-from pattern.web import download
-from pattern import URL, MIMETYPE_IMAGE
-from pattern.web import Crawler
-from util import clean_title
-
-save_folder = 'cache/crawl'
-
-class GavCrawl(Crawler):
- def visit(self, link, source=None):
- print 'visited:', repr(link.url), 'from:', link.referrer
- txt = plaintext(link.source) ## , keep={'h1':[], 'h2':[], 'strong':[], 'a':['href']})
- codecs.open(save_folder + '/' + clean_title(link.url) + '.txt').write(txt)
-
- def fail(self, link):
- print 'failed:', repr(link.url)
+## TODO PDFs and DOCXs
+## TODO fix urls w/ anchors
def crawl():
- p = GavCrawl(links=['http://www.gavilan.edu/'], delay=3)
- while not p.done:
- p.crawl(method=DEPTH, cached=False, throttle=3)
+ import scrapy, logging
+ from scrapy.crawler import CrawlerProcess
+
+ logger = logging.getLogger()
+ logger.setLevel(level=logging.CRITICAL)
+ logging.basicConfig(level=logging.CRITICAL)
+ logger.disabled = True
+
+
+ avoid = ['ezproxy','community\.gavilan\.edu','archive\/tag','archive\/category', 'my\.gavilan\.edu', 'augusoft',
+ 'eis-prod', 'ilearn\.gavilan', 'mailto', 'cgi-bin', 'edu\/old\/schedule', ]
+
+ class MySpider(scrapy.Spider):
+ name = 'myspider'
+ #start_urls = ['https://gavilan.curriqunet.com/catalog/iq/1826']
+ start_urls = ['https://www.gavilan.edu']
+
+
+ """
+ logging.getLogger("scrapy").setLevel(logging.CRITICAL)
+ logging.getLogger("scrapy.utils.log").setLevel(logging.CRITICAL)
+ logging.getLogger("scrapy.extensions.telnet").setLevel(logging.CRITICAL)
+ logging.getLogger("scrapy.middleware").setLevel(logging.CRITICAL)
+ logging.getLogger("scrapy.core.engine").setLevel(logging.CRITICAL)
+ logging.getLogger("scrapy.middleware").setLevel(logging.CRITICAL)
+
+ logger.disabled = True"""
+
+ def parse(self, response):
+ print('visited:', repr(response.url), 'status:', response.status)
+
+ if re.search(r'\.pdf$', response.url):
+ m = re.search(r'\/([^\/]+\.'+ext+')$', response.url)
+ if m:
+ print("saving to ", save_folder + '/' + clean_fn(response.url))
+ pdf_response = requests.get(response.url)
+ with open(save_folder + '/' + clean_fn(response.url), 'wb') as f:
+ f.write(pdf_response.content)
+ text = extract_text(save_folder + '/' + clean_fn(response.url))
+ codecs.open(save_folder + '/' + clean_fn(response.url) + '.txt','w','utf-8').write(text)
+
+ for ext in ['doc','docx','ppt','pptx']:
+ if re.search(r'\.'+ext+'$', response.url):
+ m = re.search(r'\/([^\/]+\.'+ext+')$', response.url)
+ if m:
+ print("saving to ", save_folder + '/' + clean_fn(response.url))
+ pdf_response = requests.get(response.url)
+ with open(save_folder + '/' + clean_fn(response.url), 'wb') as f:
+ f.write(pdf_response.content)
+ #text = extract_text(save_folder + '/' + clean_fn(response.url) + '.txt')
+ output = pypandoc.convert_file(save_folder + '/' + clean_fn(response.url), 'html', extra_args=['--extract-media=%s' % hash ])
+ txt_output = trafilatura.extract(response.text,include_links=True, deduplicate=True, include_images=True, include_formatting=True)
+ if txt_output:
+ codecs.open(save_folder + '/' + clean_fn(response.url) + '.txt','w','utf-8').write(txt_output)
+
+ for ext in ['jpg','jpeg','gif','webp']:
+ if re.search(r'\.'+ext+'$', response.url):
+ m = re.search(r'\/([^\/]+\.'+ext+')$', response.url)
+ if m:
+ print("saving to ", save_folder + '/' + clean_fn(response.url))
+ pdf_response = requests.get(response.url)
+ with open(save_folder + '/' + clean_fn(response.url), 'wb') as f:
+ f.write(pdf_response.content)
+
+ f_out = codecs.open(save_folder + '/' + clean_fn(response.url) + '.txt','w','utf-8')
+
+ this_output = trafilatura.extract(response.text,include_links=True, deduplicate=True, include_images=True, include_formatting=True)
+ if this_output:
+ f_out.write(this_output)
+ f_out.close()
+ links = response.css('a::attr(href)').getall()
+
+ # Follow each link and parse its contents
+
+ for link in links:
+ go = 1
+ full_link = response.urljoin(link)
+ print('++++++ trying ', full_link)
+
+ if not re.search(r'gavilan\.edu',full_link):
+ go = 0
+ print('--- not gav edu')
+ else:
+ if re.search(r'hhh\.gavilan\.edu',full_link):
+ pass
+ elif not re.search(r'^https?:\/\/www\.gavilan\.edu',full_link):
+ # need to add www to gavilan.edu
+ m = re.search(r'^(https?:\/\/)gavilan\.edu(\/.*)$',full_link)
+ if m:
+ full_link = m.group(1) + 'www.' + m.group(2)
+ for a in avoid:
+ if re.search(a,full_link):
+ go = 0
+ print('--- avoid ', a)
+
+ if go: yield scrapy.Request(full_link, callback=self.parse,
+ headers={"User-Agent": "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"})
+ else:
+ print("------ avoiding ", full_link)
+ # Instantiate a CrawlerProcess object
+ process = CrawlerProcess()
+
+ # Add the MySpider spider to the process
+ process.crawl(MySpider)
+
+ # Start the process
+ logging.basicConfig(level=logging.CRITICAL)
+ logging.getLogger('scrapy').propagate = False
+ logging.getLogger("trafilatura").setLevel(logging.CRITICAL)
+ logging.getLogger("trafilatura").propagate = False
+ logging.getLogger("pdfminer").setLevel(logging.CRITICAL)
+ logging.getLogger("pdfminer").propagate = False
+ logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+ logging.getLogger("urllib3").propagate = False
+ logging.basicConfig(level=logging.CRITICAL)
+ process.start()
-def samples():
- crawler = Crawler(links=[], domains=[], delay=20.0, sort=FIFO)
+save_folder = 'cache/crawl'
+clean_folder = 'cache/cleancrawl'
- url = URL('http://www.clips.ua.ac.bemedia/pattern_schema.gif')
- print url.mimetype in MIMETYPE_IMAGE
+def clean_fn(s):
+ s = re.sub(r'[\s:]+','',s)
+ s = re.sub(r'\/','+',s)
+ return s
+
+def format_html(html):
+ soup = bs4.BeautifulSoup(html, 'html.parser')
+ return soup.prettify()
+
- #html = download('http://www.clips.ua.ac.be/', unicode=True)
- s = URL('http://www.clips.ua.ac.be').download()
- s = plaintext(s, keep={'h1':[], 'h2':[], 'strong':[], 'a':['href']})
+
+def txt_clean_index():
+ files = os.listdir(save_folder)
+ line_freq = defaultdict(int)
+
+ # first pass
+ for f in files:
+ lines = codecs.open(save_folder + '/' + f,'r','utf-8').readlines()
+ for L in lines:
+ L = L.strip()
+ line_freq[L] += 1
+
+ # second pass
+ for f in files:
+ print("\n\n",f)
+ lines = codecs.open(save_folder + '/' + f,'r','utf-8').readlines()
+ out = codecs.open(clean_folder + '/' + f,'w','utf-8')
+ for L in lines:
+ L = L.strip()
+ if L in line_freq and line_freq[L] > 3:
+ continue
+ print(L)
+ out.write(L + '\n')
+ out.close()
- # getting absolute urls
- from pattern.web import URL, DOM, abs
- url = URL('http://www.clips.ua.ac.be')
- dom = DOM(url.download())
- for link in dom('a'):
- print abs(link.attributes.get('href',''), base=url.redirect or url.string)
+def search_embeddings():
+ model = SentenceTransformer('all-MiniLM-L6-v2')
+ save_embeds = pickle.load( open( "cache/embeddings.p", "rb" ) )
+ columns = list(zip(*save_embeds))
+ files = columns[0]
+ sentences = columns[1]
+ embeddings = columns[2]
- # get pdfs
- from pattern.web import URL, PDF
+ print(files[:20])
+ print(sentences[:20])
+ print(embeddings[:20])
- url = URL('http://www.clips.ua.ac.be/sites/default/files/ctrs-002_0.pdf')
- pdf = PDF(url.download())
- print pdf.string
+ s = ''
+ while s != 'q':
+ s = input("search or 'q' to quit: ")
+ if s == 'q':
+ return
+ query_embedding = model.encode(s)
+ # Compute the cosine similarity between the query embedding and the sentence embeddings
+ cosine_scores = util.cos_sim(query_embedding, embeddings)
+
+ # Sort the sentences by their cosine similarity to the query sentence
+ results = sorted(zip(sentences, cosine_scores, files), key=lambda x: x[1], reverse=True)
+
+ # Print the top 5 results
+ for i, (sentence, score, file) in enumerate(results[:5]):
+ print(f'Top {i+1}: {file} - {sentence} - (Score: {score})')
+
+
+def create_embeddings():
+ model = SentenceTransformer('all-MiniLM-L6-v2')
+ files = os.listdir('cache/crawl')
+ output = []
+ save_embeds = [] # ['file','sentence','embedding']
+ files.sort()
+ for f in files:
+ m = re.match(r'https?..www\.gavilan\.edu\+(.*)\.\w\w\w\w?\.txt$',f)
+ if m:
+ lines = codecs.open('cache/crawl/' + f,'r','utf-8').readlines()
+ lines = [L.strip() for L in lines]
+ lines = [L for L in lines if L]
+ embeddings = model.encode(lines)
+
+ print("\n-----", f)
+
+ #Print the embeddings
+ for sentence, embedding in zip(lines, embeddings):
+ print("Sentence:", sentence)
+ #print("Embedding:", embedding)
+
+ save_embeds.append([f,sentence,embedding])
+ pickle.dump( save_embeds, open( "cache/embeddings.p", "wb" ) )
if __name__ == "__main__":
@@ -897,16 +1098,25 @@ if __name__ == "__main__":
# 5: ['import freshdesk content', freshdesk ],
6: ['download all a courses pages', grab_course_pages],
7: ['demo vector search', demo_vector_search],
+ 8: ['crawl',crawl],
+ 9: ['clean text index', txt_clean_index],
+ 10: ['make web dir struct', manual_index],
+ 11: ['create search embeddings', create_embeddings],
+ 12: ['do a search', search_embeddings],
}
- for key in options:
- print(str(key) + '.\t' + options[key][0])
+ if len(sys.argv) > 1 and re.search(r'^\d+',sys.argv[1]):
+ resp = int(sys.argv[1])
+ print("\n\nPerforming: %s\n\n" % options[resp][0])
- print('')
- resp = input('Choose: ')
+ else:
+ print ('')
+ for key in options:
+ print(str(key) + '.\t' + options[key][0])
+
+ print('')
+ resp = input('Choose: ')
# Call the function in the options dict
options[ int(resp)][1]()
-
-
-
+
diff --git a/courses.py b/courses.py
index cf8be67..8583511 100644
--- a/courses.py
+++ b/courses.py
@@ -1086,6 +1086,7 @@ def add_evals(section=0):
s = [ x.strip() for x in codecs.open('cache/sp23_eval_sections.csv','r').readlines()]
s = list(funcy.flatten(s))
s.sort()
+ print(s)
xyz = input('hit return to continue')
#c = getCoursesInTerm(168,0,1)
@@ -1306,7 +1307,6 @@ def set_ext_tools():
if __name__ == "__main__":
options = { 1: ['Cross check schedule with ztc responses',make_ztc_list] ,
- 30: ['List latestart classes', list_latestarts ],
2: ['Add announcements to homepage', change_course_ann_homepage],
3: ['Cross-list classes', xlist ],
4: ['List students who passed quiz X', get_quiz_passers],
@@ -1335,6 +1335,7 @@ if __name__ == "__main__":
27: ['Fine tune term dates and winter session', course_dates_terms],
28: ['Cross list a semester from file', semester_cross_lister],
29: ['Check all courses & their sections in semester', all_semester_course_sanity_check],
+ #30: ['List latestart classes', list_latestarts ],
# TODO wanted: group shell for each GP (guided pathway) as a basic student services gateway....
#
}
diff --git a/depricated.py b/depricated.py
index d79baaf..0fb44e7 100644
--- a/depricated.py
+++ b/depricated.py
@@ -1807,3 +1807,95 @@ def freshdesk():
+
+
+#### content.py
+
+
+from pattern.web import plaintext, extension
+from pattern.web import download
+#from pattern import URL, MIMETYPE_IMAGE
+from pattern.web import Crawler, DEPTH, FIFO, MIMETYPE_IMAGE, MIMETYPE_PDF
+
+class GavCrawl(Crawler):
+ def visit(self, link, source=None):
+ print('visited:', repr(link.url), 'from:', link.referrer)
+ print(' ', link.url.mimetype)
+ #txt = plaintext(source, keep={'h1':[], 'h2':[], 'h3':[], 'h4':[], 'td':[], 'strong':[], 'b':[], 'a':['href'], 'img':['src'], 'ul':[], 'ol':[], 'li':[], 'dd':[], 'dt':[], 'i':[]})
+ #codecs.open(save_folder + '/' + mycleaner(clean_title(link.url)) + '.txt','w','utf-8').write(tomd.convert(txt))
+
+ codecs.open(save_folder + '/' + clean_fn(link.url) + '.txt','w','utf-8').write(trafilatura.extract(source,include_links=True, deduplicate=True, include_images=True, include_formatting=True))
+
+
+ def fail(self, link):
+ print('failed:', repr(link.url))
+ if re.search(r'\.pdf$', link.url):
+ m = re.search(r'\/([^\/]+\.pdf)$', link.url)
+ if m:
+ save_file = m.group(1)
+ print("saving to ", save_folder + '/' + save_file)
+ pdf_response = requests.get(link.url)
+ with open(save_folder + '/' + save_file, 'wb') as f:
+ f.write(pdf_response.content)
+ text = extract_text(save_folder + '/' + save_file)
+ #print(text)
+ codecs.open(save_folder + '/' + save_file + '.txt','w','utf-8').write(text)
+ else:
+ print("no match for pdf url: ", link.url)
+
+ for ext in ['jpg','jpeg','gif','webp']:
+ if re.search(r'\.'+ext+'$', link.url):
+ m = re.search(r'\/([^\/]+\.'+ext+')$', link.url)
+ if m:
+ save_file = m.group(1)
+ print("saving to ", save_folder + '/' + save_file)
+ pdf_response = requests.get(link.url)
+ with open(save_folder + '/' + save_file, 'wb') as f:
+ f.write(pdf_response.content)
+ else:
+ print('no match for '+ext+' url: ', link.url)
+
+def crawl2():
+ #p = GavCrawl(links=['http://www.gavilan.edu/'], domains=['gavilan.edu', 'gavilan.curriqunet.com','www.boarddocs.com'], delay=0.75)
+ #p = GavCrawl(links=['https://gavilan.edu/finaid/2022-23DirectLoanApplication1.pdf'], domains=['gavilan.edu', 'gavilan.curriqunet.com','www.boarddocs.com'], delay=0.75)
+ p = GavCrawl(links=['https://gavilan.curriqunet.com/catalog/iq/1826'], domains=['gavilan.edu', 'gavilan.curriqunet.com','www.boarddocs.com'], delay=0.75)
+
+
+
+
+ while not p.done:
+ try:
+ p.crawl(method=DEPTH, cached=False, throttle=0.76)
+ except Exception as e:
+ print("Exception: ", e)
+
+
+
+def samples():
+ crawler = Crawler(links=[], domains=[], delay=20.0, sort=FIFO)
+
+ url = URL('http://www.clips.ua.ac.bemedia/pattern_schema.gif')
+ print(url.mimetype in MIMETYPE_IMAGE)
+
+
+ #html = download('http://www.clips.ua.ac.be/', unicode=True)
+ s = URL('http://www.clips.ua.ac.be').download()
+ s = plaintext(s, keep={'h1':[], 'h2':[], 'strong':[], 'a':['href']})
+
+
+ # getting absolute urls
+ from pattern.web import URL, DOM, abs
+
+ url = URL('http://www.clips.ua.ac.be')
+ dom = DOM(url.download())
+ for link in dom('a'):
+ print(abs(link.attributes.get('href',''), base=url.redirect or url.string))
+
+ # get pdfs
+ from pattern.web import URL, PDF
+
+ url = URL('http://www.clips.ua.ac.be/sites/default/files/ctrs-002_0.pdf')
+ pdf = PDF(url.download())
+ print(pdf.string)
+
+
diff --git a/gpt.py b/gpt.py
index 42dfe0d..36d5394 100644
--- a/gpt.py
+++ b/gpt.py
@@ -4,8 +4,8 @@ import openai
from canvas_secrets import openai_org, openai_api_key
-openai.organization = "org-66WLoZQEtBrO42Z9S8rfd10M"
-openai.api_key = "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"
+openai.organization = openai_org
+openai.api_key = openai_api_key
#print(openai.Model.list())
my_prompt = "Write a series of texts trying to sell a pen to a stranger."
diff --git a/interactive.py b/interactive.py
index 33d3413..7fb76f4 100644
--- a/interactive.py
+++ b/interactive.py
@@ -1,4 +1,3 @@
-import curses
import heapq, re, csv, os, shutil, datetime, urllib
import itertools, time, markdown, csv, json, os.path, webbrowser, threading
from functools import wraps
@@ -15,6 +14,20 @@ import localcache
from server import *
from canvas_secrets import flask_secretkey
+from content import my_site
+
+import socket
+this_host = socket.gethostname()
+
+print('\n\n' + this_host, '\n\n')
+
+has_curses = 0
+if this_host != 'ROGDESKTOP':
+ import curses
+ has_curses = 1
+else:
+ print("Skipping curses stuff")
+
q = Queue()
@@ -25,7 +38,6 @@ PORT_NUMBER = 8080 # Maybe set this to 9000.
datafile = 'lambda.csv'
-#writing_path = 'c:/users/peter/Nextcloud/Documents/writing/'
####
@@ -95,7 +107,16 @@ def flask_thread(q):
+ @app.route('/mirror')
+ def mirror():
+ return codecs.open('cache/crawl/index.html','r','utf-8').read()
+
+ @app.route('/mirror/')
+ def mirror_file(filename):
+ return markdown.markdown( codecs.open('cache/crawl/'+filename,'r','utf-8').read() ) + \
+ "" + codecs.open('cache/crawl/'+filename,'r','utf-8').read() + ""
+
@app.route('/clearscreens')
def clears():
clearscreens()
@@ -166,6 +187,7 @@ def flask_thread(q):
@app.route('/x/writing/images/')
def writing_img(fname):
+ # TODO
img_path = "/media/hd2/peter_home/Documents/writing_img/"
print(img_path + fname + " - writing images folder")
img_ext = fname.split('.')[-1]
diff --git a/server.py b/server.py
index aaac0f8..502e352 100644
--- a/server.py
+++ b/server.py
@@ -1,5 +1,5 @@
import json, codecs, re, markdown, os, pypandoc, striprtf, sqlite3, random, urllib
-import subprocess, html
+import subprocess, html, time
from striprtf.striprtf import rtf_to_text
from flask import render_template, Response
from flask import send_from_directory
@@ -16,8 +16,33 @@ from localcache import arrange_data_for_web, depts_with_classcounts, dept_with_s
from yattag import Doc
+import socket
+this_host = socket.gethostname()
+print('\n\n server host: ' + this_host, '\n\n')
+
LECPATH = "/media/hd2/peter_home_offload/lecture/"
- host = 'http://192.168.1.6:5000'
+host = 'http://192.168.1.6:5000'
+news_path = '/media/hd2/peter_home/Documents/scripts/browser/'
+writing_path = '/media/hd2/peter_home/Documents/writing/'
+img_path = '/media/hd2/peter_home/Documents/writing_img/'
+pics_path = '/media/hd2/peter_home/misc/'
+
+
+if this_host == 'ROGDESKTOP':
+ LECPATH = "d:/peter_home_offload/lecture/"
+ host = 'http://192.168.1.7:5000'
+ news_path = 'd:/peter_home/Documents/scripts/browser/'
+ writing_path = 'd:/peter_home/Documents/writing/'
+ img_path = 'd:/peter_home/Documents/writing_img/'
+ pics_path = 'd:/peter_home/misc/'
+
+
+
+
+
+
+
+
import paho.mqtt.client as mqtt
@@ -55,20 +80,20 @@ def on_message(client, userdata, msg):
print(" %s mqtt msg: %s data: %s" % (now, msg.topic, msg.payload.decode()))
+if 0:
+ while(mqtt_offline):
+ try:
+ client = mqtt.Client()
+ client.on_connect = on_connect
+ client.on_message = on_message
-while(mqtt_offline):
- try:
- client = mqtt.Client()
- client.on_connect = on_connect
- client.on_message = on_message
+ client.connect("192.168.1.6", 1883, 60)
- client.connect("192.168.1.6", 1883, 60)
+ mqtt_offline = 0
- mqtt_offline = 0
-
- except OSError as oe:
- print('no internet? try again in 5 seconds.')
- time.sleep(5)
+ except OSError as oe:
+ print('no internet? try again in 5 seconds.')
+ time.sleep(5)
@@ -114,18 +139,6 @@ def screenoff():
######
-news_path = '/media/hd2/peter_home/Documents/scripts/browser/'
-
-if platform.system() == 'Windows':
- writing_path = 'c:/users/peter/Nextcloud/Documents/writing/'
-else:
- writing_path = '/media/hd2/peter_home/Documents/writing/'
- img_path = '/media/hd2/peter_home/Documents/writing_img/'
-
-if platform.system() == 'Windows':
- pics_path = 'c:/users/peter/Nextcloud/misc/'
-else:
- pics_path = '/media/hd2/peter_home/misc/'
br = "
"
nl = "\n"