diff --git a/courses.py b/courses.py index f5fc03b..b9dcd8a 100644 --- a/courses.py +++ b/courses.py @@ -1592,13 +1592,13 @@ def do_gav_connect(): print(crns) print("Press enter to begin.") a = input() - + print("Fetching all course names...") c = getCoursesInTerm(term, get_fresh, 0) i = 0 for course in c: if course['sis_course_id'] in crns: - print("Adding gav connect to", course['name']) + print(" Adding gav connect to", course['name']) print() result = add_gav_connect(course['id']) if result: diff --git a/interactive.py b/interactive.py index 145647c..bf27406 100644 --- a/interactive.py +++ b/interactive.py @@ -6,9 +6,6 @@ from flask import send_file from flask_socketio import SocketIO, emit from werkzeug.routing import PathConverter from queue import Queue -from textual.app import App, ComposeResult -from textual.containers import ScrollableContainer -from textual.widgets import Button, Footer, Header, Static from importlib import reload import server @@ -359,22 +356,106 @@ def serve(): +from textual.app import App, ComposeResult +from textual.containers import ScrollableContainer, Container, Horizontal +from textual.widgets import Button, Footer, Header, Static, Label +from textual.widgets import Welcome +from textual import events +from textual.app import App, ComposeResult +from textual.widgets import RichLog class CanvasApp(App): """A Textual app to manage canvas.""" - BINDINGS = [("d", "toggle_dark", "Toggle dark mode")] + BINDINGS = [("d", "toggle_dark", "Toggle dark mode"), ("q", "quit_app", "Quit")] + CSS_PATH = "interactive.tcss" + + current_label = "" + + # RHS side panel: Status & Saved Preferences + current_term = "" + last_db_pull = "" + orientation_shell_id = "" + stem_shell_id = "" + + # Most common actions + + ## Courses + # enroll orientation / stem + # add/remove course evals + # add/remove gav_connect + # bulk add calendar events + + ## Outcomes + # term status report + # pull fresh course / program list from cq + # build gavilan.cc catalog + # + + ## Users + # new teachers this semester + # download and format a user's logs + # show all convos for a user + # fetch/update employee list + + ## Stats pipelines - highlight outliers + # user hits + # user messaging + # user writing sample - is discussion, message, or assignment available to create a writing sample? How is it? + # course activity + # finished course: does it appear to be used for grades? + # in progress course: has this teacher/class combo had valid-looking gredes in the past? + + + ## Semester setup + # cross list / merge from file + # merge ad-hoc + # Move courses to winter term + # x-ref schedule and check published courses + # create a docx semester weekly calendar + + + ## Logistics + # cross reference training status of users + # enroll into gott courses + # create sandbox shells + # build gott certificates + + ## Content + # build ppt from md + # download course content to a folder / word file + + + ## Background actions on/off, exceptions + # fetch rosters hourly + # update database every 5 hours + def compose(self) -> ComposeResult: """Create child widgets for the app.""" yield Header() + #yield Welcome() + yield Label(self.current_label, classes="box", id="feedback") + yield RichLog() yield Footer() def action_toggle_dark(self) -> None: """An action to toggle dark mode.""" self.dark = not self.dark + def action_quit_app(self) -> None: + self.exit() + + def on_button_pressed(self) -> None: + self.exit() + + def on_key(self, event: events.Key) -> None: + self.query_one(RichLog).write(event) + self.current_label += event.character + fb = self.query_one("#feedback") + fb.update(self.current_label) + def text_app(): app = CanvasApp() diff --git a/interactive.tcss b/interactive.tcss new file mode 100644 index 0000000..b994d25 --- /dev/null +++ b/interactive.tcss @@ -0,0 +1,9 @@ +Screen { + layout: vertical; +} + +.box { + height: 1fr; + border: solid green; + width: 100%; +} \ No newline at end of file diff --git a/localcache.py b/localcache.py index f5142f6..02a826b 100644 --- a/localcache.py +++ b/localcache.py @@ -21,13 +21,14 @@ sqlite_file = local_data_folder + 'data.db' #'data_su20_4hr_blocks.db' mylog = codecs.open(local_data_folder + 'canvas_data_log.txt','w') thefiles_dat = {} -try: +'''try: for L in open('cache/canvas_data_index.txt','r').readlines(): L = L.strip() (fname,start,finish) = L.split(',') thefiles_dat[fname] = start except Exception as e: print("cache/canvas_data_index.txt was not found") +''' thefiles = open('cache/canvas_data_index_temp.txt','a') # rename me if nothing crashes :) @@ -1819,8 +1820,6 @@ def do_encoding(): def printer(x): print(x) - - def all_students_history(handler=printer, limit=1000): qry = """SELECT u.name AS user_name, diff --git a/localcache2.py b/localcache2.py index 12bb6ba..0d04d76 100644 --- a/localcache2.py +++ b/localcache2.py @@ -211,6 +211,22 @@ ORDER BY num DESC, u.sortable_name""" % (where1,where2) print(users_to_enroll) return users_to_enroll +def all_sem_courses_teachers(): + SEM = "202430" + q = f"""SELECT c.id, c.name, c.course_code, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id FROM canvas.courses AS c +JOIN canvas.enrollments AS e ON e.course_id=c.id +JOIN canvas.users AS u ON u.id=e.user_id +JOIN canvas.pseudonyms AS p ON p.user_id=u.id +WHERE c.sis_source_id LIKE '{SEM}-%' +AND NOT c.workflow_state='deleted' +AND e.type='TeacherEnrollment' +ORDER BY u.sortable_name;""" + (connection,cursor) = db() + cursor.execute(q) + courses = cursor.fetchall() + print(courses) + return courses + @@ -222,6 +238,7 @@ if __name__ == "__main__": 2: ['courses in term', courses_in_term], 3: ['pages in term', pages_in_term], 4: ['new students this semester', users_new_this_semester], + 5: ['all semester courses + teachers', all_sem_courses_teachers], } diff --git a/outcomes2022.py b/outcomes2022.py index 6eb430c..30c8c8c 100644 --- a/outcomes2022.py +++ b/outcomes2022.py @@ -404,8 +404,9 @@ def remove_all_bad_points(): def full_term_overview(verbose=1): - out2 = codecs.open(f'cache/slo_status_{TERM}.json','w','utf-8') - out3 = codecs.open(f'cache/slo_status_{TERM}.txt','w','utf-8') + out2 = codecs.open(f'cache/slo/slo_status_{TERM}.json','w','utf-8') + out3 = codecs.open(f'cache/slo/slo_status_{TERM}.csv','w','utf-8') + csv_fields = 'outcome_count,id,name,dept,code,crn,assessed_count,points_ok'.split(',') fn1 = f"cache/courses_in_term_{TERM}.json" all_courses = json.loads(codecs.open(fn1,'r','utf-8').read()) all_courses_status = {} @@ -480,13 +481,17 @@ def full_term_overview(verbose=1): # Print out the groups out2.write(json.dumps(course_groups,indent=2)) if verbose: + cwriter = csv.DictWriter(out3,fieldnames=csv_fields) + cwriter.writeheader() for group, dicts in course_groups.items(): sorted_dicts = sorted(dicts, key=lambda x: f"{x['dept']}{x['code']}") print(f"{group} - {len(sorted_dicts)} item(s)") - out3.write(f"{group} - {len(sorted_dicts)} item(s)\n") + cwriter.writerows(sorted_dicts) + # out3.write(f"{group} - {len(sorted_dicts)} item(s)\n") for d in sorted_dicts: print(d) - out3.write(str(d) + "\n") + #out3.write(str(d) + "\n") + print("\n") out3.write("\n") diff --git a/pipelines.py b/pipelines.py index fcda51b..6c28e29 100644 --- a/pipelines.py +++ b/pipelines.py @@ -207,11 +207,18 @@ dean['PHYS'] = 'jn' dean['POLS'] = 'nl' dean['PSCI'] = 'jn' dean['PSYC'] = 'nl' +dean['PSYCH'] = 'nl' dean['SJS'] = 'nl' dean['SOC'] = 'nl' dean['SPAN'] = 'nl' dean['THEA'] = 'nl' +dean_names = {} +dean_names['et'] = 'Enna Trevathan' +dean_names['ss'] = 'Susan Sweeney' +dean_names['nl'] = 'Noah Lystrup' +dean_names['jn'] = 'Jennifer Nari' + class FetchError(Exception): pass @@ -452,7 +459,7 @@ async def canvas_data_2024(): client_secret: str = os.environ["DAP_CLIENT_SECRET"] connection_string: str = "postgresql://postgres:rolley34@192.168.1.6/db" - desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',') + desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,pseudonyms,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',') credentials = Credentials.create(client_id=client_id, client_secret=client_secret) async with DatabaseConnection(connection_string).open() as db_connection: @@ -482,7 +489,7 @@ async def setup_canvas_data_2024(): client_secret: str = os.environ["DAP_CLIENT_SECRET"] connection_string: str = "postgresql://postgres:rolley34@192.168.1.6/db" - desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',') + desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,pseudonyms,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',') credentials = Credentials.create(client_id=client_id, client_secret=client_secret) async with DatabaseConnection(connection_string).open() as db_connection: diff --git a/stats.py b/stats.py index 27345bf..b7bbd4e 100644 --- a/stats.py +++ b/stats.py @@ -1381,6 +1381,52 @@ def report_student_stats(): # Save the figure in an HTML file pio.write_html(fig, 'cache/student_pct_onlinecourse.html') +def test_rpy(): + from rpy2 import robjects + from rpy2.robjects import Formula, Environment + from rpy2.robjects.vectors import IntVector, FloatVector + from rpy2.robjects.lib import grid + from rpy2.robjects.packages import importr, data + from rpy2.rinterface import RRuntimeWarning + import warnings + + # The R 'print' function + rprint = robjects.globalenv.get("print") + stats = importr('stats') + grdevices = importr('grDevices') + base = importr('base') + datasets = importr('datasets') + + grid.activate() + import math, datetime + import rpy2.robjects.lib.ggplot2 as ggplot2 + import rpy2.robjects as ro + from rpy2.robjects.packages import importr + base = importr('base') + + mtcars = data(datasets).fetch('mtcars')['mtcars'] + + pp = ggplot2.ggplot(mtcars) + \ + ggplot2.aes_string(x='wt', y='mpg', col='factor(cyl)') + \ + ggplot2.geom_point() + \ + ggplot2.geom_smooth(ggplot2.aes_string(group = 'cyl'), + method = 'lm') + pp.plot() + + + +def test_rpy2(): + import rpy2 + print(rpy2.__version__) + import rpy2.robjects as robjects + from rpy2.robjects.packages import importr + # import R's "base" package + base = importr('base') + + # import R's "utils" package + utils = importr('utils') + pi = robjects.r['pi'] + print(f"pi={pi[0]}") if __name__ == "__main__": @@ -1403,6 +1449,7 @@ if __name__ == "__main__": 17: ['rearrange section data to yearly form', sections_grouped_by_year_mode], 30: ['visualize course modes multi semester', visualize_course_modes_multi_semester], 31: ['Report on student stats', report_student_stats], + 32: ['test rpy', test_rpy], } print ('') diff --git a/tasks.py b/tasks.py index ed52f46..bd910b9 100644 --- a/tasks.py +++ b/tasks.py @@ -1334,8 +1334,8 @@ def word_calendar(): from docx.shared import Inches import datetime - # Define the start date - start_date = datetime.date(2023, 8, 28) + # Define the start date of semester + start_date = datetime.date(2024, 1, 29) # Prepare a list of 18 weeks beginning from the start date dates = [start_date + datetime.timedelta(weeks=x) for x in range(18)] diff --git a/users.py b/users.py index bac0d83..018c84a 100644 --- a/users.py +++ b/users.py @@ -2234,7 +2234,7 @@ def compare_db_tables(): def training_find_goos(): - from localcache import all_sem_courses_teachers + from localcache2 import all_sem_courses_teachers from localcache import course_mode from localcache import sem_schedule from pipelines import dean @@ -2268,16 +2268,19 @@ def training_find_goos(): print() def cross_ref_training(): - from localcache import all_sem_courses_teachers + from localcache2 import all_sem_courses_teachers from localcache import course_mode from localcache import sem_schedule - from pipelines import dean + from pipelines import dean, dean_names from openpyxl import Workbook, load_workbook from openpyxl.chart import BarChart, Series, Reference from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx") print(wb.sheetnames) + # report for email + report = codecs.open('cache/gott_report.txt','w','utf-8') + all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read()) records = {} @@ -2302,7 +2305,8 @@ def cross_ref_training(): courses = all_sem_courses_teachers() for c in courses: - goo = c[7] + print(c) + goo = c[6] crn = c[2].split(' ')[-1].split('/')[0] name = c[2] teacher = c[4] @@ -2328,10 +2332,12 @@ def cross_ref_training(): flagfont = PatternFill("solid", fgColor="00FFFFCC") for thedean in ['et','nl','ss','jn']: - sheet.cell(row=r, column=1).value = thedean + sheet.cell(row=r, column=1).value = dean_names[thedean] sheet.cell(row=r, column=1).font = deptfont r += 2 + report.write(f"Dean: {dean_names[thedean]}\n") + for D in alldepts: if not D in dean: print(f"MISSING DEAN for dept: {D}") @@ -2347,32 +2353,38 @@ def cross_ref_training(): completed = 0 waived = 0 sects = teachers[t] - goo = sects[0][7] + print(f"Sections for {t}: {sects}") + goo = sects[0][6] print(t) sheet.cell(row=r, column=1).value = f"{t}" sheet.cell(row=r, column=2).value = f"{goo}" r += 1 if goo in records['GOTT1']: sheet.cell(row=r, column=2).value = f"✓ GOTT 1 Trained" + print(f" GOTT 1") completed =1 r += 1 if goo in records['Other Certifications']: sheet.cell(row=r, column=2).value = f"✓ GOTT Waived - Outside Training" + print(f" GOTT outside training") completed = 1 waived = 1 r += 1 if goo in records['GOTT2']: sheet.cell(row=r, column=2).value = f"✓ GOTT 2 Trained" + print(f" GOTT 2") completed = 1 waived = 1 r += 1 if goo in records['POCR Reviewed']: sheet.cell(row=r, column=2).value = f"✓ POCR Reviewed" + print(f" POCR") completed = 1 waived = 1 r += 1 if goo in records['TITLE V GOTT ACADEMY']: sheet.cell(row=r, column=2).value = f"✓ TITLE V GOTT ACADEMY 2014" + print(f" GOTT Academy") completed = 1 waived = 1 r += 1 @@ -2382,32 +2394,45 @@ def cross_ref_training(): else: sheet.cell(row=r, column=2).value = f"- MISSING GOTT 1" sheet.cell(row=r-1,column=1).fill = flagfont + report.write(f"\tMISSING GOTT 1: {t}\n") r += 1 if goo in records['GOTT4']: sheet.cell(row=r, column=2).value = f"✓ GOTT 4 Trained" + print(f" GOTT 4") r += 1 if goo in records['GOTT5']: sheet.cell(row=r, column=2).value = f"✓ GOTT 5 Trained" + print(f" GOTT 5") r += 1 if goo in records['GOTT6']: sheet.cell(row=r, column=2).value = f"✓ GOTT 6 Trained" + print(f" GOTT ") r += 1 if goo in records['SU21 Workshop']: sheet.cell(row=r, column=2).value = f"✓ SU21 Workshop" + print(f" summer 21 workshop") r += 1 if goo in records['HUM.STEM']: sheet.cell(row=r, column=2).value = f"✓ Humanizing Stem" + print(f" humanizing stem") r += 1 if goo in records['BOOT CAMP']: sheet.cell(row=r, column=2).value = f"✓ Boot Camp Self Paced" + print(f" bc self paced") r += 1 if goo in records['GOTT ABC']: sheet.cell(row=r, column=2).value = f"✓ {records['GOTT ABC'][goo][2]} Self Paced" + print(f" GOTT abc self paced") r += 1 for s in sects: - sheet.cell(row=r, column=2).value = f"{s[8]}" - sheet.cell(row=r, column=3).value = f"{s[2]}" + sheet.cell(row=r, column=2).value = f"{s[7]}" + sheet.cell(row=r, column=3).value = f"{s[1]}" r += 1 + + if (not completed) and (not waived): + report.write(f"\t\t{s[7]}\t{s[1]}\n") + if (not completed) and (not waived): + report.write(f"\n") #for c in sheet.columns: # print(c) @@ -2415,10 +2440,11 @@ def cross_ref_training(): sheet.column_dimensions['A'].width = 20 sheet.column_dimensions['B'].width = 30 sheet.column_dimensions['C'].width = 75 - wb.save("C:/Users/peter/Downloads/GOTT_Completion_masterlist 102023_summarized.xlsx") + formatted_date = datetime.datetime.now().strftime('%Y%m%d') + wb.save(f"C:/Users/peter/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx") def cross_ref_training_withcsv(): - from localcache import all_sem_courses_teachers + from localcache2 import all_sem_courses_teachers from localcache import course_mode from localcache import sem_schedule