misc

courses.py

@@ -1592,7 +1592,7 @@ def do_gav_connect():
     print(crns)
     print("Press enter to begin.")
     a = input()
-
+    print("Fetching all course names...")
     c = getCoursesInTerm(term, get_fresh, 0)
     i = 0
 

interactive.py

@@ -6,9 +6,6 @@ from flask import send_file
 from flask_socketio import SocketIO, emit
 from werkzeug.routing import PathConverter
 from queue import Queue
-from textual.app import App, ComposeResult
-from textual.containers import ScrollableContainer
-from textual.widgets import Button, Footer, Header, Static
 from importlib import reload 
 
 import server
@@ -359,22 +356,106 @@ def serve():
 
 
 
+from textual.app import App, ComposeResult
+from textual.containers import ScrollableContainer, Container, Horizontal
+from textual.widgets import Button, Footer, Header, Static, Label
+from textual.widgets import Welcome
+from textual import events
+from textual.app import App, ComposeResult
+from textual.widgets import RichLog
 
 
 class CanvasApp(App):
     """A Textual app to manage canvas."""
 
-    BINDINGS = [("d", "toggle_dark", "Toggle dark mode")]
+    BINDINGS = [("d", "toggle_dark", "Toggle dark mode"), ("q", "quit_app", "Quit")]
+    CSS_PATH = "interactive.tcss"
+
+    current_label = ""
+
+    # RHS side panel: Status & Saved Preferences
+    current_term = ""
+    last_db_pull = ""
+    orientation_shell_id = ""
+    stem_shell_id = ""
+
+    # Most common actions
+
+    ## Courses
+    # enroll orientation / stem
+    # add/remove course evals
+    # add/remove gav_connect
+    # bulk add calendar events
+
+    ## Outcomes
+    # term status report
+    # pull fresh course / program list from cq
+    # build gavilan.cc catalog
+    #
+
+    ## Users
+    # new teachers this semester
+    # download and format a user's logs
+    # show all convos for a user
+    # fetch/update employee list
+
+    ## Stats pipelines - highlight outliers
+    # user hits
+    # user messaging
+    # user writing sample - is discussion, message, or assignment available to create a writing sample? How is it?
+    # course activity
+    # finished course: does it appear to be used for grades?
+    # in progress course: has this teacher/class combo had valid-looking gredes in the past?
+    
+
+    ## Semester setup
+    # cross list / merge from file
+    # merge ad-hoc
+    # Move courses to winter term
+    # x-ref schedule and check published courses
+    # create a docx semester weekly calendar
+
+
+    ## Logistics
+    # cross reference training status of users
+    # enroll into gott courses
+    # create sandbox shells
+    # build gott certificates
+
+    ## Content
+    # build ppt from md
+    # download course content to a folder / word file
+
+
+    ## Background actions on/off, exceptions
+    # fetch rosters hourly
+    # update database every 5 hours
+
 
     def compose(self) -> ComposeResult:
         """Create child widgets for the app."""
         yield Header()
+        #yield Welcome()
+        yield Label(self.current_label, classes="box", id="feedback")
+        yield RichLog()
         yield Footer()
 
     def action_toggle_dark(self) -> None:
         """An action to toggle dark mode."""
         self.dark = not self.dark
 
+    def action_quit_app(self) -> None:
+        self.exit()
+
+    def on_button_pressed(self) -> None:
+        self.exit()
+
+    def on_key(self, event: events.Key) -> None:
+        self.query_one(RichLog).write(event)
+        self.current_label += event.character
+        fb = self.query_one("#feedback")
+        fb.update(self.current_label)
+
 
 def text_app():
     app = CanvasApp()

interactive.tcss

@@ -0,0 +1,9 @@
+Screen {
+    layout: vertical;
+}
+
+.box {
+    height: 1fr;
+    border: solid green;
+    width: 100%;
+}

localcache.py

@@ -21,13 +21,14 @@ sqlite_file = local_data_folder + 'data.db'    #'data_su20_4hr_blocks.db'
 mylog = codecs.open(local_data_folder + 'canvas_data_log.txt','w')
 
 thefiles_dat = {}
-try:
+'''try:
     for L in open('cache/canvas_data_index.txt','r').readlines():
         L = L.strip()
         (fname,start,finish) = L.split(',')
         thefiles_dat[fname] = start
 except Exception as e:
     print("cache/canvas_data_index.txt was not found")
+'''
 
 thefiles = open('cache/canvas_data_index_temp.txt','a')    # rename me if nothing crashes :)
 
@@ -1819,8 +1820,6 @@ def do_encoding():
 def printer(x):
     print(x)
 
-
-
 def all_students_history(handler=printer, limit=1000):
     qry = """SELECT
   u.name AS user_name,

localcache2.py

@@ -211,6 +211,22 @@ ORDER BY num DESC, u.sortable_name""" % (where1,where2)
     print(users_to_enroll)
     return users_to_enroll
 
+def all_sem_courses_teachers():
+    SEM = "202430"
+    q = f"""SELECT c.id, c.name, c.course_code, u.name, u.sortable_name, u.id AS user_cid, p.sis_user_id  FROM canvas.courses AS c
+JOIN canvas.enrollments AS e ON e.course_id=c.id
+JOIN canvas.users AS u ON u.id=e.user_id
+JOIN canvas.pseudonyms AS p ON p.user_id=u.id
+WHERE c.sis_source_id LIKE '{SEM}-%'
+AND NOT c.workflow_state='deleted'
+AND e.type='TeacherEnrollment'
+ORDER BY u.sortable_name;"""
+    (connection,cursor) = db()
+    cursor.execute(q)
+    courses = cursor.fetchall()
+    print(courses)
+    return courses
+
 
 
 
@@ -222,6 +238,7 @@ if __name__ == "__main__":
             2: ['courses in term', courses_in_term],
             3: ['pages in term', pages_in_term],
             4: ['new students this semester', users_new_this_semester],
+            5: ['all semester courses + teachers', all_sem_courses_teachers],
     }
 
 

outcomes2022.py

@@ -404,8 +404,9 @@ def remove_all_bad_points():
 
 
 def full_term_overview(verbose=1):
-    out2 = codecs.open(f'cache/slo_status_{TERM}.json','w','utf-8')
+    out2 = codecs.open(f'cache/slo/slo_status_{TERM}.json','w','utf-8')
-    out3 = codecs.open(f'cache/slo_status_{TERM}.txt','w','utf-8')
+    out3 = codecs.open(f'cache/slo/slo_status_{TERM}.csv','w','utf-8')
+    csv_fields = 'outcome_count,id,name,dept,code,crn,assessed_count,points_ok'.split(',')
     fn1 = f"cache/courses_in_term_{TERM}.json"
     all_courses = json.loads(codecs.open(fn1,'r','utf-8').read())
     all_courses_status = {}
@@ -480,13 +481,17 @@ def full_term_overview(verbose=1):
     # Print out the groups
     out2.write(json.dumps(course_groups,indent=2))
     if verbose:
+        cwriter = csv.DictWriter(out3,fieldnames=csv_fields)
+        cwriter.writeheader()
         for group, dicts in course_groups.items():
             sorted_dicts = sorted(dicts, key=lambda x: f"{x['dept']}{x['code']}")
             print(f"{group} - {len(sorted_dicts)} item(s)")
-            out3.write(f"{group} - {len(sorted_dicts)} item(s)\n")
+            cwriter.writerows(sorted_dicts)
+            # out3.write(f"{group} - {len(sorted_dicts)} item(s)\n")
             for d in sorted_dicts:
                 print(d)
-                out3.write(str(d) + "\n")
+                #out3.write(str(d) + "\n")
+                
             print("\n")
             out3.write("\n")
 

pipelines.py

@@ -207,11 +207,18 @@ dean['PHYS'] = 'jn'
 dean['POLS'] = 'nl'
 dean['PSCI'] = 'jn'
 dean['PSYC'] = 'nl'
+dean['PSYCH'] = 'nl'
 dean['SJS'] = 'nl'
 dean['SOC'] = 'nl'
 dean['SPAN'] = 'nl'
 dean['THEA'] = 'nl'
 
+dean_names = {}
+dean_names['et'] = 'Enna Trevathan'
+dean_names['ss'] = 'Susan Sweeney'
+dean_names['nl'] = 'Noah Lystrup'
+dean_names['jn'] = 'Jennifer Nari'
+
 
 class FetchError(Exception):
     pass
@@ -452,7 +459,7 @@ async def canvas_data_2024():
     client_secret: str = os.environ["DAP_CLIENT_SECRET"]
     connection_string: str = "postgresql://postgres:rolley34@192.168.1.6/db"
     
-    desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',')
+    desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,pseudonyms,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',')
     credentials = Credentials.create(client_id=client_id, client_secret=client_secret)
 
     async with DatabaseConnection(connection_string).open() as db_connection:
@@ -482,7 +489,7 @@ async def setup_canvas_data_2024():
     client_secret: str = os.environ["DAP_CLIENT_SECRET"]
     connection_string: str = "postgresql://postgres:rolley34@192.168.1.6/db"
     
-    desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',')
+    desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,pseudonyms,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',')
     credentials = Credentials.create(client_id=client_id, client_secret=client_secret)
 
     async with DatabaseConnection(connection_string).open() as db_connection:

stats.py

@@ -1381,6 +1381,52 @@ def report_student_stats():
     # Save the figure in an HTML file
     pio.write_html(fig, 'cache/student_pct_onlinecourse.html')
 
+def test_rpy():
+    from rpy2 import robjects
+    from rpy2.robjects import Formula, Environment
+    from rpy2.robjects.vectors import IntVector, FloatVector
+    from rpy2.robjects.lib import grid
+    from rpy2.robjects.packages import importr, data
+    from rpy2.rinterface import RRuntimeWarning
+    import warnings
+
+    # The R 'print' function
+    rprint = robjects.globalenv.get("print")
+    stats = importr('stats')
+    grdevices = importr('grDevices')
+    base = importr('base')
+    datasets = importr('datasets')
+
+    grid.activate()
+    import math, datetime
+    import rpy2.robjects.lib.ggplot2 as ggplot2
+    import rpy2.robjects as ro
+    from rpy2.robjects.packages import importr
+    base = importr('base')
+
+    mtcars = data(datasets).fetch('mtcars')['mtcars']
+
+    pp = ggplot2.ggplot(mtcars) + \
+         ggplot2.aes_string(x='wt', y='mpg', col='factor(cyl)') + \
+         ggplot2.geom_point() + \
+         ggplot2.geom_smooth(ggplot2.aes_string(group = 'cyl'),
+                             method = 'lm')
+    pp.plot()
+
+
+
+def test_rpy2():
+    import rpy2
+    print(rpy2.__version__)
+    import rpy2.robjects as robjects
+    from rpy2.robjects.packages import importr
+    # import R's "base" package
+    base = importr('base')
+
+    # import R's "utils" package
+    utils = importr('utils')
+    pi = robjects.r['pi']
+    print(f"pi={pi[0]}")
 
 
 if __name__ == "__main__":
@@ -1403,6 +1449,7 @@ if __name__ == "__main__":
                 17: ['rearrange section data to yearly form', sections_grouped_by_year_mode],
                 30: ['visualize course modes multi semester', visualize_course_modes_multi_semester],
                 31: ['Report on student stats', report_student_stats],
+                32: ['test rpy', test_rpy],
               }
     print ('')
 

tasks.py

@@ -1334,8 +1334,8 @@ def word_calendar():
     from docx.shared import Inches
     import datetime
 
-    # Define the start date
+    # Define the start date of semester
-    start_date = datetime.date(2023, 8, 28)
+    start_date = datetime.date(2024, 1, 29)
 
     # Prepare a list of 18 weeks beginning from the start date
     dates = [start_date + datetime.timedelta(weeks=x) for x in range(18)]

users.py

@@ -2234,7 +2234,7 @@ def compare_db_tables():
 
 
 def training_find_goos():
-    from localcache import all_sem_courses_teachers
+    from localcache2 import all_sem_courses_teachers
     from localcache import course_mode
     from localcache import sem_schedule
     from pipelines import dean
@@ -2268,16 +2268,19 @@ def training_find_goos():
         print()
 
 def cross_ref_training():
-    from localcache import all_sem_courses_teachers
+    from localcache2 import all_sem_courses_teachers
     from localcache import course_mode
     from localcache import sem_schedule
-    from pipelines import dean
+    from pipelines import dean, dean_names
     from openpyxl import Workbook, load_workbook
     from openpyxl.chart import BarChart, Series, Reference
     from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, Fill
     wb = load_workbook("C:/Users/peter/Downloads/GOTT_Completion_masterlist 2023 DEC.xlsx")
     print(wb.sheetnames)
 
+    # report for email
+    report = codecs.open('cache/gott_report.txt','w','utf-8')
+
     all_teachers = json.loads(codecs.open('cache/ilearn_staff.json','r','utf-8').read())
 
     records = {}
@@ -2302,7 +2305,8 @@ def cross_ref_training():
 
     courses = all_sem_courses_teachers()
     for c in courses:
-        goo = c[7]
+        print(c)
+        goo = c[6]
         crn = c[2].split(' ')[-1].split('/')[0]
         name = c[2]
         teacher = c[4]
@@ -2328,10 +2332,12 @@ def cross_ref_training():
     flagfont = PatternFill("solid", fgColor="00FFFFCC")
 
     for thedean in ['et','nl','ss','jn']:
-        sheet.cell(row=r, column=1).value = thedean
+        sheet.cell(row=r, column=1).value = dean_names[thedean]
         sheet.cell(row=r, column=1).font = deptfont
         r += 2
 
+        report.write(f"Dean: {dean_names[thedean]}\n")
+
         for D in alldepts:
             if not D in dean:
                 print(f"MISSING DEAN for dept: {D}")
@@ -2347,32 +2353,38 @@ def cross_ref_training():
                     completed = 0
                     waived = 0
                     sects = teachers[t]
-                    goo = sects[0][7]
+                    print(f"Sections for {t}: {sects}")
+                    goo = sects[0][6]
                     print(t)
                     sheet.cell(row=r, column=1).value = f"{t}"
                     sheet.cell(row=r, column=2).value = f"{goo}"
                     r += 1
                     if goo in records['GOTT1']:
                         sheet.cell(row=r, column=2).value = f"✓ GOTT 1 Trained"
+                        print(f"   GOTT 1")
                         completed =1
                         r += 1
                     if goo in records['Other Certifications']:
                         sheet.cell(row=r, column=2).value = f"✓ GOTT Waived - Outside Training"
+                        print(f"   GOTT outside training")
                         completed = 1
                         waived = 1
                         r += 1
                     if goo in records['GOTT2']:
                         sheet.cell(row=r, column=2).value = f"✓ GOTT 2 Trained"
+                        print(f"   GOTT 2")
                         completed = 1
                         waived = 1
                         r += 1
                     if goo in records['POCR Reviewed']:
                         sheet.cell(row=r, column=2).value = f"✓ POCR Reviewed"
+                        print(f"   POCR")
                         completed = 1
                         waived = 1
                         r += 1
                     if goo in records['TITLE V GOTT ACADEMY']:
                         sheet.cell(row=r, column=2).value = f"✓ TITLE V GOTT ACADEMY 2014"
+                        print(f"   GOTT Academy")
                         completed = 1
                         waived = 1
                         r += 1
@@ -2382,43 +2394,57 @@ def cross_ref_training():
                         else:
                             sheet.cell(row=r, column=2).value = f"- MISSING GOTT 1"
                             sheet.cell(row=r-1,column=1).fill = flagfont
+                            report.write(f"\tMISSING GOTT 1: {t}\n")
                         r += 1
                     if goo in records['GOTT4']:
                         sheet.cell(row=r, column=2).value = f"✓ GOTT 4 Trained"
+                        print(f"   GOTT 4")
                         r += 1
                     if goo in records['GOTT5']:
                         sheet.cell(row=r, column=2).value = f"✓ GOTT 5 Trained"
+                        print(f"   GOTT 5")
                         r += 1
                     if goo in records['GOTT6']:
                         sheet.cell(row=r, column=2).value = f"✓ GOTT 6 Trained"
+                        print(f"   GOTT ")
                         r += 1
                     if goo in records['SU21 Workshop']:
                         sheet.cell(row=r, column=2).value = f"✓ SU21 Workshop"
+                        print(f"   summer 21 workshop")
                         r += 1
                     if goo in records['HUM.STEM']:
                         sheet.cell(row=r, column=2).value = f"✓ Humanizing Stem"
+                        print(f"   humanizing stem")
                         r += 1
                     if goo in records['BOOT CAMP']:
                         sheet.cell(row=r, column=2).value = f"✓ Boot Camp Self Paced"
+                        print(f"   bc self paced")
                         r += 1
                     if goo in records['GOTT ABC']:
                         sheet.cell(row=r, column=2).value = f"✓ {records['GOTT ABC'][goo][2]} Self Paced"
+                        print(f"   GOTT abc self paced")
                         r += 1
                     for s in sects:
-                        sheet.cell(row=r, column=2).value = f"{s[8]}"
+                        sheet.cell(row=r, column=2).value = f"{s[7]}"
-                        sheet.cell(row=r, column=3).value = f"{s[2]}"
+                        sheet.cell(row=r, column=3).value = f"{s[1]}"
                         r += 1
 
+                        if (not completed) and (not waived):
+                            report.write(f"\t\t{s[7]}\t{s[1]}\n")
+                    if (not completed) and (not waived):
+                        report.write(f"\n")
+    
     #for c in sheet.columns:
     #    print(c)
     #    print(f"{c} width: {sheet.column_dimensions[c].value}")
     sheet.column_dimensions['A'].width = 20
     sheet.column_dimensions['B'].width = 30
     sheet.column_dimensions['C'].width = 75
-    wb.save("C:/Users/peter/Downloads/GOTT_Completion_masterlist 102023_summarized.xlsx")
+    formatted_date = datetime.datetime.now().strftime('%Y%m%d')
+    wb.save(f"C:/Users/peter/Downloads/GOTT_Completion_masterlist_{formatted_date}_summarized.xlsx")
 
 def cross_ref_training_withcsv():
-    from localcache import all_sem_courses_teachers
+    from localcache2 import all_sem_courses_teachers
     from localcache import course_mode
     from localcache import sem_schedule