Merge remote-tracking branch 'origin/master'

trying to merge

courses.py

@@ -1,21 +1,21 @@
-#from ast import Try, TryStar
 import json, re, requests, codecs, sys, time, funcy, os
 import pandas as pd
 from datetime import datetime   
 import pytz
-#from dateutil import parser
 from datetime import datetime
-#from symbol import try_stmt
 from util import print_table, int_or_zero, float_or_zero, dept_from_name, num_from_name
 from pipelines import fetch, fetch_stream, fetch_collapse, header, url
 from schedules import get_semester_schedule
-#from pipelines import sems
 from localcache import course_quick_stats, get_courses_in_term_local, course_student_stats, all_sem_courses_teachers, full_reload
 from localcache2 import db, users_new_this_semester, users_new_this_2x_semester, course_from_id, user_ids_in_shell
 from localcache2 import student_count, teacher_list, course_from_id, course_sched_entry_from_id
 from collections import defaultdict
 from semesters import find_term
 
+#from dateutil import parser
+#from ast import Try, TryStar
+#from symbol import try_stmt
+#from pipelines import sems
 
 
 stem_course_id = '11015'   # TODO 

curric2022.py

@@ -1,11 +1,10 @@
+import util
 import requests,json,os,re, bisect, csv, codecs, funcy, sys, shutil, time
 from datetime import datetime
 import sortedcontainers as sc
 from collections import defaultdict
 from toolz.itertoolz import groupby,sliding_window
 from sortedcontainers import SortedList
-#from durable.lang import *
-#from durable.engine import *
 from pampy import match, _
 from bs4 import BeautifulSoup as bs
 

curriculum.py

@@ -1,10 +1,8 @@
+import util
 import requests,json,os,re, bisect, csv, codecs
 import sortedcontainers as sc
 from collections import defaultdict
 from toolz.itertoolz import groupby
-#from docx.shared import Inches
-#from docx import Document
-#import docx
 from durable.lang import *
 from durable.engine import *
 from pampy import match, _
@@ -15,9 +13,6 @@ from schedules import get_semester_schedule
 
 from canvas_secrets import cq_url, cq_user, cq_pasw
 
-
-#sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
-
 TRACING = codecs.open('cache/progdebug.txt','w','utf-8')
 param = "?method=getCourses"
 

curriculum2020.py

@@ -1,3 +1,4 @@
+import util
 from pampy import match, _
 import json, pypandoc, requests,json,os,re, bisect, csv, codecs
 import sortedcontainers as sc

degrees.py

@@ -5,6 +5,7 @@ from minizinc import Instance, Model, Solver
 from datetime import datetime
 from collections import defaultdict
 import pandas as pd
+import util
 
 
 debug_out = codecs.open("cache/degrees_debug.txt", "w", "utf-8")

flexday.py

@@ -3,6 +3,7 @@ from localcache2 import user_from_goo
 from canvas_secrets import url
 from pipelines import fetch
 from users import getEmail
+import util
 
 
 def user_db_sync():

gpt.py

@@ -1,6 +1,6 @@
 import os, json, sys, codecs, re
 from datetime import datetime, timedelta
-
+import util
 from openai import OpenAI
 
 

localcache.py

@@ -9,13 +9,10 @@ from datetime import timedelta
 from dateutil.parser import parse
 from os.path import exists, getmtime
 from pipelines import sync_non_interactive, url, header
-#, gp, dean
+import util
 from semesters import short_to_sis
 
 
-#from courses import getCoursesInTerm
-#from courses import user_in_depts_live
-
 mycourses = {}
 
 local_data_folder = 'cache/canvas_data/'

localcache2.py

@@ -1,5 +1,5 @@
 # Local data, saving and manipulating
-
+import util
 import os, re, gzip, codecs, funcy, pytz, json, random, functools, requests, sys, csv, time, psycopg2
 import pandas as pd
 import numpy as np
@@ -8,7 +8,6 @@ from datetime import datetime as dt
 from datetime import timedelta
 from dateutil.parser import parse
 from os.path import exists, getmtime
-#from pipelines import sync_non_interactive, url, header, gp, dean
 from tabulate import tabulate
 
 from semesters import short_to_sis

outcomes.py

@@ -1,18 +1,14 @@
-
-
+import util
 import requests, json, codecs, csv, re, sys, os, shutil, time
 from collections import defaultdict
-
 from pipelines import fetch, url, header
 from courses import getCoursesInTerm, getTerms
-
 from concurrent.futures import ThreadPoolExecutor
 
 
 f = codecs.open('cache/slo/log.txt','w','utf-8')
 
 VERBOSE = 1
-
 TERM = '180'
 
 SLO_CURRENT_SOURCE = 'cache/slo/2018_slo.csv'          # term 21

outcomes2022.py

@@ -19,6 +19,7 @@
 # - Issue:
 #     + Course naming / sections joined...
 
+import util
 import concurrent.futures
 import pandas as pd
 from pipelines import fetch, url, header

pipelines.py

@@ -1,11 +1,7 @@
-#from sqlite3 import paramstyle
-#from time import strptime
-#from util import UnicodeDictReader
-import codecs, json, requests, re, csv, datetime, pysftp, os, jsondiff, os.path
+import util
+import codecs, json, requests, re, csv, datetime, os, jsondiff, os.path
 import sys, shutil, hmac, hashlib, base64, schedule, time, pathlib
-#import pdb
 from datetime import timedelta
-#from collections import defaultdict
 
 from canvas_secrets import apiKey, apiSecret, FTP_SITE, FTP_USER, FTP_PW, url, domain, account_id, header, header_media, g_id, g_secret
 from canvas_secrets import instructure_url, instructure_username, instructure_private_key
@@ -396,6 +392,7 @@ def file_doesnt_exist(name):
 
 # From instructure sftp site
 def fetch_current_rosters():
+    import pysftp
     cnopts = pysftp.CnOpts()
     cnopts.hostkeys = None
     with pysftp.Connection(instructure_url,username=instructure_username, private_key=instructure_private_key,cnopts=cnopts) as sftp:
@@ -578,6 +575,7 @@ def do_request(path):
 
 # Upload a json file to www
 def put_file(remotepath,localpath, localfile,prompt=1):
+    import pysftp
     show_all = 0
     folder = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
     cnopts = pysftp.CnOpts()

semesters.py

@@ -1,8 +1,7 @@
 # Try to gather all the different formats and ways of labeling a semester, along with their associated dates.
 
 import json, funcy, re, sys
-
-# sem_to_short = { 'Summer 2021': 'su21', 'Fall 2021':'fa21', 'Winter 2022':'wi22', 'Spring 2022':'sp22', 'Summer 2022':'su22', 'Fall 2022':'fa22' }
+import util
 
 season_to_number = { 'Fall': '70', 'Summer': '50', 'Spring': '30', 'Winter': '10'}
 

templates.py

@@ -1,6 +1,5 @@
-
 import os, re, codecs
-
+import util
 from pipelines import get_doc, get_doc_generic, put_file
 
 # build web pages from fragments

util.py

@@ -1,22 +1,64 @@
+import os
+import builtins as _builtins
+import codecs as _codecs
 
+# --- Safe file I/O monkey patches ---
+# Ensure parent folders exist for write/append modes when using open/codecs.open
+_orig_open = _builtins.open
+_orig_codecs_open = _codecs.open
 
+def _ensure_parent_dir(path):
+    try:
+        if isinstance(path, (str, bytes, os.PathLike)):
+            d = os.path.dirname(os.fspath(path))
+            if d and not os.path.exists(d):
+                os.makedirs(d, exist_ok=True)
+    except Exception:
+        # Never block the open call due to directory check errors
+        pass
 
+def _open_with_dirs(file, mode='r', *args, **kwargs):
+    try:
+        if isinstance(file, (str, bytes, os.PathLike)) and any(m in mode for m in ('w','a','x','+')):
+            _ensure_parent_dir(file)
+    finally:
+        return _orig_open(file, mode, *args, **kwargs)
+
+def _codecs_open_with_dirs(filename, mode='r', encoding=None, errors='strict', buffering=1):
+    try:
+        if isinstance(filename, (str, bytes, os.PathLike)) and any(m in mode for m in ('w','a','x','+')):
+            _ensure_parent_dir(filename)
+    finally:
+        return _orig_codecs_open(filename, mode, encoding, errors, buffering)
+
+# Apply patches once
+_builtins.open = _open_with_dirs
+_codecs.open = _codecs_open_with_dirs
+
+# Patch pandas to_csv to auto-create parent folder if available
+try:
+    import pandas as _pd  # noqa: F401
+    _orig_to_csv = _pd.DataFrame.to_csv
+    def _to_csv_with_dirs(self, path_or_buf=None, *args, **kwargs):
+        if isinstance(path_or_buf, (str, bytes, os.PathLike)):
+            _ensure_parent_dir(path_or_buf)
+        return _orig_to_csv(self, path_or_buf, *args, **kwargs)
+    _pd.DataFrame.to_csv = _to_csv_with_dirs
+except Exception:
+    pass
 
 import re, csv
 from collections import defaultdict
-from bs4 import BeautifulSoup as bs
 import pytz, datetime, dateutil, json
 from datetime import timedelta
 from dateutil import tz
 
-import functools
-
 from functools import reduce
 
 # Teacher name format changed. Remove commas and switch first to last   
 def fix_t_name(str):
     str = str.strip()
-    str = re.sub('\s+',' ',str)
+    str = re.sub(r'\s+',' ',str)
     parts = str.split(', ')
     if len(parts)>1:
         return parts[1].strip() + " " + parts[0].strip()
@@ -27,7 +69,7 @@ def split_class_dept(c):
     return c.split(' ')[0]
 def split_class_code(c):
     num = c.split(' ')[1]
-    parts = re.match('(\d+)([a-zA-Z]+)',num)
+    parts = re.match(r'(\d+)([a-zA-Z]+)',num)
     #ret = "Got %s, " % c
     if parts:
         r = int(parts.group(1))
@@ -37,7 +79,7 @@ def split_class_code(c):
     return int(num)
 def split_class_code_letter(c):
     num = c.split(' ')[1]
-    parts = re.match('(\d+)([A-Za-z]+)',num)
+    parts = re.match(r'(\d+)([A-Za-z]+)',num)
     if parts:
         return parts.group(2)
     return ''
@@ -84,6 +126,8 @@ def extract_key_values(lst, x):
     return reduce(lambda acc, item: acc + [item[x]] if isinstance(item, dict) and x in item else acc, lst, [])
 
 def stripper(s):
+    from bs4 import BeautifulSoup as bs
+
     REMOVE_ATTRIBUTES = [
     'lang','language','onmouseover','onmouseout','script','style','font',
     'dir','face','size','color','style','class','width','height','hspace',
@@ -171,14 +215,14 @@ def unix_time_millis(dt):
 
 # ENGL250 returns ENGL    
 def dept_from_name(n):
-    m = re.search('^([a-zA-Z]+)\s?[\d\/]+',n)
+    m = re.search(r'^([a-zA-Z]+)\s?[\d\/]+',n)
     if m: return m.group(1)
     print(("Couldn't find dept from: " + n))
     return ''
     
 # ENGL250 returns 250    
 def num_from_name(n):
-    m = re.search('^([a-zA-Z]+)\s?([\d\/]+[A-Z]?)',n)
+    m = re.search(r'^([a-zA-Z]+)\s?([\d\/]+[A-Z]?)',n)
     if m: return m.group(2)
     print(("Couldn't find num from: " + n))
     return ''
@@ -281,6 +325,8 @@ def clean_fn(s):
     return s
     
 def format_html(html):
+    from bs4 import BeautifulSoup as bs
+
     soup = bs(html, 'html.parser')
     return soup.prettify()