Merge remote-tracking branch 'origin/master'

trying to merge

courses.py

@@ -1046,7 +1046,7 @@ def enroll_id_list_to_shell(id_list, shell_id, v=0):
 
 # multiple semesters
 def enroll_stem_students_live():
-    semesters = [288,289]
+    semesters = [289]
     
     for S in semesters:
         enroll_stem_students_live_semester(S)
@@ -1770,7 +1770,17 @@ def xlist_cwe():
 
     this_sem_190_id = 22890   # they get 190s and 290s
     this_sem_192_id = 22894   # they get 192s
-    this_sem_term = 289
+    # this_sem_term = 289
+
+    term = find_term( input("term? (ex: fa25) ") )
+
+    if not term or (not 'canvas_term_id' in term) or (not 'code' in term):
+        print(f"Couldn't find term.")
+        return
+
+    this_sem_term = term['canvas_term_id']
+    SEM = term['code']
+
     
     get_fresh = 1
     sem_courses = getCoursesInTerm(this_sem_term, get_fresh, 0)

gpt.py

@@ -13,7 +13,7 @@ client = OpenAI(
 
 DEFAULT_MODEL = "gpt-4o"
 
-SAVE_ATTACHEMENTS = 0
+SAVE_ATTACHEMENTS = 1
 
 
 def gpt_chat(instruction, prompt, model=DEFAULT_MODEL):
@@ -33,6 +33,19 @@ def gpt_chat(instruction, prompt, model=DEFAULT_MODEL):
     
     return result
 
+def summarize_u_info(msg):
+    system_role = """You are a practical efficient cataloger and indexer of information. What follows is one or more emails which most likely (but not necessarily) contains useful information of some sort. Please determine if the message has useful info, and if so, attempt to reformat it as such:
+    {"source": <original sender or source>, "date": <month and year MM/YY>, "tags": <short list of tags and topics, multi words use snake_case>, "short": <one sentence summary>, "summary": <complete summary with details if warranted>,"events":<see below>}
+    If one or more specific dates and times of an event (or deadline) of some sort are mentioned, please attempt to extract the datetime and one line summary of each event. The "events" field of the response should be a list of these: {"dt":<date and time>, "length":<if known>, "title":<title>,"description":<short description>}
+    For all summaries: speak in a first-person direct voice, authoritatively. For example, instead of saying 'x shared details about offerings at Bakersfield College, including a, b and c', just say 'Bakersfield college offers a, b and c'.
+    Some sample tags: event, advice, rule, warning, hr, it, disted, canvas, meeting, senate, dept_chairs, deans, administration, guided_pathways, site, file, article, contract, deadline, construction, catalog, schedule, curriqunet, banner, cvc, password, payroll, training, tip, graduation, photos, faculty, evaluation, convocation, flex, board, curriculum, ztc, oei, grant, accessibility, legislation, hyflex, hybrid, handbook, guideline, lti, api, integration, request, rule, scholarship, library, aws, opportunity, background, calendar, semester, accreditation, professional_development, zoom, ai, html, code, video, pocr, badge, liason, newsletter, act, law, equipment, best_practices, outcomes, slo, plo, ilo, data, cares_act, census, admissions, financial_aid, enrollment, title_5, syllabus, ccn, communication, club, survey, checklist, covid_19, pipr, program_review, policy, vpat, compliance, emergency, finals, rfp, fraud, research, library, tutoring, stem, writing_center, learning_commons, hire, promotion, help_desk, housing, url, tenure, tlc, mental_health, president, vpaa, vpss, vp, mac, meaningful_assessment, negotiation, union, libguide, evidence, retirement, 
+    Remember that not every email is notable. If this is the case, just return an empty json object.
+    Always answer in valid json, nothing else."""
+    x = gpt_chat(system_role,msg)
+    print(x)
+    return x
+
+
 
 def gpt_faq(faq):
     system_role = """Ignore all previous instructions. You are an expert on the management and implementation of all aspects of online courses and running a distance education program, especially in higher education and community college. You are also an expert in pedagogy and the tools and techniques of effective teaching and learning, based on research and experience. You know how to focus on concrete examples of effective teaching which actually work in the real world and the advice you give is specific and immediately useful to teachers in the classroom. You provide accurate, factual, thoughtful, nuanced answers, and are brilliant at reasoning. Your users are experts in teaching and college administration, and are comfortable with the California and Federal Educational code, and know they need to comply with it. There is no need to explain this beyond noting any relevant laws. Don't be verbose in your answers, but do provide details and examples where it might help the explanation.  
@@ -76,44 +89,248 @@ def sample_send_email():
 
 
 
-def fetch_useful_info():
+
+
+
+def fetch_useful_info(save_attachments=True, folder_name='useful info ref'):
+    import os, re
+    from pathlib import Path
+    from itertools import groupby
+    from datetime import datetime
     import win32com.client
+
+    CACHE = Path("cache")
+    ATT_DIR = CACHE / "attachments_useful_info"
+    LOG_PATH = CACHE / "email_usefulinfo_sorted.txt"
+    ATT_DIR.mkdir(parents=True, exist_ok=True)
+    CACHE.mkdir(parents=True, exist_ok=True)
+
+    # --- helpers ---
+    prefix_re = re.compile(r'^\s*(re|fw|fwd|aw|sv|vb|tr|wg)\s*:\s*', re.I)  # common locales too
+    bracket_tag_re = re.compile(r'^\s*(\[[^\]]+\]\s*)+', re.I)
+
+    def normalize_subject(s: str) -> str:
+        if not s:
+            return "(no subject)"
+        s = s.strip()
+        # strip leading [TAGS] like [EXTERNAL] [Some System]
+        s = bracket_tag_re.sub('', s)
+        # strip any chain of RE:/FWD: etc. at start
+        while prefix_re.match(s):
+            s = prefix_re.sub('', s, count=1)
+        # collapse whitespace
+        s = re.sub(r'\s+', ' ', s).strip()
+        return s or "(no subject)"
+
+    def safe_name(s: str) -> str:
+        # reasonable Windows-safe folder name for a subject
+        s = re.sub(r'[<>:"/\\|?*\x00-\x1F]', '_', s)
+        return s[:120]
+
+    def iso(dt):
+        # Outlook COM datetime -> ISO string (local)
+        try:
+            return dt.strftime('%Y-%m-%d %H:%M:%S')
+        except Exception:
+            return str(dt)
+
+    def save_all_attachments(mail, subject_dir: Path):
+        saved = []
+        try:
+            atts = mail.Attachments
+            count = atts.Count
+            if count == 0:
+                return saved
+            subject_dir.mkdir(parents=True, exist_ok=True)
+            # iterate COM collection by index (1-based)
+            for i in range(1, count + 1):
+                att = atts.Item(i)
+                # build unique filename to avoid collisions
+                base = str(att.FileName) if getattr(att, "FileName", None) else f"attachment_{i}"
+                base = safe_name(base)
+                # prefix by sent time for clarity
+                ts = datetime.now().strftime('%Y%m%d_%H%M%S')
+                out = subject_dir / f"{ts}_{base}"
+                att.SaveAsFile(str(out))
+                saved.append(str(out.as_posix()))
+        except Exception:
+            # swallow attachment oddities; continue
+            pass
+        return saved
+
+    # --- Outlook ---
     outlook = win32com.client.Dispatch("Outlook.Application").GetNamespace("MAPI")
     root_folder = outlook.Folders.Item(1)
-    print (f"Root folder: {root_folder.Name}")
+    print(f"Root folder: {root_folder.Name}")
 
-    #And to know the names of the subfolders you have:
+    uinfo = root_folder.Folders[folder_name]
-    print("\nFolders:")
-    for folder in root_folder.Folders:
-        print ("  " + folder.Name)
 
-    log = codecs.open("cache/email_usefulinfo.txt","w","utf-8")
+    # Collect first to a list so we can sort/group
+    records = []
+    items = uinfo.Items
+    # Optional: sort by SentOn ascending inside Outlook (helps performance for big folders)
+    try:
+        items.Sort("[SentOn]", True)  # True => ascending
+    except Exception:
+        pass
 
-    # access a subfolder 
+    for message in items:
-    print("\nUseful Info Reference:")
+        # Skip non-mail items
-    uinfo = root_folder.Folders['useful info ref']
+        if getattr(message, "Class", None) != 43:  # 43 = MailItem
-    for message in uinfo.Items:
+            continue
-        atch_list = "Attachments: "
+        subj_raw = getattr(message, "Subject", "") or ""
-        atch_count = 0
+        subj_norm = normalize_subject(subj_raw)
-        if SAVE_ATTACHEMENTS:
+
-            attachments = message.Attachments
+        # sender fallback chain
+        sender = None
+        for attr in ("Sender", "SenderName", "SenderEmailAddress"):
             try:
-                attachment = attachments.Item(1)
+                sender = getattr(message, attr)
-                for attachment in message.Attachments:
+                if sender:
-                    print("    -> " + str(attachment))
-                    #loc = "C:\\Users\\peter\\Documents\\gavilan\\ca_merged\\canvasapp\\cache\\attachments\\" + str(attachment)
-                    loc = "C:\\Users\\phowell\\source\\repos\\canvasapp\\cache\\attachments\\" + str(attachment)
-                    attachment.SaveAsFile(loc)
-                    atch_list += str(attachment) + ', '
-                    atch_count += 1
                     break
-            except Exception as e:
+            except Exception:
                 pass
-        print("  " + message.Subject)
+        sender = str(sender) if sender else "UNKNOWN"
-        log.write(f"\n\n---\nSubject: {message.Subject}\nFrom: {message.Sender}\n")
+
-        if atch_count:
+        sent_on = getattr(message, "SentOn", None)
-            log.write(f"{atch_list}\n")
+        sent_iso = iso(sent_on)
-        log.write(f"Date: {message.SentOn}\n\n{message.body}\n")
+
+        attachments_saved = []
+        if save_attachments:
+            attachments_saved = save_all_attachments(
+                message,
+                ATT_DIR / safe_name(subj_norm)
+            )
+
+        body = ""
+        try:
+            body = message.Body or ""
+        except Exception:
+            try:
+                body = message.HTMLBody or ""
+            except Exception:
+                body = ""
+
+        records.append({
+            "subject_norm": subj_norm,
+            "subject_raw": subj_raw,
+            "sender": sender,
+            "sent_on": sent_on,
+            "sent_iso": sent_iso,
+            "attachments": attachments_saved,
+            "body": body,
+        })
+
+    # Sort by normalized subject, then by sent time (ascending)
+    records.sort(key=lambda r: (r["subject_norm"].lower(), r["sent_on"] or datetime.min))
+
+    # Write grouped log
+    with open(LOG_PATH, "w", encoding="utf-8", newline="") as log:
+        current_subject = None
+        for subj, group_iter in groupby(records, key=lambda r: r["subject_norm"]):
+            thread = list(group_iter)
+            log.write(f"\n\n### {subj}  —  {len(thread)} message(s)\n")
+            for r in thread:
+                if r["attachments"]:
+                    att_line = "Attachments: " + ", ".join(r["attachments"]) + "\n"
+                else:
+                    att_line = ""
+                log.write(
+                    f"\n---\n"
+                    f"Subject: {r['subject_raw']}\n"
+                    f"From: {r['sender']}\n"
+                    f"Date: {r['sent_iso']}\n"
+                    f"{att_line}\n"
+                    f"{r['body']}\n"
+                )
+
+    print(f"Wrote grouped log -> {LOG_PATH.as_posix()}")
+
+
+
+
+    
+def process_useful_info():
+    import re
+    from pathlib import Path
+    from typing import Callable, List, Dict, Optional
+
+    HEADER_RE = re.compile(r'\r?\n###\s(.*)\r?\n')  # your pattern, CRLF-safe
+    COUNT_RE  = re.compile(r'^(?P<subject>.*?)\s+—\s+(?P<count>\d+)\s+message', re.I)
+
+    def parse_groups(text: str) -> List[Dict]:
+        """
+        Return a list of groups found in the log file.
+        Each group is a dict: {header, subject, count, content}
+        """
+        groups = []
+        matches = list(HEADER_RE.finditer(text))
+        if not matches:
+            return groups
+
+        for i, m in enumerate(matches):
+            header = m.group(1).strip()  # e.g. "Subject X — 3 message(s)" OR just "Subject X"
+            start = m.end()
+            end   = matches[i+1].start() if i + 1 < len(matches) else len(text)
+            content = text[start:end]
+
+            # Try to extract subject and count if present
+            subject = header
+            count: Optional[int] = None
+            cm = COUNT_RE.search(header)
+            if cm:
+                subject = cm.group('subject').strip()
+                try:
+                    count = int(cm.group('count'))
+                except Exception:
+                    count = None
+
+            groups.append({
+                "header": header,
+                "subject": subject,
+                "count": count,
+                "content": content
+            })
+        return groups
+
+    def for_each_group(
+        log_path: str = "cache/email_usefulinfo_sorted.txt",
+        f: Callable[[int, Dict], None] = lambda idx, g: None,
+        start: int = 1,
+        count: int = -1
+    ) -> None:
+        """
+        Read the grouped log, split into groups, and call f(index, group) on each.
+        start: 1-based index to begin processing (useful for resuming).
+        """
+        p = Path(log_path)
+        text = p.read_text(encoding="utf-8", errors="replace")
+
+        groups = parse_groups(text)
+
+        done = 0
+
+        if start < 1:
+            start = 1
+        for idx, g in enumerate(groups, start=1):
+            if idx < start:
+                continue
+            f(idx, g)
+            done += 1
+            if count != -1 and done >= count:
+                return
+
+    def demo_f(idx, g):
+        print(f"[{idx}] {g['subject']}  (count: {g['count']})")
+        x = summarize_u_info(g['content'])
+        # Do your processing here. g['content'] is the whole thread block.
+
+    for_each_group(
+        log_path="cache/email_usefulinfo_sorted.txt",
+        f=demo_f,
+        start=1,  # change to resume at Nth group
+        count=10
+    )
 
 
 
@@ -257,6 +474,7 @@ if __name__ == "__main__":
                 3: ['fetch "useful info" mailbox', fetch_useful_info],
                 4: ['fetch "faq" mailbox and gpt summarize', fetch_faq],
                 5: ['list faq mailbox', list_faq],
+                6: ['process useful info msgs', process_useful_info],
     }
 
 

outcomes2022.py

@@ -363,7 +363,17 @@ def repair_outcome_points(course_id):
 def add_o_dept_dry_run():
     add_o_dept(1)
 
-def add_o_whole_term():
+def add_o_whole_term(term=0):
+    if not term:
+        tt = find_term( input("term? (ex: fa25) ") )
+
+        if not tt or (not 'canvas_term_id' in tt) or (not 'code' in tt):
+            print(f"Couldn't find term.")
+            return
+
+        term = tt['canvas_term_id']
+        sem = tt['code']
+
     course_groups = full_term_overview(term, 0)
 
     dept_shells_to_add = [ a for a in course_groups['no outcomes'] ]

semesters.py

@@ -8,7 +8,7 @@ season_to_number = { 'Fall': '70', 'Summer': '50', 'Spring': '30', 'Winter': '10
 # Inverse
 number_to_season = {v: k for k, v in season_to_number.items()}
 
-s_to_n = {'sp':'30','su':'50','fa':'70'}
+s_to_n = {'wi':'30','sp':'30','su':'50','fa':'70'}
 season_to_short = {
     'Summer': 'su',
     'Fall': 'fa',
@@ -178,6 +178,7 @@ for std, code_str, start in zip(standard, code, begin):
     records[(season, year)] = {
         'standard': std,
         'code': code_str,
+        'bannercode': short_to_sis(code_str),
         'begin': start,
         'canvas_term_id': None,
         'canvas_name': None,
@@ -203,6 +204,7 @@ for key, rec in records.items():
         f"{year} {season}",
         f"{season} {year}",
         rec['code'],
+        rec['bannercode'],
         rec['canvas_name'] if rec['canvas_name'] else None,
         str(rec['canvas_term_id']) 
     ]