From 001e2c6723d6a443cd1579944493521f96382768 Mon Sep 17 00:00:00 2001
From: Peter Howell <peter.howell@gmail.com>
Date: Wed, 13 Aug 2025 09:52:17 -0700
Subject: [PATCH] flex stuff

---
 flexday.py   | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 pipelines.py |  2 ++
 2 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/flexday.py b/flexday.py
index fb4b70c..b7a3c98 100644
--- a/flexday.py
+++ b/flexday.py
@@ -7,7 +7,7 @@ from users import getEmail
 
 def user_db_sync():
     # currently in db
-    conusr = fetch("http://192.168.1.6:8080/dir_api.php?users=1")
+    conusr = fetch("http://deep1:8080/dir_api.php?users=1")
     conusr_emails = set([x.lower() for x in funcy.pluck('email',conusr)])
 
     #fetch all staff from ilearn                        ILRN            unique emails                                             
@@ -23,7 +23,7 @@ def user_db_sync():
             print("INSERT INTO conf_users (goo,email,name) VALUES ('%s', '%s', '%s');" % (goo,e,E['short_name']) )
             
 
-
+# no longer relevant cause we don't use personnel table anymore
 def user_db_sync2():
     #fetch all personnel dir entries from dir_api.php.  PERSL           unique emails                                             
     persl = fetch("http://hhh.gavilan.edu/phowell/map/dir_api.php?personnel=1")
@@ -300,6 +300,90 @@ def generate_insert_statements(table_name='conf_sessions'):
         print(S)
     return insert_statements
 
+def cross_check_users():
+    """
+    Reads:
+      - ilearn_staff.json  (list of Canvas users; uses fields: sis_user_id, name, email, login_id)
+      - conf_users.csv     (existing rows; headers include: goo)
+    Writes:
+      - conf_users_inserts.sql with INSERTs for users missing from conf_users
+
+    Assumptions:
+      - "goo" is sis_user_id with the leading 'G00' removed (e.g., G00138124 -> 138124).
+      - Skip deactivated users (name contains "(Deactivated)" or login_id == "deactivated").
+      - Skip users whose sis_user_id doesn’t match G00\d+.
+      - New rows default to active=0 and p2id=NULL.
+    """
+
+    import json, csv, re
+
+    STAFF_JSON = "cache/ilearn_staff.json"
+    CONF_CSV   = "cache/conf_users.csv"
+    OUT_SQL    = "cache/conf_users_inserts.sql"
+    TABLE      = "conf_users"
+
+    # ---- Load existing goo set from conf_users.csv ----
+    existing_goos = set()
+    with open(CONF_CSV, newline="", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            s = (row.get("goo") or "").strip()
+            if s.isdigit():
+                existing_goos.add(int(s))
+
+    # ---- Load Canvas staff ----
+    with open(STAFF_JSON, "r", encoding="utf-8") as f:
+        users = json.load(f)
+
+    def esc(s):
+        if s is None:
+            return "NULL"
+        return "'" + s.replace("'", "''") + "'"
+
+    inserts = []
+    scanned = 0
+    skipped_deactivated = 0
+    skipped_bad_sis = 0
+    missing = 0
+
+    for u in users:
+        scanned += 1
+        name = (u.get("name") or "").strip()
+        login = (u.get("login_id") or "").strip()
+        sis = (u.get("sis_user_id") or "").strip()
+
+        # Skip non G00-form sis_user_id
+        m = re.fullmatch(r"G00(\d+)", sis)
+        if not m:
+            skipped_bad_sis += 1
+            continue
+
+        # Skip deactivated
+        #low_name = name.lower()
+        #if "(deactivated)" in low_name or login.lower() == "deactivated":
+        #    skipped_deactivated += 1
+        #    continue
+
+        goo = int(m.group(1))
+        if goo in existing_goos:
+            continue
+
+        email = (u.get("email") or "").strip() or None
+
+        sql = (
+            f"INSERT INTO `{TABLE}` (goo, email, name, active, p2id) "
+            f"VALUES ({goo}, {esc(email)}, {esc(name)}, 0, NULL);"
+        )
+        inserts.append(sql)
+        missing += 1
+
+    with open(OUT_SQL, "w", encoding="utf-8") as f:
+        f.write("-- Generated INSERTs for missing conf_users rows\n")
+        f.write("\n".join(inserts))
+        f.write("\n")
+
+    print(f"Wrote {missing} INSERTs to {OUT_SQL}")
+    print(f"Scanned: {scanned} | Existing goos: {len(existing_goos)} | Skipped deactivated: {skipped_deactivated} | Skipped bad sis: {skipped_bad_sis}")
 
 if __name__ == "__main__":
     print ("")
@@ -307,7 +391,8 @@ if __name__ == "__main__":
                 2: ['generate sql to fix conf_user dups', correct_dup_user_rows] , 
                 3: ['add names to new accounts', find_unnamed_people],
                 4: ['search for user', search_user],
-                5: ['generate insert statements', generate_insert_statements ]
+                5: ['generate insert statements', generate_insert_statements ],
+                6: ['cross check users', cross_check_users ],
     }
 
 
diff --git a/pipelines.py b/pipelines.py
index 5c71431..54b9322 100644
--- a/pipelines.py
+++ b/pipelines.py
@@ -219,6 +219,8 @@ async def canvas_data_2024():
     client_id: str = os.environ["DAP_CLIENT_ID"]
     client_secret: str = os.environ["DAP_CLIENT_SECRET"]
     #connection_string: str = "postgresql://postgres:rolley34@192.168.1.6/db"
+
+    # todo: use secrets
     connection_string: str = "postgresql://postgres:rolley34@deep1/db"
     
     desired_tables = "users,courses,communication_channels,context_modules,conversation_message_participants,conversation_messages,conversation_participants,conversations,course_sections,enrollment_states,enrollment_dates_overrides,enrollment_terms,enrollments,learning_outcome_groups,learning_outcome_question_results,learning_outcomes,pseudonyms,quizzes,scores,submissions,submission_versions,wiki_pages,wikis".split(',')