From e20a3a6611f6737861bc1eac21c4076e8213f997 Mon Sep 17 00:00:00 2001
From: Peter Howell <peter.howell@gmail.com>
Date: Fri, 29 Aug 2025 22:55:26 +0000
Subject: [PATCH] gpt

---
 gpt.py | 39 +++++++++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/gpt.py b/gpt.py
index 9f6d90a..7a21b97 100644
--- a/gpt.py
+++ b/gpt.py
@@ -469,6 +469,34 @@ def process_useful_info(start=0, num=0):
     from localcache2 import init_usefulinfo_schema, insert_usefulinfo_record
     init_usefulinfo_schema()
 
+    def _loose_parse_json(s):
+        # try direct
+        try:
+            return json.loads(s)
+        except Exception:
+            pass
+        # strip code fences
+        try:
+            m = re.search(r"```(?:json)?\s*(.*?)```", s, flags=re.S|re.I)
+            if m:
+                inner = m.group(1)
+                try:
+                    return json.loads(inner)
+                except Exception:
+                    s = inner
+        except Exception:
+            pass
+        # grab from first { to last }
+        try:
+            i = s.find('{')
+            j = s.rfind('}')
+            if i != -1 and j != -1 and j > i:
+                frag = s[i:j+1]
+                return json.loads(frag)
+        except Exception:
+            pass
+        return None
+
     def demo_f(idx, g):
         print(f"[{idx}] {g['subject']}  (count: {g['count']})")
         content = g['content']
@@ -486,10 +514,10 @@ def process_useful_info(start=0, num=0):
             "count": g.get('count'),
             "attachments": attach_paths,
         }
-        try:
-            parsed = json.loads(x)
+        parsed = _loose_parse_json(x)
+        if isinstance(parsed, dict) and parsed:
             record["summary"] = parsed
-        except Exception:
+        else:
             record["summary_raw"] = x
         with open(OUT_JSONL, "a", encoding="utf-8") as outf:
             outf.write(json.dumps(record, ensure_ascii=False) + "\n")
@@ -497,9 +525,12 @@ def process_useful_info(start=0, num=0):
         # Also persist to PostgreSQL using localcache2 with only parsed JSON
         if 'summary' in record:
             try:
-                insert_usefulinfo_record(record['summary'])
+                sid = insert_usefulinfo_record(record['summary'])
+                print('Inserted summary id:', sid)
             except Exception as e:
                 print('[warn] DB insert failed:', e)
+        else:
+            print('Skipped insert: could not parse JSON summary for this group.')
 
     # Interactive prompts if parameters not provided
     try: