Compare commits
No commits in common. "0dbf0f7bdc52ac9a4c15d3b0408de4e0104468f5" and "d49e0d0c1721c3e0308cf3a993d924de351f5373" have entirely different histories.
0dbf0f7bdc
...
d49e0d0c17
103
gpt.py
103
gpt.py
|
|
@ -11,17 +11,17 @@ client = OpenAI(
|
||||||
organization=openai_org
|
organization=openai_org
|
||||||
)
|
)
|
||||||
|
|
||||||
DEFAULT_MODEL = "gpt-5-nano"
|
DEFAULT_MODEL = "gpt-4o"
|
||||||
|
|
||||||
SAVE_ATTACHEMENTS = 1
|
SAVE_ATTACHEMENTS = 1
|
||||||
|
|
||||||
|
|
||||||
def gpt_chat(instruction, prompt, model=DEFAULT_MODEL):
|
def gpt_chat(instruction, prompt, model=DEFAULT_MODEL):
|
||||||
# 1) Strip extremely long Outlook protection URLs first
|
# 1) Strip extremely long Outlook protection URLs first
|
||||||
#try:
|
try:
|
||||||
# prompt = re.sub(r"\S*protection\.outlook\.com\S*", "", prompt, flags=re.I)
|
prompt = re.sub(r"\S*protection\.outlook\.com\S*", "", prompt, flags=re.I)
|
||||||
#except Exception:
|
except Exception:
|
||||||
# pass
|
pass
|
||||||
|
|
||||||
# 2) Crude token estimation and truncation (target ~29k tokens)
|
# 2) Crude token estimation and truncation (target ~29k tokens)
|
||||||
# Approximates 1 token ≈ 4 characters.
|
# Approximates 1 token ≈ 4 characters.
|
||||||
|
|
@ -77,8 +77,8 @@ def gpt_chat(instruction, prompt, model=DEFAULT_MODEL):
|
||||||
def summarize_u_info(msg):
|
def summarize_u_info(msg):
|
||||||
system_role = """You are a practical efficient cataloger and indexer of information. What follows is one or more emails which most likely (but not necessarily) contains useful information of some sort. Please determine if the message has useful info, and if so, attempt to reformat it as such:
|
system_role = """You are a practical efficient cataloger and indexer of information. What follows is one or more emails which most likely (but not necessarily) contains useful information of some sort. Please determine if the message has useful info, and if so, attempt to reformat it as such:
|
||||||
{"source": <original sender or source>, "date": <month and year MM/YY>, "tags": <short list of tags and topics, multi words use snake_case>, "short": <one sentence summary>, "summary": <complete summary with details if warranted>,"events":<see below>}
|
{"source": <original sender or source>, "date": <month and year MM/YY>, "tags": <short list of tags and topics, multi words use snake_case>, "short": <one sentence summary>, "summary": <complete summary with details if warranted>,"events":<see below>}
|
||||||
If one or more specific dates and times of an event (or deadline) of some sort are mentioned, please attempt to extract the datetime and 1-2 line summary of each event. The "events" field of the response should be a list of these: {"dt":<date and time>, "length":<if known>, "title":<title>,"description":<short description>,"contact":<person/email who seems to be in charge or know more>}. Try to include relevant links in the description, if they seem relevant.
|
If one or more specific dates and times of an event (or deadline) of some sort are mentioned, please attempt to extract the datetime and one line summary of each event. The "events" field of the response should be a list of these: {"dt":<date and time>, "length":<if known>, "title":<title>,"description":<short description>}
|
||||||
For all summaries: speak in a first-person direct voice, authoritatively. For example, instead of saying 'x shared details about offerings at Bakersfield College, including a, b and c', just say 'Bakersfield college offers a, b and c'. If there are links that seem relevant in the email, include them in the summary.
|
For all summaries: speak in a first-person direct voice, authoritatively. For example, instead of saying 'x shared details about offerings at Bakersfield College, including a, b and c', just say 'Bakersfield college offers a, b and c'.
|
||||||
Some sample tags: event, advice, rule, warning, hr, it, disted, canvas, meeting, senate, dept_chairs, deans, administration, guided_pathways, site, file, article, contract, deadline, construction, catalog, schedule, curriqunet, banner, cvc, password, payroll, training, tip, graduation, photos, faculty, evaluation, convocation, flex, board, curriculum, ztc, oei, grant, accessibility, legislation, hyflex, hybrid, handbook, guideline, lti, api, integration, request, rule, scholarship, library, aws, opportunity, background, calendar, semester, accreditation, professional_development, zoom, ai, html, code, video, pocr, badge, liason, newsletter, act, law, equipment, best_practices, outcomes, slo, plo, ilo, data, cares_act, census, admissions, financial_aid, enrollment, title_5, syllabus, ccn, communication, club, survey, checklist, covid_19, pipr, program_review, policy, vpat, compliance, emergency, finals, rfp, fraud, research, library, tutoring, stem, writing_center, learning_commons, hire, promotion, help_desk, housing, url, tenure, tlc, mental_health, president, vpaa, vpss, vp, mac, meaningful_assessment, negotiation, union, libguide, evidence, retirement,
|
Some sample tags: event, advice, rule, warning, hr, it, disted, canvas, meeting, senate, dept_chairs, deans, administration, guided_pathways, site, file, article, contract, deadline, construction, catalog, schedule, curriqunet, banner, cvc, password, payroll, training, tip, graduation, photos, faculty, evaluation, convocation, flex, board, curriculum, ztc, oei, grant, accessibility, legislation, hyflex, hybrid, handbook, guideline, lti, api, integration, request, rule, scholarship, library, aws, opportunity, background, calendar, semester, accreditation, professional_development, zoom, ai, html, code, video, pocr, badge, liason, newsletter, act, law, equipment, best_practices, outcomes, slo, plo, ilo, data, cares_act, census, admissions, financial_aid, enrollment, title_5, syllabus, ccn, communication, club, survey, checklist, covid_19, pipr, program_review, policy, vpat, compliance, emergency, finals, rfp, fraud, research, library, tutoring, stem, writing_center, learning_commons, hire, promotion, help_desk, housing, url, tenure, tlc, mental_health, president, vpaa, vpss, vp, mac, meaningful_assessment, negotiation, union, libguide, evidence, retirement,
|
||||||
Remember that not every email is notable. If this is the case, just return an empty json object.
|
Remember that not every email is notable. If this is the case, just return an empty json object.
|
||||||
Always answer in valid json, nothing else."""
|
Always answer in valid json, nothing else."""
|
||||||
|
|
@ -287,88 +287,6 @@ def fetch_useful_info(save_attachments=True, folder_name='useful info ref'):
|
||||||
|
|
||||||
print(f"Wrote grouped log -> {LOG_PATH.as_posix()}")
|
print(f"Wrote grouped log -> {LOG_PATH.as_posix()}")
|
||||||
|
|
||||||
# Insert original emails & attachments into DB (summary linkage added later)
|
|
||||||
from localcache2 import init_usefulinfo_schema, insert_usefulinfo_email
|
|
||||||
init_usefulinfo_schema()
|
|
||||||
|
|
||||||
# Helpers to extract attachment text
|
|
||||||
try:
|
|
||||||
import PyPDF2
|
|
||||||
except Exception:
|
|
||||||
PyPDF2 = None
|
|
||||||
try:
|
|
||||||
import docx
|
|
||||||
except Exception:
|
|
||||||
docx = None
|
|
||||||
try:
|
|
||||||
from pptx import Presentation
|
|
||||||
except Exception:
|
|
||||||
Presentation = None
|
|
||||||
|
|
||||||
def _extract_pdf_text(p):
|
|
||||||
if not PyPDF2:
|
|
||||||
return ''
|
|
||||||
try:
|
|
||||||
with open(p, 'rb') as fh:
|
|
||||||
r = PyPDF2.PdfReader(fh)
|
|
||||||
txt = []
|
|
||||||
for i, pg in enumerate(r.pages[:10]):
|
|
||||||
try: txt.append(pg.extract_text() or '')
|
|
||||||
except Exception: pass
|
|
||||||
return "\n".join(txt)[:12000]
|
|
||||||
except Exception:
|
|
||||||
return ''
|
|
||||||
def _extract_docx_text(p):
|
|
||||||
if not docx:
|
|
||||||
return ''
|
|
||||||
try:
|
|
||||||
d = docx.Document(p)
|
|
||||||
return "\n".join([para.text for para in d.paragraphs if para.text])[:12000]
|
|
||||||
except Exception:
|
|
||||||
return ''
|
|
||||||
def _extract_pptx_text(p):
|
|
||||||
if not Presentation:
|
|
||||||
return ''
|
|
||||||
try:
|
|
||||||
pres = Presentation(p)
|
|
||||||
chunks = []
|
|
||||||
for slide in pres.slides:
|
|
||||||
for shape in slide.shapes:
|
|
||||||
try:
|
|
||||||
if hasattr(shape, 'has_text_frame') and shape.has_text_frame:
|
|
||||||
for para in shape.text_frame.paragraphs:
|
|
||||||
txt = ''.join(run.text for run in para.runs)
|
|
||||||
if txt: chunks.append(txt)
|
|
||||||
except Exception: pass
|
|
||||||
return "\n".join(chunks)[:12000]
|
|
||||||
except Exception:
|
|
||||||
return ''
|
|
||||||
|
|
||||||
for rec in records:
|
|
||||||
atts = []
|
|
||||||
for ap in rec['attachments']:
|
|
||||||
ext = os.path.splitext(ap)[1].lower()
|
|
||||||
txt = ''
|
|
||||||
if ext == '.pdf':
|
|
||||||
txt = _extract_pdf_text(ap)
|
|
||||||
elif ext == '.docx':
|
|
||||||
txt = _extract_docx_text(ap)
|
|
||||||
elif ext == '.pptx':
|
|
||||||
txt = _extract_pptx_text(ap)
|
|
||||||
atts.append({'path': ap, 'text': txt or None})
|
|
||||||
try:
|
|
||||||
insert_usefulinfo_email(
|
|
||||||
subject_raw=rec['subject_raw'],
|
|
||||||
subject_norm=rec['subject_norm'],
|
|
||||||
sender=rec['sender'],
|
|
||||||
sent_iso=rec['sent_iso'],
|
|
||||||
body=rec['body'],
|
|
||||||
attachments=atts,
|
|
||||||
summary_id=None,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print('[usefulinfo][email-insert-failed]', rec.get('subject_raw'), str(e))
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -609,13 +527,6 @@ def process_useful_info(start=0, num=0):
|
||||||
try:
|
try:
|
||||||
sid = insert_usefulinfo_record(record['summary'])
|
sid = insert_usefulinfo_record(record['summary'])
|
||||||
print('Inserted summary id:', sid)
|
print('Inserted summary id:', sid)
|
||||||
# Link original emails to this summary by normalized subject
|
|
||||||
try:
|
|
||||||
from localcache2 import link_emails_to_summary
|
|
||||||
link_count = link_emails_to_summary(g['subject'], sid)
|
|
||||||
print('Linked emails to summary:', link_count)
|
|
||||||
except Exception as e2:
|
|
||||||
print('[usefulinfo][email-link-failed]', str(e2))
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('[warn] DB insert failed:', e)
|
print('[warn] DB insert failed:', e)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import markdown, json, threading
|
||||||
#from functools import wraps
|
#from functools import wraps
|
||||||
# from content import my_site
|
# from content import my_site
|
||||||
|
|
||||||
from flask import Flask, request, send_from_directory, Response, render_template, jsonify
|
from flask import Flask, request, send_from_directory, Response, render_template
|
||||||
from flask import send_file
|
from flask import send_file
|
||||||
from flask_socketio import SocketIO, emit
|
from flask_socketio import SocketIO, emit
|
||||||
from werkzeug.routing import PathConverter
|
from werkzeug.routing import PathConverter
|
||||||
|
|
@ -42,7 +42,7 @@ else:
|
||||||
q = Queue()
|
q = Queue()
|
||||||
|
|
||||||
|
|
||||||
HOST_NAME = 'deep1' #
|
HOST_NAME = '192.168.1.6' #
|
||||||
PORT_NUMBER = 8080 # Maybe set this to 9000.
|
PORT_NUMBER = 8080 # Maybe set this to 9000.
|
||||||
|
|
||||||
datafile = 'lambda.csv'
|
datafile = 'lambda.csv'
|
||||||
|
|
@ -126,7 +126,35 @@ def flask_thread(q):
|
||||||
return markdown.markdown( codecs.open('cache/crawl/'+filename,'r','utf-8').read() ) + \
|
return markdown.markdown( codecs.open('cache/crawl/'+filename,'r','utf-8').read() ) + \
|
||||||
"<pre>" + codecs.open('cache/crawl/'+filename,'r','utf-8').read() + "</pre>"
|
"<pre>" + codecs.open('cache/crawl/'+filename,'r','utf-8').read() + "</pre>"
|
||||||
|
|
||||||
# Smart home routes removed; handled in smarthome.py
|
@app.route('/clearscreens')
|
||||||
|
def clears():
|
||||||
|
clearscreens()
|
||||||
|
return homepage()
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/displaypi/on')
|
||||||
|
def dpi():
|
||||||
|
displaypi_on()
|
||||||
|
return homepage()
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/displaypi/off')
|
||||||
|
def dpi2():
|
||||||
|
displaypi_off()
|
||||||
|
return homepage()
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/screensoff')
|
||||||
|
def screenoff_a():
|
||||||
|
screenoff()
|
||||||
|
return homepage()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/light')
|
||||||
|
def light():
|
||||||
|
desklight()
|
||||||
|
return homepage()
|
||||||
|
|
||||||
|
|
||||||
@app.route('/image/<filename>', methods=['GET','POST'])
|
@app.route('/image/<filename>', methods=['GET','POST'])
|
||||||
|
|
@ -233,19 +261,6 @@ def flask_thread(q):
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
def home():
|
def home():
|
||||||
return server.homepage()
|
return server.homepage()
|
||||||
|
|
||||||
# Bridge new server routes so this app serves them too
|
|
||||||
@app.route('/useful-info')
|
|
||||||
def useful_info_page_bridge():
|
|
||||||
return server.useful_info_page()
|
|
||||||
|
|
||||||
@app.route('/api/useful-info')
|
|
||||||
def useful_info_api_bridge():
|
|
||||||
return server.useful_info_api()
|
|
||||||
|
|
||||||
@app.route('/health')
|
|
||||||
def health():
|
|
||||||
return jsonify({'app': 'interactive.py', 'status': 'ok'})
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# STATIC ROUTES
|
# STATIC ROUTES
|
||||||
|
|
@ -338,10 +353,11 @@ def serve():
|
||||||
x.start()
|
x.start()
|
||||||
#webbrowser.open_new_tab("http://localhost:5000")
|
#webbrowser.open_new_tab("http://localhost:5000")
|
||||||
|
|
||||||
# mqtt disabled in this app
|
y = threading.Thread(target=mqtt_loop)
|
||||||
|
y.start()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
|
||||||
from random import randint
|
from random import randint
|
||||||
import asciimatics
|
import asciimatics
|
||||||
from asciimatics.scene import Scene
|
from asciimatics.scene import Scene
|
||||||
|
|
@ -422,6 +438,17 @@ def text_app():
|
||||||
Screen.wrapper(demo, catch_interrupt=False)
|
Screen.wrapper(demo, catch_interrupt=False)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
'''try:
|
||||||
|
Screen.wrapper(demo, catch_interrupt=False)
|
||||||
|
sys.exit(0)
|
||||||
|
except ResizeScreenError as e:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
print(traceback.format_exc())
|
||||||
|
sys.exit(0)'''
|
||||||
|
#mv = MenuView(demo)
|
||||||
|
#Screen.wrapper()
|
||||||
|
#Screen.wrapper(demo)
|
||||||
|
|
||||||
def demo(screen):
|
def demo(screen):
|
||||||
scene1 = Scene([MenuView(screen)],-1)
|
scene1 = Scene([MenuView(screen)],-1)
|
||||||
|
|
@ -444,7 +471,7 @@ def demo2(screen):
|
||||||
if last_event.key_code in (ord('Q'), ord('q')):
|
if last_event.key_code in (ord('Q'), ord('q')):
|
||||||
return
|
return
|
||||||
screen.refresh()
|
screen.refresh()
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -503,7 +530,7 @@ def demo2(screen):
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
options = { 1: ['start web server',serve] ,
|
options = { 1: ['start web server',serve] ,
|
||||||
#2: ['start text app', text_app],
|
2: ['start text app', text_app],
|
||||||
}
|
}
|
||||||
print ('')
|
print ('')
|
||||||
|
|
||||||
|
|
@ -521,3 +548,4 @@ if __name__ == "__main__":
|
||||||
|
|
||||||
# Call the function in the options dict
|
# Call the function in the options dict
|
||||||
options[ int(resp)][1]()
|
options[ int(resp)][1]()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -157,7 +157,7 @@ def setup_table(table='requests'):
|
||||||
for L in open('cache/request_table.txt','r').readlines():
|
for L in open('cache/request_table.txt','r').readlines():
|
||||||
L = L.strip()
|
L = L.strip()
|
||||||
#print(L)
|
#print(L)
|
||||||
(col,type) = re.split(r"\s\s\s\s",L)
|
(col,type) = re.split("\s\s\s\s",L)
|
||||||
if not first:
|
if not first:
|
||||||
q += ",\n"
|
q += ",\n"
|
||||||
first = 0
|
first = 0
|
||||||
|
|
@ -1159,13 +1159,13 @@ def guess_dept(t):
|
||||||
method = 2 # xlisted takes dept first listed
|
method = 2 # xlisted takes dept first listed
|
||||||
|
|
||||||
if method==1:
|
if method==1:
|
||||||
p = r"^([A-Z/]+)\d+"
|
p = "^([A-Z/]+)\d+"
|
||||||
m = re.search(p, t['code'])
|
m = re.search(p, t['code'])
|
||||||
if m:
|
if m:
|
||||||
return m.group(1)
|
return m.group(1)
|
||||||
return '?'
|
return '?'
|
||||||
if method==2:
|
if method==2:
|
||||||
p = r"^([A-Z]+)[\d/]+"
|
p = "^([A-Z]+)[\d/]+"
|
||||||
m = re.search(p, t['code'])
|
m = re.search(p, t['code'])
|
||||||
if m:
|
if m:
|
||||||
return m.group(1)
|
return m.group(1)
|
||||||
|
|
@ -1700,7 +1700,7 @@ ORDER BY u.sortablename;"""
|
||||||
|
|
||||||
def build_db_schedule():
|
def build_db_schedule():
|
||||||
# from the schedule json files
|
# from the schedule json files
|
||||||
target = r"\_sched\_expanded\.json"
|
target = "\_sched\_expanded\.json"
|
||||||
def finder(st):
|
def finder(st):
|
||||||
return re.search(target,st)
|
return re.search(target,st)
|
||||||
|
|
||||||
|
|
@ -2132,3 +2132,4 @@ if __name__ == "__main__":
|
||||||
# Call the function in the options dict
|
# Call the function in the options dict
|
||||||
options[ int(resp)][1]()
|
options[ int(resp)][1]()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
109
localcache2.py
109
localcache2.py
|
|
@ -634,40 +634,6 @@ def init_usefulinfo_schema():
|
||||||
);
|
);
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
-- Original Emails and Attachments
|
|
||||||
CUR.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS useful_info_email (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
summary_id BIGINT NULL REFERENCES useful_info_summary(id) ON DELETE SET NULL,
|
|
||||||
subject_raw TEXT,
|
|
||||||
subject_norm TEXT,
|
|
||||||
sender TEXT,
|
|
||||||
sent_iso TEXT,
|
|
||||||
body TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
CUR.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS useful_info_attachment (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
path TEXT UNIQUE,
|
|
||||||
text TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
CUR.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS useful_info_email_attachment (
|
|
||||||
email_id BIGINT NOT NULL REFERENCES useful_info_email(id) ON DELETE CASCADE,
|
|
||||||
attachment_id BIGINT NOT NULL REFERENCES useful_info_attachment(id) ON DELETE CASCADE,
|
|
||||||
PRIMARY KEY (email_id, attachment_id)
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
CON.commit()
|
CON.commit()
|
||||||
finally:
|
finally:
|
||||||
CUR.close(); CON.close()
|
CUR.close(); CON.close()
|
||||||
|
|
@ -922,81 +888,6 @@ def insert_usefulinfo_record(parsed):
|
||||||
return summary_id
|
return summary_id
|
||||||
|
|
||||||
|
|
||||||
def insert_usefulinfo_email(subject_raw, subject_norm, sender, sent_iso, body, attachments=None, summary_id=None):
|
|
||||||
"""Insert an original email and any attachments.
|
|
||||||
attachments: list of dicts like {'path': str, 'text': str or None}
|
|
||||||
summary_id: optional FK to useful_info_summary; can be None and linked later.
|
|
||||||
Returns email_id.
|
|
||||||
"""
|
|
||||||
attachments = attachments or []
|
|
||||||
CON, CUR = db()
|
|
||||||
email_id = None
|
|
||||||
try:
|
|
||||||
CUR.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO useful_info_email (summary_id, subject_raw, subject_norm, sender, sent_iso, body)
|
|
||||||
VALUES (%s, %s, %s, %s, %s, %s)
|
|
||||||
RETURNING id
|
|
||||||
""",
|
|
||||||
(summary_id, subject_raw, subject_norm, sender, sent_iso, body)
|
|
||||||
)
|
|
||||||
row = CUR.fetchone()
|
|
||||||
email_id = row[0] if row else None
|
|
||||||
|
|
||||||
for att in attachments:
|
|
||||||
path = (att or {}).get('path')
|
|
||||||
text = (att or {}).get('text')
|
|
||||||
if not path:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
CUR.execute("SAVEPOINT sp_att")
|
|
||||||
CUR.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO useful_info_attachment (path, text)
|
|
||||||
VALUES (%s, %s)
|
|
||||||
ON CONFLICT (path) DO UPDATE SET text = COALESCE(EXCLUDED.text, useful_info_attachment.text)
|
|
||||||
RETURNING id
|
|
||||||
""",
|
|
||||||
(path, text)
|
|
||||||
)
|
|
||||||
arow = CUR.fetchone()
|
|
||||||
att_id = arow[0] if arow else None
|
|
||||||
if email_id and att_id:
|
|
||||||
CUR.execute(
|
|
||||||
"INSERT INTO useful_info_email_attachment (email_id, attachment_id) VALUES (%s, %s) ON CONFLICT DO NOTHING",
|
|
||||||
(email_id, att_id)
|
|
||||||
)
|
|
||||||
CUR.execute("RELEASE SAVEPOINT sp_att")
|
|
||||||
except Exception as ex_a:
|
|
||||||
try:
|
|
||||||
CUR.execute("ROLLBACK TO SAVEPOINT sp_att")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
print('[usefulinfo][attachment-insert-failed]', path, str(ex_a))
|
|
||||||
|
|
||||||
CON.commit()
|
|
||||||
finally:
|
|
||||||
CUR.close(); CON.close()
|
|
||||||
return email_id
|
|
||||||
|
|
||||||
|
|
||||||
def link_emails_to_summary(subject_norm, summary_id):
|
|
||||||
"""Link any emails with the given normalized subject to the provided summary.
|
|
||||||
Returns number of rows updated.
|
|
||||||
"""
|
|
||||||
CON, CUR = db()
|
|
||||||
try:
|
|
||||||
CUR.execute(
|
|
||||||
"UPDATE useful_info_email SET summary_id=%s WHERE summary_id IS NULL AND subject_norm=%s",
|
|
||||||
(summary_id, subject_norm)
|
|
||||||
)
|
|
||||||
n = CUR.rowcount
|
|
||||||
CON.commit()
|
|
||||||
return n
|
|
||||||
finally:
|
|
||||||
CUR.close(); CON.close()
|
|
||||||
|
|
||||||
|
|
||||||
## Migration/diagnostic helpers removed per request. Assume a clean schema created by init_usefulinfo_schema.
|
## Migration/diagnostic helpers removed per request. Assume a clean schema created by init_usefulinfo_schema.
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
303
server.py
303
server.py
|
|
@ -1,11 +1,11 @@
|
||||||
import json, codecs, re, markdown, os, pypandoc, striprtf, sqlite3, random, urllib
|
import json, codecs, re, markdown, os, pypandoc, striprtf, sqlite3, random, urllib
|
||||||
import subprocess, html, time
|
import subprocess, html, time
|
||||||
from markdownify import markdownify as md
|
from markdownify import markdownify as md
|
||||||
from striprtf.striprtf import rtf_to_text
|
from striprtf.striprtf import rtf_to_text
|
||||||
from flask import Flask, render_template, Response, jsonify, request
|
from flask import render_template, Response
|
||||||
from flask import send_from_directory
|
from flask import send_from_directory
|
||||||
import hashlib, funcy, platform, requests
|
import hashlib, funcy, platform, requests
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
#from orgpython import to_html
|
#from orgpython import to_html
|
||||||
|
|
||||||
|
|
@ -25,7 +25,7 @@ datafile2 = "cache/datafile.txt"
|
||||||
|
|
||||||
|
|
||||||
LECPATH = "/media/hd2/peter_home_offload/lecture/"
|
LECPATH = "/media/hd2/peter_home_offload/lecture/"
|
||||||
host = 'http://deep1:5000'
|
host = 'http://192.168.1.6:5000'
|
||||||
news_path = '/media/hd2/peter_home/Documents/scripts/browser/'
|
news_path = '/media/hd2/peter_home/Documents/scripts/browser/'
|
||||||
writing_path = '/media/hd2/peter_home/Documents/writing/'
|
writing_path = '/media/hd2/peter_home/Documents/writing/'
|
||||||
img_path = '/media/hd2/peter_home/Documents/writing_img/'
|
img_path = '/media/hd2/peter_home/Documents/writing_img/'
|
||||||
|
|
@ -49,8 +49,90 @@ if this_host == 'ROGDESKTOP':
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
import sys
|
import paho.mqtt.client as mqtt
|
||||||
app = Flask(__name__, template_folder='templates')
|
|
||||||
|
|
||||||
|
#################################################################################################################
|
||||||
|
#################################################################################################################
|
||||||
|
######
|
||||||
|
###### mqtt
|
||||||
|
######
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
client = 0
|
||||||
|
|
||||||
|
mqtt_offline = 1
|
||||||
|
|
||||||
|
mqtt_time = 0.1
|
||||||
|
|
||||||
|
|
||||||
|
def mqtt_loop():
|
||||||
|
while 1:
|
||||||
|
if client and not mqtt_offline:
|
||||||
|
client.loop(mqtt_time)
|
||||||
|
|
||||||
|
|
||||||
|
# called when MQTT server connects
|
||||||
|
def on_connect(client, userdata, flags, rc):
|
||||||
|
print("Connected with result code "+str(rc))
|
||||||
|
client.subscribe("local/#")
|
||||||
|
|
||||||
|
# The callback for when a PUBLISH message is received from the server.
|
||||||
|
def on_message(client, userdata, msg):
|
||||||
|
now = datetime.now().strftime('%Y %m %d %H %M')
|
||||||
|
print(" %s mqtt msg: %s data: %s" % (now, msg.topic, msg.payload.decode()))
|
||||||
|
|
||||||
|
|
||||||
|
if 0:
|
||||||
|
while(mqtt_offline):
|
||||||
|
try:
|
||||||
|
client = mqtt.Client()
|
||||||
|
client.on_connect = on_connect
|
||||||
|
client.on_message = on_message
|
||||||
|
|
||||||
|
client.connect("192.168.1.6", 1883, 60)
|
||||||
|
|
||||||
|
mqtt_offline = 0
|
||||||
|
|
||||||
|
except OSError as oe:
|
||||||
|
print('no internet? try again in 5 seconds.')
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def displaypi_on():
|
||||||
|
global client
|
||||||
|
msg = 'local/frame_on'
|
||||||
|
client.publish(msg, 'webserver')
|
||||||
|
print("sent %s" % msg)
|
||||||
|
|
||||||
|
def displaypi_off():
|
||||||
|
global client
|
||||||
|
msg = 'local/frame_off'
|
||||||
|
client.publish(msg, 'webserver')
|
||||||
|
print("sent %s" % msg)
|
||||||
|
|
||||||
|
def desklight():
|
||||||
|
global client
|
||||||
|
|
||||||
|
msg = 'local/peter/desklamps'
|
||||||
|
client.publish(msg, 'webserver')
|
||||||
|
print("sent %s" % msg)
|
||||||
|
|
||||||
|
def clearscreens():
|
||||||
|
global client
|
||||||
|
|
||||||
|
msg = 'local/clearscreens'
|
||||||
|
client.publish(msg, 'webserver')
|
||||||
|
print("sent %s" % msg)
|
||||||
|
|
||||||
|
def screenoff():
|
||||||
|
global client
|
||||||
|
|
||||||
|
msg = 'local/peter/monitors'
|
||||||
|
client.publish(msg, 'webserver')
|
||||||
|
print("sent %s" % msg)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -99,138 +181,39 @@ def tag(x,y): return "<%s>%s</%s>" % (x,y,x)
|
||||||
def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
|
def tagc(x,c,y): return '<%s class="%s">%s</%s>' % (x,c,y,x)
|
||||||
def a(t,h): return '<a href="%s">%s</a>' % (h,t)
|
def a(t,h): return '<a href="%s">%s</a>' % (h,t)
|
||||||
|
|
||||||
@app.route('/')
|
def homepage():
|
||||||
def homepage():
|
#output = pypandoc.convert_file(writing_path+fname, 'html', format="rst")
|
||||||
return tag('h1','Canvas Tools') + br + \
|
return tag('h1','This is my server.') + "<br />" + \
|
||||||
a('Useful Emails','/useful-info') + br + br + \
|
a('Toggle light','/light') + br + br + \
|
||||||
a('Reload server','/rl') + br + \
|
a('Clear screens','/clearscreens') + br + br + \
|
||||||
a('Shut down','/sd')
|
a('Toggle monitors','/screensoff') + br + br + \
|
||||||
|
a('Pi frame on','/displaypi/on') + br + br + \
|
||||||
@app.route('/useful-info')
|
a('Pi frame off','/displaypi/off') + br + br + \
|
||||||
def useful_info_page():
|
a('Knowledge Base','/x/writing/index') + br + \
|
||||||
return render_template('useful_info.html')
|
a('Markdown -> HTML converter','/md') + br + \
|
||||||
|
a('Gav web mirror','/mirror') + br + \
|
||||||
@app.route('/useful-info/<tag>')
|
a('Graph of users','/x/user/1') + "<br />" + \
|
||||||
def useful_info_page_with_tag(tag):
|
a('Courses in a dept','/x/dept/csis') + "<br />" + \
|
||||||
# Same template; Vue reads the tag from the URL path
|
a('People in a course','/x/roster/10633') + br + br + \
|
||||||
return render_template('useful_info.html')
|
a('Summarize courses a user has been seen in','/x/user_course_history_summary/9964') + br + br + \
|
||||||
|
a('Peters Lecture Series','/lectures') + br + br + \
|
||||||
@app.route('/api/useful-info')
|
a('Reload server','/rl') + "<br />" + \
|
||||||
def useful_info_api():
|
a('want to shut down?','/sd') + "<br />" + \
|
||||||
# Filters: start, end (ISO date), tags (comma-separated)
|
a('', '') + br
|
||||||
start = request.args.get('start') or None
|
|
||||||
end = request.args.get('end') or None
|
|
||||||
tags = request.args.get('tags') or ''
|
|
||||||
tag_list = [t.strip() for t in tags.split(',') if t.strip()]
|
|
||||||
try:
|
|
||||||
from localcache2 import db
|
|
||||||
CON, CUR = db()
|
|
||||||
try:
|
|
||||||
params = []
|
|
||||||
where = []
|
|
||||||
if start:
|
|
||||||
where.append('s.created_at >= %s')
|
|
||||||
params.append(start)
|
|
||||||
if end:
|
|
||||||
where.append('s.created_at <= %s')
|
|
||||||
params.append(end)
|
|
||||||
where_sql = ('WHERE ' + ' AND '.join(where)) if where else ''
|
|
||||||
CUR.execute(
|
|
||||||
f"""
|
|
||||||
SELECT s.id, s.date_label, s.short_text, s.summary_text,
|
|
||||||
COALESCE(array_agg(DISTINCT t.name) FILTER (WHERE t.name IS NOT NULL), '{{}}') AS tags,
|
|
||||||
COALESCE(array_agg(DISTINCT a.path) FILTER (WHERE a.path IS NOT NULL), '{{}}') AS attachments
|
|
||||||
FROM useful_info_summary s
|
|
||||||
LEFT JOIN useful_info_summary_tag st ON st.summary_id=s.id
|
|
||||||
LEFT JOIN useful_info_tag t ON t.id=st.tag_id
|
|
||||||
LEFT JOIN useful_info_email e ON e.summary_id = s.id
|
|
||||||
LEFT JOIN useful_info_email_attachment ea ON ea.email_id = e.id
|
|
||||||
LEFT JOIN useful_info_attachment a ON a.id = ea.attachment_id
|
|
||||||
{where_sql}
|
|
||||||
GROUP BY s.id
|
|
||||||
ORDER BY to_date('01/' || s.date_label, 'DD/MM/YY') DESC NULLS LAST, s.created_at DESC
|
|
||||||
""",
|
|
||||||
tuple(params)
|
|
||||||
)
|
|
||||||
rows = CUR.fetchall()
|
|
||||||
# Filter by tags in python (any-match)
|
|
||||||
out = []
|
|
||||||
for r in rows:
|
|
||||||
rid, dlabel, short, summary, tgs, atts = r[0], r[1], r[2], r[3], r[4] or [], r[5] or []
|
|
||||||
tgs2 = list(tgs)
|
|
||||||
atts2 = list(atts)
|
|
||||||
if tag_list:
|
|
||||||
if not any(t in tgs2 for t in tag_list):
|
|
||||||
continue
|
|
||||||
out.append({'id': rid, 'date': dlabel, 'title': short, 'summary': summary, 'tags': tgs2, 'attachments': atts2})
|
|
||||||
return jsonify(out)
|
|
||||||
finally:
|
|
||||||
CUR.close(); CON.close()
|
|
||||||
except Exception as e:
|
|
||||||
return jsonify({'error': str(e)}), 500
|
|
||||||
|
|
||||||
@app.route('/data/<path:path>')
|
|
||||||
def send_cachedata(path):
|
|
||||||
# serve files under cache
|
|
||||||
return send_from_directory('cache', path)
|
|
||||||
|
|
||||||
@app.route('/api/useful-info/tag/<tag>')
|
|
||||||
def useful_info_by_tag(tag):
|
|
||||||
# Convenience endpoint to filter by a single tag
|
|
||||||
# Delegates to the main handler by injecting the tag query param
|
|
||||||
args = request.args.to_dict(flat=True)
|
|
||||||
args['tags'] = tag
|
|
||||||
with app.test_request_context('/api/useful-info', query_string=args):
|
|
||||||
return useful_info_api()
|
|
||||||
|
|
||||||
|
|
||||||
def _shutdown_server():
|
|
||||||
func = request.environ.get('werkzeug.server.shutdown')
|
|
||||||
if func is None:
|
|
||||||
raise RuntimeError('Not running with the Werkzeug Server')
|
|
||||||
func()
|
|
||||||
|
|
||||||
@app.route('/sd')
|
|
||||||
def shutdown():
|
|
||||||
try:
|
|
||||||
_shutdown_server()
|
|
||||||
return 'Server shutting down...'
|
|
||||||
except Exception as e:
|
|
||||||
return f'Error shutting down: {e}', 500
|
|
||||||
|
|
||||||
@app.route('/rl')
|
|
||||||
def reload_self():
|
|
||||||
# Attempt to restart the process in-place
|
|
||||||
try:
|
|
||||||
python = sys.executable
|
|
||||||
os.execv(python, [python] + sys.argv)
|
|
||||||
except Exception as e:
|
|
||||||
return f'Error reloading: {e}', 500
|
|
||||||
return 'Reloading...'
|
|
||||||
|
|
||||||
@app.route('/health')
|
|
||||||
def health():
|
|
||||||
return jsonify({'app': 'server.py', 'status': 'ok'}), 200
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
host = os.environ.get('HOST', '0.0.0.0')
|
|
||||||
port = int(os.environ.get('PORT', '5000'))
|
|
||||||
app.run(host=host, port=port, debug=True)
|
|
||||||
|
|
||||||
def orgline(L):
|
def orgline(L):
|
||||||
L.strip()
|
L.strip()
|
||||||
if re.search(r"^\s*$", L): return ""
|
if re.search("^\s*$", L): return ""
|
||||||
|
|
||||||
a = re.search( r'^\*\s(.*)$', L)
|
a = re.search( '^\*\s(.*)$', L)
|
||||||
if a: return "<h2>%s</h2>\n" % a.group(1)
|
if a: return "<h2>%s</h2>\n" % a.group(1)
|
||||||
|
|
||||||
b = re.search( r'TODO\s\[\#A\](.*)$', L)
|
b = re.search( 'TODO\s\[\#A\](.*)$', L)
|
||||||
if b: return "<b><i>Todo - Priority 1</i>: %s</b>" % b.group(1) + br + nl
|
if b: return "<b><i>Todo - Priority 1</i>: %s</b>" % b.group(1) + br + nl
|
||||||
|
|
||||||
d = re.search( r'^\*\*\*\s(.*)$', L)
|
d = re.search( '^\*\*\*\s(.*)$', L)
|
||||||
if d: return d.group(1)
|
if d: return d.group(1)
|
||||||
d = re.search( r'^\*\*\s(.*)$', L)
|
d = re.search( '^\*\*\s(.*)$', L)
|
||||||
if d: L = d.group(1)
|
if d: L = d.group(1)
|
||||||
|
|
||||||
return L + br + nl
|
return L + br + nl
|
||||||
|
|
@ -299,31 +282,51 @@ def writing(fname):
|
||||||
###### kiosk display
|
###### kiosk display
|
||||||
######
|
######
|
||||||
|
|
||||||
def dashboard():
|
def dashboard():
|
||||||
return 'kiosk moved'
|
return open('static/slides.html','r').read() # tag('h1','Dashboard') + br + a('home', '/')
|
||||||
|
|
||||||
def dash():
|
def dash():
|
||||||
return 'kiosk moved'
|
return open('static/dashboard.html','r').read() # tag('h1','Dashboard') + br + a('home', '/')
|
||||||
|
|
||||||
def mycalendar():
|
def mycalendar():
|
||||||
return 'kiosk moved'
|
ics = 'https://calendar.google.com/calendar/u/0?cid=cGV0ZXIuaG93ZWxsQGdtYWlsLmNvbQ'
|
||||||
|
return dash()
|
||||||
|
|
||||||
def most_recent_file_of(target, folder):
|
def most_recent_file_of( target, folder ):
|
||||||
return ''
|
|
||||||
|
|
||||||
def news():
|
def finder(st):
|
||||||
return ''
|
return re.search(target,st)
|
||||||
|
|
||||||
def randPic():
|
all = os.listdir(folder)
|
||||||
return ''
|
all.sort(key=lambda x: os.stat(os.path.join(folder,x)).st_mtime)
|
||||||
|
all.reverse()
|
||||||
|
all = list(funcy.filter( finder, all ))
|
||||||
|
|
||||||
|
print("file list is: " + str(all))
|
||||||
|
if not all:
|
||||||
|
return ''
|
||||||
|
return all[0]
|
||||||
|
|
||||||
|
def news():
|
||||||
|
folder = most_recent_file_of( r'\d\d\d\d\d\d\d\d', news_path )
|
||||||
|
pics = os.listdir( news_path + folder )
|
||||||
|
return '/static/news/' + folder + '/' + random.choice(pics)
|
||||||
|
|
||||||
|
def randPic():
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
if now.minute < 15:
|
||||||
|
return news()
|
||||||
|
return '/static/images/' + random.choice(os.listdir('static/images'))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def do_img_crop(im):
|
def do_img_crop(im):
|
||||||
return ''
|
result = subprocess.run(['ls', '-l'], stdout=subprocess.PIPE)
|
||||||
|
rr = result.stdout.decode('utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -447,7 +450,7 @@ def lectures():
|
||||||
#print(f)
|
#print(f)
|
||||||
with tag('item'):
|
with tag('item'):
|
||||||
name = f.split('.')[0]
|
name = f.split('.')[0]
|
||||||
ff = re.sub(r'\s','%20',f)
|
ff = re.sub('\s','%20',f)
|
||||||
with tag('title'): text(name)
|
with tag('title'): text(name)
|
||||||
with tag('guid'): text(f)
|
with tag('guid'): text(f)
|
||||||
b = os.path.getsize(LECPATH+f)
|
b = os.path.getsize(LECPATH+f)
|
||||||
|
|
@ -518,7 +521,7 @@ def staff_dir(search=''):
|
||||||
|
|
||||||
|
|
||||||
def find_goo(n):
|
def find_goo(n):
|
||||||
g = re.search(r'00(\d{6})', n)
|
g = re.search('00(\d\d\d\d\d\d)', n)
|
||||||
|
|
||||||
if g:
|
if g:
|
||||||
return g.groups()[0]
|
return g.groups()[0]
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue