import util from pampy import match, _ import json, pypandoc, requests,json,os,re, bisect, csv, codecs import sortedcontainers as sc from collections import defaultdict from toolz.itertoolz import groupby import pdb pat8020 = [] """ (programs) entityType entityTitle status proposalType sectionName lastUpdated lastUpdatedBy fieldName displayName lookUpDisplay fieldValue instanceSortOrder lookUpDataset (array of dicts, each has keys: name, value, and corresponding values.) subsections or fields (arrays) - ignore for now just takem in order (courses) same as above? html values: markdown convert? """ """pat8020.append( {"displayName": _} ) pat8020.append( {"entityType": _} ) pat8020.append( {"entityTitle": _} ) pat8020.append( {"lookUpDisplay": _} ) pat8020.append( {"fieldValue": _} ) """ err = "no error\n" def to_md(s): output = pypandoc.convert_text(s,'md',format='html') return output def print_return(x): print('got a hit') print() return x def cq_8020(root,indent=0): ret = [] idt = " " * indent try: m = match( root, {"attributes": { "fieldName": "Department" }, "lookUpDisplay": _ }, lambda x: idt + "Department: " + x.strip() if x.strip() else "", {"attributes": { "fieldName": "Division" }, "lookUpDisplay": _ }, lambda x: idt + "Division: " + x.strip() if x.strip() else "", {"attributes": { "fieldName": "Discipline" }, "lookUpDisplay": _ }, lambda x: idt + "Discipline: " + x.strip() if x.strip() else "", {"attributes": { "fieldName": "Program Title" }, "lookUpDisplay": _ }, lambda x: idt + "Program Title: " + x.strip() if x.strip() else "", {"attributes": { "fieldName": "Outcome" }, "fieldValue": _ }, lambda x: idt + "Outcome: " + x.strip() if x.strip() else "", {"attributes": { "fieldName": "Award Type" }, "lookUpDisplay": _ }, lambda x: idt + "Award Type: " + x, {"attributes": { "fieldName": "Course" }, "lookUpDisplay": _ }, lambda x: idt + "Course: " + x.strip() if x.strip() else "", {"attributes": { "fieldName": "Description" }, "fieldValue": _ }, lambda x: idt + "Description: " + to_md(x), {"attributes": { "fieldName": "Justification" }, "fieldValue": _ }, lambda x: idt + "Justification: " + x.strip() if x.strip() else "", {"fieldName": _}, lambda x: idt + "field name: " + x.strip() if x.strip() else "", {"fieldValue": _}, lambda x: idt + "field value: " + x.strip() if x.strip() else "", #{"entityType": _}, lambda x: idt + "entityType: " + x, {"entityTitle": _}, lambda x: idt + "entityTitle: " + x.strip() if x.strip() else "", {"lookUpDisplay": _}, lambda x: idt + "lookUpDisplay: " + to_md(x.strip()) if x.strip() else "", # Units { "name": "Max", "value": _ }, lambda x: "%sMax: %s" % (idt,x), { "name": "Min", "value": _ }, lambda x: "%sMin: %s" % (idt,x), { "name": "Text", "value": _ }, lambda x: "%sText: %s" % (idt,x), default=False ) if m: print('case 1: ' + str(m) ) ret.append(m) except Exception as e: m = 0 pass #print("GOT EXCEPTION.") #err += str(e) if (not m) and type(root) == type( {} ): """ for K,V in list(root.items()): print( [K,V]) m = match( [K,V], ["lookUpDisplay", _ ], lambda x: idt + "lookup display: " + to_md(str(x).strip()) if str(x).strip() else "", ["fieldName", _ ], lambda x: idt + "field name: " + x, ["fieldValue", _ ], lambda x: idt + "field value: " + to_md(str(x).strip()) if str(x).strip() else "", ["entityType", _ ], lambda x: idt + "entity type: " + x, ["entityTitle", _ ], lambda x: idt + "entity title: " + x, ["displayName", _ ], lambda x: idt + "display name: " + x, ["sectionSortOrder", _ ], lambda x: idt + "section sort order: " + str(x), default=False) if m: print('case 2 ' + str(m)) ret.append(m) #else: """ for V in root.values(): m = cq_8020(V,indent+2) if m: print('case 4 ' + str(m)) ret.extend(m) elif (not m) and type(root) == type([]): for V in root: m = cq_8020(V,indent+2) if m: print('case 3') ret.extend(m) return ret def cq_8021(root,indent=0): ret = [] idt = " " * indent m = 0 try: m = match( root, {"attributes": { "fieldName": "Department" }, "lookUpDisplay": _ }, lambda x: {"key":"Department", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Division" }, "lookUpDisplay": _ }, lambda x: {"key":"Division", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Discipline" }, "lookUpDisplay": _ }, lambda x: {"key":"Discipline", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Program Title" }, "lookUpDisplay": _ }, lambda x: {"key":"Program Title", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Outcome" }, "fieldValue": _ }, lambda x: {"key":"Outcome", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Award Type" }, "lookUpDisplay": _ }, lambda x: {"key":"Award Type", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Course" }, "lookUpDisplay": _ }, lambda x: {"key":"Course", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Description" }, "fieldValue": _ }, lambda x: {"key":"Description", "value": to_md(x.strip()) } if x.strip() else 0, {"attributes": { "fieldName": "Justification" }, "fieldValue": _ }, lambda x: {"key":"Justification", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Assessment" }, "fieldValue": _ }, lambda x: {"key":"Assessment", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Disk Name" }, "fieldValue": _ }, lambda x: {"key":"Disk Name", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Attached File Name" }, "fieldValue": _ }, lambda x: {"key":"Attached File Name", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Title" }, "fieldValue": _ }, lambda x: {"key":"Title", "value": x.strip() } if x.strip() else 0, {"fieldName": _}, lambda x: {"key": x.strip()} if x.strip() else 0, {"fieldValue": _}, lambda x: {"value": x.strip()} if x.strip() else 0, {"entityType": _}, lambda x: {"key": "Type", "value": x.strip()} if x.strip() else 0, {"entityTitle": _}, lambda x: {"key": "Title", "value": x.strip()} if x.strip() else 0, {"lookUpDisplay": _}, lambda x: {"value": x.strip()} if x.strip() else 0, # Units { "name": "Max", "value": _ }, lambda x: {"key": "max", "value": x.strip()} if x.strip() else 0, { "name": "Min", "value": _ }, lambda x: {"key": "min", "value": x.strip()} if x.strip() else 0, { "name": "Text", "value": _ }, lambda x: {"value": x.strip()} if x.strip() else 0, default=False ) if m: print('case 1: ' + str(m) ) ret.append(m) except Exception as e: m = 0 pass #print("GOT EXCEPTION.") #err += str(e) if (not m) and type(root) == type( {} ): """ for K,V in list(root.items()): print( [K,V]) m = match( [K,V], ["lookUpDisplay", _ ], lambda x: idt + "lookup display: " + to_md(str(x).strip()) if str(x).strip() else "", ["fieldName", _ ], lambda x: idt + "field name: " + x, ["fieldValue", _ ], lambda x: idt + "field value: " + to_md(str(x).strip()) if str(x).strip() else "", ["entityType", _ ], lambda x: idt + "entity type: " + x, ["entityTitle", _ ], lambda x: idt + "entity title: " + x, ["displayName", _ ], lambda x: idt + "display name: " + x, ["sectionSortOrder", _ ], lambda x: idt + "section sort order: " + str(x), default=False) if m: print('case 2 ' + str(m)) ret.append(m) #else: """ for V in root.values(): m = cq_8021(V,indent+2) if m: print('case 4 ' + str(m)) ret.extend(m) #for mm in m: # if 'key' in mm and 'value' in mm: # ret.extend(mm) elif (not m) and type(root) == type([]): for V in root: m = cq_8021(V,indent+2) if m: print('case 3') ret.extend(m) return ret def cq_8021_start(): root = json.loads( open('cache/programs/programs_1.txt','r').read()) outt = open('cache/test_prog8020.txt','w') outt_err = open('cache/test_prog8020err.txt','w') result = cq_8021(root) outt.write( json.dumps(result, indent=2)) #outt_err.write( err ) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ## ## In this attempt I try to keep the data structure intact, but swapping in parts I recognize for a ## more compact version. # ## Recursively do this.... ## ## As I elaborate on it, the non-swapped parts will hopefully stand out more and more, and I can ## track down all the problems. ## def cq_8022(root,indent=0): ret = [] idt = " " * indent m = 0 try: m = match( root, # Clear empties { "attributes": { "fieldName": _ }, "fieldValue": "" }, "NULL", { "attributes": { "fieldName": _ }, "lookUpDisplay": "", "fieldValue": _ }, lambda x,y: {"key":x,"value":y}, { "attributes": { "fieldName": _ }, "lookUpDisplay": _, "fieldValue": "" }, lambda x,y: {"key":x,"value":y}, {"attributes": { "fieldName": "Exception Identifier" }, "fieldValue": _ }, lambda x: {"key":"Exception ID", "value": x}, {"attributes": { "fieldName": "Department" }, "lookUpDisplay": _ }, lambda x: {"key":"Department", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Division" }, "lookUpDisplay": _ }, lambda x: {"key":"Division", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Discipline" }, "lookUpDisplay": _ }, lambda x: {"key":"Discipline", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Program Title" }, "fieldValue": _ }, lambda x: {"key":"Program Title", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Outcome" }, "fieldValue": _ }, lambda x: {"key":"Outcome", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Award Type" }, "lookUpDisplay": _ }, lambda x: {"key":"Award Type", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Course" }, "lookUpDisplay": _ }, lambda x: {"key":"Course", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Description" }, "fieldValue": _ }, lambda x: {"key":"Description", "value": to_md(x.strip()) } if x.strip() else 0, {"attributes": { "fieldName": "Justification" }, "fieldValue": _ }, lambda x: {"key":"Justification", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Assessment" }, "fieldValue": "-" }, lambda x: "NULL", {"attributes": { "fieldName": "Assessment" }, "fieldValue": _ }, lambda x: {"key":"Assessment", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Disk Name" }, "fieldValue": _ }, lambda x: {"key":"Disk Name", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Attached File Name" }, "fieldValue": _ }, lambda x: {"key":"Attached File Name", "value": x.strip() } if x.strip() else 0, {"attributes": { "fieldName": "Title" }, "fieldValue": _ }, lambda x: {"key":"Title", "value": x.strip() } if x.strip() else 0, {"entityType": _}, lambda x: {"key": "Type", "value": x.strip()} if x.strip() else 0, {"entityTitle": _}, lambda x: {"key": "Title", "value": x.strip()} if x.strip() else 0, {"lookUpDisplay": _}, lambda x: {"value": x.strip()} if x.strip() else 0, {"attributes": { "fieldName": "Course" }, "lookUpDisplay": _ }, lambda x: {"key": "Course", "value": x.strip()} if x.strip() else 0, # Units { "name": "Max", "value": _ }, lambda x: {"key": "max", "value": x.strip()} if x.strip() else 0, { "name": "Min", "value": _ }, lambda x: {"key": "min", "value": x.strip()} if x.strip() else 0, { "name": "Text", "value": _ }, lambda x: {"value": x.strip()} if x.strip() else 0, # Programs { "attributes": { "fieldName": "Course Block Definition" }, "fieldValue": _ }, lambda x: { "key":"Course block d.", "value": x.strip() }, { "attributes": { "fieldName": "Unit Min" }, "fieldValue": _ }, lambda x: { "key":"Unit min", "value": x }, { "attributes": { "fieldName": "Unit Max" }, "fieldValue": _ }, lambda x: { "key":"Unit max", "value": x }, { "attributes": { "fieldName": "Units Low" }, "fieldValue": _ }, lambda x: { "key":"Units low", "value": x }, { "attributes": { "fieldName": "Units High" }, "fieldValue": _ }, lambda x: { "key":"Units high", "value": x }, { "attributes": { "fieldName": "Override Unit Calculation" }, "fieldValue": _ }, lambda x: { "key":"override unit calc", "value": x }, { "attributes": { "fieldName": "Override Defalut Unit Calculations" }, "fieldValue": _ }, lambda x: { "key":"override default unit calc", "value": x }, { "attributes_unchanged": { "sectionOrInstance": "section" }, "subsections_unchanged": [], "fields": [] }, lambda x: "NULL" , { "attributes": { "sectionOrInstance": "section" }, "subsections": [], "fields": [] }, lambda x: "NULL" , { "attributes_unchanged": { "sectionOrInstance": "section" }, "subsections": [], "fields": [] }, lambda x: "NULL" , { "attributes": { "sectionName": "[Discipline and Course chained combo]", "sectionSortOrder": _ }, "fields": _ }, lambda w,x: { "sortOrder":w, "key":"course", "value": x }, # #{ "key": "_", "value": "_" } default=False ) if m: print(' '*indent + 'case 1: ' + str(m) ) return 1,m except Exception as e: m = 0 if (not m) and type(root) == type( [] ): # an array that only has DICTS, which only have 2 (or 3) keys, key,value,(sortOrder) # we want to collapse it into a dict. this_change = 0 maybe_new_dict = {} is_collapsable = 1 for z in root: if type(z)==type({}): for ea in list(z.keys()): if not ea in ['sortOrder','key','value']: is_collapsable = 0 else: is_collapsable = 0 if not is_collapsable: break if is_collapsable: kk = list(z.keys()) if 'sortOrder' in kk and 'key' in kk and 'value' in kk: maybe_new_dict[str(z['sortOrder'])+'_'+z['key']] = z['value'] elif 'key' in kk and 'value' in kk: maybe_new_dict[z['key']] = z['value'] else: maybe_new_dict['value'] = z['value'] if is_collapsable: return 1,maybe_new_dict my_list = [] for x in root: changed, m = cq_8022(x, indent+1) this_change += changed if changed: if m != "NULL": my_list.append(m) print(' '*indent + 'case 5: ' +str(m)) else: my_list.append(x) if this_change: changed2,m2 = cq_8022(my_list,indent+1) return changed2+this_change , m2 if (not m) and type(root) == type( {} ): my_d_clone = {} this_change = 0 for k,V in root.items(): changed,m = cq_8022(V,indent+1) this_change += changed if this_change: print(' '*indent + 'case 4: ' +str(m)) my_d_clone[k] = m else: #my_d_clone[k+'_unchanged'] = V my_d_clone[k] = V if this_change: changed2,m2 = cq_8022(my_d_clone,indent+1) return changed2+this_change , m2 return 0,root """if not changed and k == "fields" and type(V) == list: #new_dict = {"err":[] } new_list = [] for item in V: if item == "NULL": continue if type(item) == dict: if len(item.keys())==2 and ("key" in item.keys()) and ("value" in item.keys()): #print("\n" + str(item.keys())) #pdb.set_trace() new_list.append( {"key": item["key"], "value": item["value"] } ) else: changed,m = cq_8022(item, indent+1) this_change += changed if changed: new_list.append(m) else: new_list.append(item) m = new_list this_change += 1 elif (not m) and type(root) == type([]): myclone = [] this_change = 0 for V in root: changed,m = cq_8022(V,indent+1) this_change += changed if m: print('case 3 (' + str(indent) + ') ' + str(m)) myclone.append(m) else: myclone.append(V) if this_change: return cq_8022(myclone,indent+1) return this_change,myclone""" def cq_8022_start(): root = json.loads( open('cache/programs/programs_demo.txt','r').read()) outt = open('cache/test_prog8020.txt','w') outt_err = open('cache/test_prog8020err.txt','w') #changed = 1 #while changed: changed,result = cq_8022(root) outt.write( json.dumps(result, indent=2)) #outt_err.write( err ) # # # # # # # # # # # # # # # # # # May 2021 def sortable_class(li): dept = li[1] rest = '' # little error case here n = re.match(r'([A-Za-z]+)(\d+)',li[2]) if n: num = int(n.group(2)) else: m = re.match(r'(\d+)([A-Za-z]+)$',li[2]) if m: num = int(m.group(1)) rest = m.group(2) else: num = int(li[2]) if num < 10: num = '00'+str(num) elif num < 100: num = '0'+str(num) else: num = str(num) return dept+num+rest def c_name(c): delivery = set() units = [] slos = [] hybridPct = '' active = 'Active' id = c['entityMetadata']['entityId'] if c['entityMetadata']['status'] != 'Active': active = 'Inactive' #return () for r in c['entityFormData']['rootSections']: if r['attributes']['sectionName'] == 'Course Description': for ss in r['subsections']: for f in ss['fields']: if f['attributes']['fieldName'] == 'Course Discipline': dept = f['lookUpDisplay'] if f['attributes']['fieldName'] == 'Course Number': num = f['fieldValue'] if f['attributes']['fieldName'] == 'Course Title': title = f['fieldValue'] #print "\n" + title if f['attributes']['fieldName'] == 'Course Description': desc = re.sub(r'\n',' ', f['fieldValue']) if r['attributes']['sectionName'] == 'Units/Hours/Status': for ss in r['subsections']: if ss['attributes']['sectionName'] == '': for f in ss['fields']: if f['attributes']['fieldName'] == 'Minimum Units' and f['fieldValue'] not in units: units.insert(0,f['fieldValue']) if f['attributes']['fieldName'] == 'Maximum Units' and f['fieldValue'] and f['fieldValue'] not in units: units.append(f['fieldValue']) # Newer entered courses have this filled out if r['attributes']['sectionName'] == 'Distance Education Delivery': for ss in r['subsections']: if ss['attributes']['sectionName'] == 'Distance Education Delivery': for ssa in ss['subsections']: for f in ssa['fields']: if f['attributes']['fieldName'] == 'Delivery Method': delivery.add(f['lookUpDisplay']) if ss['attributes']['sectionName'] == "": if ss['fields'][0]['attributes']['fieldName'] == "If this course is Hybrid, what percent is online?": hybridPct = str(ss['fields'][0]['fieldValue']) # Older ones seem to have it this way if r['attributes']['sectionName'] == 'Distance Education': for ss in r['subsections']: for f2 in ss['fields']: if 'fieldName' in f2['attributes'] and f2['attributes']['fieldName'] == 'Methods of Instruction': #print f2['fieldValue'] if f2['fieldValue'] == 'Dist. Ed Internet Delayed': delivery.add('Online') # SLO if r['attributes']['sectionName'] == 'Student Learning Outcomes': for ss in r['subsections']: if 'subsections' in ss: if ss['attributes']['sectionName'] == 'Learning Outcomes': for s3 in ss['subsections']: for ff in s3['fields']: if ff['attributes']['fieldName'] == 'Description': slos.append(ff['fieldValue']) #print ff #[0]['fields']: #print ff['fieldValue'] #for f2 in ss['fields']: # if 'fieldName' in f2['attributes'] and f2['attributes']['fieldName'] == 'Methods of Instruction': # if f2['fieldValue'] == 'Dist. Ed Internet Delayed': # delivery.append('online(x)') if len(units)==1: units.append('') if len(delivery)==0: delivery.add('') u0 = 0 try: u0 = units[0] except: pass u1 = 0 try: u1 = units[2] except: pass return id,dept,num,active,title,u0,u1,'/'.join(delivery),hybridPct,desc,slos def show_classes2020(): pass def show_classes2020_start(): outt = open('cache/test_class2021_all.txt','w') max_active = {} # hold the id of the class if seen. only include the highest id class in main list. used_course = {} # hold the actual course info, the version we'll actually use. slo_by_id = {} # values are a list of slos. slo_by_id_included = {} # just the ids of active or most recent versions. #tmp = codecs.open('cache/course_temp.txt','w','utf-8') for f in os.listdir('cache/courses'): if re.search('classes_',f): print(f) cls = json.loads(open('cache/courses/'+f,'r').read()) for c in cls: dir_data = list(c_name(c)) #tmp.write(str(dir_data) + "\n\n") slo_by_id[dir_data[0]] = dir_data[10] # info = list(map(str,dir_data[:10])) info.append(dir_data[10]) #pdb.set_trace() #print info course_key = sortable_class(info) curqnt_id = int(info[0]) if course_key in max_active: if curqnt_id < max_active[course_key]: continue max_active[course_key] = curqnt_id if course_key in used_course: while course_key in used_course: course_key += '_' used_course[course_key] = info print("\t%s" % course_key) outt.write( json.dumps(info, indent=2)) out2 = open('cache/test_class2021.txt','w') out2.write( json.dumps(used_course, indent=2) ) if __name__ == "__main__": print ('') options = { 1: ['take 1 - programs', cq_8021_start], 2: ['take 2 - programs', cq_8022_start], 3: ['take 1 - classes', show_classes2020_start], } for key in options: print((str(key) + '.\t' + options[key][0])) print('') #resp = eval(input('Choose: ')) resp = input('Choose: ') # Call the function in the options dict options[ int(resp)][1]()