cleaning up roster fetch. log.

pipelines.py

@@ -1216,6 +1216,9 @@ def fetch_current_rosters():
     with pysftp.Connection(instructure_url,username=instructure_username, private_key=instructure_private_key,cnopts=cnopts) as sftp:
         sftp.chdir('SIS')
         files = sftp.listdir()
+        ff = open('cache/pipeline.log.txt','a')
+        ff.write(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + "\n")
+        ff.write("--> %s I see these files at instructure ftp site: \n" % dt_label )
         print("--> %s I see these files at instructure ftp site: " % dt_label )
         [print("   %s" % f) for f in files]
         i = 0
@@ -1243,6 +1246,7 @@ def fetch_current_rosters():
             except:
                 print(' * enrollments.csv not present')
             print('   Saved %i data files in rosters folder.' % i)
+            ff.write( f"   Saved {i} data files: {seen_files}\n")
             
             if i:
                 if 'courses.csv' in seen_files:
@@ -1258,13 +1262,17 @@ def fetch_current_rosters():
                     sem = {'30':'spring', '50':'summer', '70':'fall' }
                     this_sem = sem[ss]
                     print(" -> This semester is: %s, %s" % (this_sem,year))
-                if len(seen_files)==3:
+                
+                    #if len(seen_files)==3:
+
                     print(' -> %s building data file...' % dt_label)
                     convert_roster_files(this_sem,year,dt_label)
                     print(' -> moving files...')
+                    ff.write( f"   Moved files to folder: {this_sem} {year} {dt_label}\n")
                     move_to_folder(this_sem,year,dt_label,seen_files)
                 else:
                     print(" * No courses file. Not moving files.")
+                    ff.write( f" * No courses file. Not moving files.\n")
         else:
             print("--> Don't see files.")
     sftp.close()

semesters.py

@@ -38,6 +38,16 @@ for s in list(zip(standard,code,begin)):
     semester_list[cl] = sem_record
 
 
+# TODO remove original in localcache.py
+def to_sis_sem(s):
+    season = s[0:2]
+    year = "20" + s[2:5]
+    a = {'sp':'30','su':'50','fa':'70'}
+    season = a[season]
+    return year+season
+
+
+
 # print(json.dumps(semester_list,indent=2))