simplified shell scripts and imporved multiple dataset notebook generation

Analysis_Tools/new_flight.sh

@@ -14,17 +14,6 @@ mkdir $1/${PLANE}_combined_$1
 #
 popd
 
-DATE=$(./programs/dMoYr_yyyymmdd.py $1)
-
-cp template_efis.ipynb ${PLANE}\ Data/flights/$1/${DATE}_efis.ipynb
-cp template_ems.ipynb  ${PLANE}\ Data/flights/$1/${DATE}_ems.ipynb
-
-pushd ${PLANE}\ Data/flights/notebooks
-#
-ln -s ../${1}/${DATE}_efis.ipynb .
-ln -s ../${1}/${DATE}_ems.ipynb .
-#
-popd
 
 open ${PLANE}\ Data/flights/$1
 

Analysis_Tools/programs/walk_logs.py

@@ -8,11 +8,13 @@ processes all LOG0*.TXT files
 
 import glob
 import os
+import subprocess
 import argparse
 
 #PLANE="Nnumber"
 PLANE="N72KH"
 
+
 def prepare(date):
     print("splitting logs")
     pathstring="../%s Data/flights/%s/%s_combined_%s/" %(PLANE,date,PLANE,date) # combined log
@@ -36,11 +38,13 @@ def prepare(date):
         os.system(cmd)
  
 def process(date,logs):
+    num_datasets=0
     if logs=='all' or logs=='ems':
         print("processing EMS")
         #sp="../%s Data/flights/%s/%s_EIS_%s/" %(PLANE,date,PLANE,date) # GRT EIS
         pathstring="../%s Data/flights/%s/%s_EMS_%s/" %(PLANE,date,PLANE,date) # Dynon EMS
         fl=glob.glob(pathstring+"LOG0*.TXT")
+        num_datasets=len(fl)
         for f in fl:
             fn=f.split("/")[-1].split(".")[0] # filename
             # s="python ./eis_bin2csv.py '%s%s.TXT' > '%s%s.csv'" % (sp,fn,sp,fn) # GRT EIS
@@ -51,6 +55,7 @@ def process(date,logs):
         print("processing EFIS")
         pathstring="../%s Data/flights/%s/%s_EFIS_%s/" %(PLANE,date,PLANE,date)
         fl=glob.glob(pathstring+"LOG0*.TXT")
+        num_datasets=len(fl)
         for f in fl:
             fn=f.split("/")[-1].split(".")[0] # filename
             cmd="python ./efis2csv.py '%s%s.TXT' > '%s%s.csv'" % (pathstring,fn,pathstring,fn)
@@ -60,6 +65,7 @@ def process(date,logs):
         print("processing GPS")
         pathstring="../%s Data/flights/%s/%s_GPS_%s/" %(PLANE,date,PLANE,date) # path
         fl=glob.glob(pathstring+"LOG0*.TXT") # file list
+        num_datasets=len(fl)
         for f in fl:
             fn=f.split("/")[-1].split(".")[0] # filename
             s="python ./gps_formatter.py '%s%s.TXT' > '%s%s.csv'" % (pathstring,fn,pathstring,fn)
@@ -68,6 +74,34 @@ def process(date,logs):
             #print s
             os.system(cmd)
 
+    return(num_datasets)
+
+
+def create_notebooks(date, num_datasets):
+    FNDATE=subprocess.getoutput("python ./dMoYr_yyyymmdd.py %s" %(date))
+    # if there is more than one dataset, create the right number of notebooks
+    if (num_datasets > 1):
+        for i in range(0,num_datasets):
+            print("data set %d ipynb" %(i))
+            subprocess.getoutput('cp ../template_efis.ipynb "../%s Data/flights/%s/%s-%d_efis.ipynb"' % (PLANE,date,FNDATE,i))
+            subprocess.getoutput('cp ../template_ems.ipynb "../%s Data/flights/%s/%s-%d_ems.ipynb"' % (PLANE,date,FNDATE,i))
+            subprocess.getoutput('sed -i .bak "s/LOGNUM=LOGCOUNT-1/LOGNUM=%d/g" "../%s Data/flights/%s/%s-%d_efis.ipynb"' % (i,PLANE,date,FNDATE,i))
+            subprocess.getoutput('sed -i .bak "s/LOGNUM=LOGCOUNT-1/LOGNUM=%d/g" "../%s Data/flights/%s/%s-%d_ems.ipynb"' % (i,PLANE,date,FNDATE,i))
+            subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s-%d_efis.ipynb"' % (PLANE,date,FNDATE,i))
+            subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s-%d_ems.ipynb"' % (PLANE,date,FNDATE,i))
+    else: # single dataset
+        print("set up ipynb")
+        subprocess.getoutput('cp ../template_efis.ipynb "../%s Data/flights/%s/%s_efis.ipynb"' % (PLANE,date,FNDATE))
+        subprocess.getoutput('cp ../template_ems.ipynb "../%s Data/flights/%s/%s_ems.ipynb"' % (PLANE,date,FNDATE))
+        subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s_efis.ipynb"' % (PLANE,date,FNDATE))
+        subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s_ems.ipynb"' % (PLANE,date,FNDATE))
+
+    
+# link in notebooks folder:
+# pushd ${PLANE}\ Data/flights/notebooks
+# ln -s ../${1}/${DATE}_efis.ipynb .
+# ln -s ../${1}/${DATE}_ems.ipynb .
+# popd
 
 def main():
     parser = argparse.ArgumentParser(description='Process all log files')
@@ -79,7 +113,8 @@ def main():
     args = parser.parse_args()
 
     prepare(args.date)
-    process(args.date,args.logname)
+    num_datasets=process(args.date, args.logname)
+    create_notebooks(args.date, num_datasets)
 
 
 

Analysis_Tools/walk_logs.sh

@@ -9,8 +9,5 @@ pushd ./programs/
 python walk_logs.py $1
 popd
 
-DATE=$(./programs/dMoYr_yyyymmdd.py $1)
 
-jupyter nbconvert --to notebook --execute ${PLANE}\ Data/flights/$1/${DATE}_efis.ipynb
-jupyter nbconvert --to notebook --execute ${PLANE}\ Data/flights/$1/${DATE}_ems.ipynb