ken/airplane-data-logger
Template
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simplified shell scripts and imporved multiple dataset notebook generation

Browse Source
This commit is contained in:
ken
2024-07-21 17:13:42 -07:00
parent 6d974e6553
commit 85f4cc0846
3 changed files with 36 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Analysis_Tools/new_flight.sh
-11
View File
@@ -14,17 +14,6 @@ mkdir $1/${PLANE}_combined_$1
#
popd
DATE=$(./programs/dMoYr_yyyymmdd.py $1)
cp template_efis.ipynb ${PLANE}\ Data/flights/$1/${DATE}_efis.ipynb
cp template_ems.ipynb ${PLANE}\ Data/flights/$1/${DATE}_ems.ipynb
pushd ${PLANE}\ Data/flights/notebooks
#
ln -s ../${1}/${DATE}_efis.ipynb .
ln -s ../${1}/${DATE}_ems.ipynb .
#
popd
open ${PLANE}\ Data/flights/$1
Analysis_Tools/programs/walk_logs.py
+36 -1
View File
@@ -8,11 +8,13 @@ processes all LOG0*.TXT files
import glob
import os
import subprocess
import argparse
#PLANE="Nnumber"
PLANE="N72KH"
def prepare(date):
print("splitting logs")
pathstring="../%s Data/flights/%s/%s_combined_%s/" %(PLANE,date,PLANE,date) # combined log
@@ -36,11 +38,13 @@ def prepare(date):
os.system(cmd)
def process(date,logs):
num_datasets=0
if logs=='all' or logs=='ems':
print("processing EMS")
#sp="../%s Data/flights/%s/%s_EIS_%s/" %(PLANE,date,PLANE,date) # GRT EIS
pathstring="../%s Data/flights/%s/%s_EMS_%s/" %(PLANE,date,PLANE,date) # Dynon EMS
fl=glob.glob(pathstring+"LOG0*.TXT")
num_datasets=len(fl)
for f in fl:
fn=f.split("/")[-1].split(".")[0] # filename
# s="python ./eis_bin2csv.py '%s%s.TXT' > '%s%s.csv'" % (sp,fn,sp,fn) # GRT EIS
@@ -51,6 +55,7 @@ def process(date,logs):
print("processing EFIS")
pathstring="../%s Data/flights/%s/%s_EFIS_%s/" %(PLANE,date,PLANE,date)
fl=glob.glob(pathstring+"LOG0*.TXT")
num_datasets=len(fl)
for f in fl:
fn=f.split("/")[-1].split(".")[0] # filename
cmd="python ./efis2csv.py '%s%s.TXT' > '%s%s.csv'" % (pathstring,fn,pathstring,fn)
@@ -60,6 +65,7 @@ def process(date,logs):
print("processing GPS")
pathstring="../%s Data/flights/%s/%s_GPS_%s/" %(PLANE,date,PLANE,date) # path
fl=glob.glob(pathstring+"LOG0*.TXT") # file list
num_datasets=len(fl)
for f in fl:
fn=f.split("/")[-1].split(".")[0] # filename
s="python ./gps_formatter.py '%s%s.TXT' > '%s%s.csv'" % (pathstring,fn,pathstring,fn)
@@ -68,6 +74,34 @@ def process(date,logs):
#print s
os.system(cmd)
return(num_datasets)
def create_notebooks(date, num_datasets):
FNDATE=subprocess.getoutput("python ./dMoYr_yyyymmdd.py %s" %(date))
# if there is more than one dataset, create the right number of notebooks
if (num_datasets > 1):
for i in range(0,num_datasets):
print("data set %d ipynb" %(i))
subprocess.getoutput('cp ../template_efis.ipynb "../%s Data/flights/%s/%s-%d_efis.ipynb"' % (PLANE,date,FNDATE,i))
subprocess.getoutput('cp ../template_ems.ipynb "../%s Data/flights/%s/%s-%d_ems.ipynb"' % (PLANE,date,FNDATE,i))
subprocess.getoutput('sed -i .bak "s/LOGNUM=LOGCOUNT-1/LOGNUM=%d/g" "../%s Data/flights/%s/%s-%d_efis.ipynb"' % (i,PLANE,date,FNDATE,i))
subprocess.getoutput('sed -i .bak "s/LOGNUM=LOGCOUNT-1/LOGNUM=%d/g" "../%s Data/flights/%s/%s-%d_ems.ipynb"' % (i,PLANE,date,FNDATE,i))
subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s-%d_efis.ipynb"' % (PLANE,date,FNDATE,i))
subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s-%d_ems.ipynb"' % (PLANE,date,FNDATE,i))
else: # single dataset
print("set up ipynb")
subprocess.getoutput('cp ../template_efis.ipynb "../%s Data/flights/%s/%s_efis.ipynb"' % (PLANE,date,FNDATE))
subprocess.getoutput('cp ../template_ems.ipynb "../%s Data/flights/%s/%s_ems.ipynb"' % (PLANE,date,FNDATE))
subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s_efis.ipynb"' % (PLANE,date,FNDATE))
subprocess.getoutput('jupyter nbconvert --to notebook --execute "../%s Data/flights/%s/%s_ems.ipynb"' % (PLANE,date,FNDATE))
# link in notebooks folder:
# pushd ${PLANE}\ Data/flights/notebooks
# ln -s ../${1}/${DATE}_efis.ipynb .
# ln -s ../${1}/${DATE}_ems.ipynb .
# popd
def main():
parser = argparse.ArgumentParser(description='Process all log files')
@@ -79,7 +113,8 @@ def main():
args = parser.parse_args()
prepare(args.date)
process(args.date,args.logname)
num_datasets=process(args.date, args.logname)
create_notebooks(args.date, num_datasets)
Analysis_Tools/walk_logs.sh
-3
View File
@@ -9,8 +9,5 @@ pushd ./programs/
python walk_logs.py $1
popd
DATE=$(./programs/dMoYr_yyyymmdd.py $1)
jupyter nbconvert --to notebook --execute ${PLANE}\ Data/flights/$1/${DATE}_efis.ipynb
jupyter nbconvert --to notebook --execute ${PLANE}\ Data/flights/$1/${DATE}_ems.ipynb