add programs

Analysis_Tools/programs/efis2csv.py

@@ -0,0 +1,144 @@
+"""
+This program converts an EFIS file to csv
+
+example:
+MacPro:programs ken$ python efis2csv.py ../N72KH\ Data/flights/19oct2018/N72KH_EFIS_19oct2018/LOG000N_F.TXT
+which prints the csv format to standard out
+
+MacPro:programs ken$ python efis2csv.py ../N72KH\ Data/flights/19oct2018/N72KH_EFIS_19oct2018/LOG000N_F.TXT > ../N72KH\ Data/flights/19oct2018/N72KH_EFIS_19oct2018/LOG000N_F.csv
+which pipes standard out to LOG000N_F.csv
+
+or run from flights/
+MacPro:flights ken$ python ../../programs/efis2csv.py 10oct2018/N72KH_EFIS_10oct2018/LOG000N_F.TXT > 10oct2018/N72KH_EFIS_10oct2018/LOG000N_F.csv
+
+
+Dynon EFIS data description:
+index, width, description, comment
+
+1, 2, Hour, 00 to 23, current hour according to EFIS-D10As internal clock
+3, 2, Minute, 00 to 59, current minute according to EFIS-D10As internal clock
+5, 2, Second, 00 to 59, current second according to EFIS-D10As internal clock
+7, 2, Fractions, 00 to 63, counter for 1/64 second. Data output frequency.
+9, 1, Pitch Sign, + or - (positive means plane is pitched up)
+10, 3, Pitch, 000 to 900, pitch up or down from level flight in 1/10 degrees (900 = 90o)
+13, 1, Roll Sign, + or - (positive means plane is banked right)
+14, 4, Roll, 0000 to 1800, roll left or right from level flight in 1/10 degrees (1800 = 180o)
+18, 3, Yaw, 000 to 359 in degrees (000 = North, 090 = East, 180 = South, 270 = West)
+21, 4, Airspeed, 0000 to 9999, airspeed in units of 1/10 m/s (1555 = 155.5 m/s)
+25, 1, Altitude Sign, + or - (positive means altitude is above sea-level)
+26, 4, Altitude, 0000 to 9999, altitude in units of meters
+30, 1, Turn Rate Sign, + or - (positive means plane is turning right)
+31, 3, Turn Rate, 000 to 999, 1/10 degrees/second rate of yaw change
+34, 1, Lateral Gs Sign, + or - (positive means plane is experiencing leftward lateral acceleration)
+35, 2, Lateral Gs, 00 to 99, lateral Gs in units of 1/100 G (99 = 0.99 Gs)
+37, 1, Vertical Gs Sign, + or - (positive means plane is experiencing upward vertical acceleration)
+38, 2, Vertical Gs, 00 to 99, vertical Gs in units of 1/10 G (99 = 9.9 Gs)
+40, 2, Angle of Attack, 00 to 99, percentage of stall angle.
+42, 6, Status Bitmask, An internal-use status bitmask containing 24 bits
+48, 2, Product ID, Ascii-hex Dynon product ID: 01=EFIS-D10A, 10=EFIS-D10, 03=EMS-D10
+50, 2, Checksum, The ascii-hex 2 byte sum of all 49 preceding bytes
+52, 2, CR/LF, Carriage Return, Linefeed = 0x0D, 0x0A
+
+"""
+
+import argparse
+import struct
+
+"""
+open file and read a byte at a time into a buffer
+when two headers have been found, process bytes inbetween
+then copy second header into the beginning of a new buffer and throw away old buffer
+lines are only valid if surrounded by headers, otherwise they may be incomplete
+"""
+
+def calculate(buf):
+    """ 
+    one buffer of data.
+    """
+    feetpermeter=3.2808399
+    mps2mph=2.2369363
+    hrs=int(buf[0:2])
+    min=int(buf[2:4])
+    sec=int(buf[4:6])
+    alti=int(buf[25:29])*feetpermeter
+    altsign=buf[24]
+    if altsign=='-': alti=alti*-1
+    airspeed=(int(buf[20:24])/10.0)*mps2mph
+
+    values = (hrs,min,sec,\
+             alti,airspeed)
+    #return values
+
+
+def efistime(buf):
+    hrs=int(buf[0:2])
+    min=int(buf[2:4])
+    sec=int(buf[4:6])
+    return(hrs,min,sec)
+
+def alti(buf):
+    feetpermeter=3.2808399
+    alti=int(buf[25:29])*feetpermeter
+    altsign=buf[24]
+    if altsign=='-': alti=alti*-1
+    return(alti)
+
+def pitch(buf):
+    pit=int(buf[9:12])/10.0
+    pitsign=buf[8]
+    if pitsign=='-': pit=pit*-1
+    return(pit)
+
+def roll(buf):
+    roll=int(buf[13:17])/10.0
+    rollsign=buf[12]
+    if rollsign=='-': roll=roll*-1
+    return(roll)
+
+def yaw(buf):
+    yaw=int(buf[17:20])
+    return(yaw)
+
+def airspeed(buf):
+    mps2mph=2.2369363
+    return((int(buf[20:24])/10.0)*mps2mph)
+
+def aoa(buf):
+    return(int(buf[39:41]))
+
+
+
+def output_values(h,m,s,alti,airspeed,aoa,pitch,roll,yaw):
+    print(("%02d:%02d:%02d, " # h,m,s
+          "%d, %d, %d, " # alti,airspeed, aoa
+          "%4.1f, %4.1f, %d") % (h,m,s,alti,airspeed,aoa,pitch,roll,yaw))
+
+
+def read_data(name):
+    with open(name, "r") as f:
+        buf1=[] # start with buffer clear
+        l1=f.readline() 
+        while l1 != "": # not EOF
+            #print(l1)
+            if (len(l1)==52): # was 53 with openlog
+                b=l1
+                output_values(efistime(b)[0],efistime(b)[1],efistime(b)[2],alti(b),airspeed(b),aoa(b),pitch(b),roll(b),yaw(b))
+            l1=f.readline()
+    return 0
+
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='EFIS csv converter',
+        epilog="./%(prog)s [options] > outputfile.csv to save the result")
+    parser.add_argument('fname', 
+        help='file name to convert with path')
+    args = parser.parse_args()
+    data=read_data(args.fname)
+    return data
+
+
+if __name__=="__main__":
+    d=main()
+

Analysis_Tools/programs/efis_plot_util_nb.py

@@ -0,0 +1,259 @@
+# %pylab makes the following imports::
+#
+#import numpy
+#import matplotlib
+#from matplotlib import pylab, mlab, pyplot
+#np = numpy
+#plt = pyplot
+#
+#from IPython.display import display
+#from IPython.core.pylabtools import figsize, getfigs
+#
+#from pylab import *
+#from numpy import *
+
+import pandas as pd
+from matplotlib.dates import date2num, DateFormatter
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mplT
+import datetime
+import time
+import subprocess
+
+
+# GLOBAL start value sv:
+sv=0
+
+
+def get_df(datafile):
+    global sv
+    df=pd.read_csv(datafile, names=['TIME','ALTI','AIRSP','AOA','PITCH','ROLL','YAW'])
+
+    # convert time data to datetime format
+    df['TIME']=pd.to_datetime(df['TIME'], format='%H:%M:%S')
+
+    sv=1
+    # ignore data before clock is set from EFIS
+    # this IS the EFIS
+    '''
+    for i in range(1,len(df)):
+        if ((df['TIME'][i]-df['TIME'][i-1]).value > 1000000000):
+            sv=i
+            print("start %d" %(sv))
+            break
+    '''
+    return df
+
+
+
+
+def add_elapsed(df):
+    st=df['TIME'][sv] # start time
+    et=[] # elapsed time
+
+    # add elapsed time column
+    for i in range(0,len(df)):
+        et.append(df['TIME'][i]-st)
+    df2=pd.DataFrame(et)
+    df2.columns=['ELAPSED']
+    #s=pd.Series([val.time() for val in df['TIME']])
+    #df2.merge(s.to_frame(), left_index=True, right_index=True)
+    ndf=df.join(df2)
+
+    # make flight duration (elapsed time) the index
+    #ndf.index=ndf['ELAPSED']
+    #del ndf['ELAPSED'] # delete the column
+    ndf = ndf.set_index('ELAPSED')
+    #ndf = ndf.set_index('TIME')
+    #ndf.drop(['TIME'], axis=1, inplace=True)
+
+    return ndf
+
+
+
+
+
+
+def log_number(date):
+    """get number of logs on date and return size + name"""
+    result=[]
+    rv=subprocess.check_output("python "+"flight_index.py "+date, shell=True)
+    rvl=rv.decode('utf-8').split('\n')
+    for i in range(0,len(rvl)-1):
+        result.append(rvl[i].lstrip(" ").split(" "))
+    return result
+
+
+
+def check_newday(df):
+    """returns boolean and the index of the last time before 00:00:00"""
+    newday=False
+    idx=0
+    # quick check if we need to search for the idx
+    r=df['TIME'][sv]-df['TIME'][len(df)-1]
+    if (r.days == -1):
+        return newday,0
+    # skip ['TIME'][0] since includes index name...
+    for i in range(1,len(df)-1):
+        td=date2num(df['TIME'][i+1])-date2num(df['TIME'][i])
+        if td < 0: 
+            print("day wrap @ ", end=' ')
+            print(i)
+            idx=i
+            newday=True
+    return newday,idx
+
+
+
+
+
+def fix_newday(df,idx):
+    # fix new day in-line so we don't parameterize or globalize the df
+    t1=df['TIME'][:idx+1]
+    t2=df['TIME'][idx+1:]
+    t2a=pd.DataFrame([x+datetime.timedelta(days=1) for x in t2])
+    t2as=t2a.iloc[:,0] # get series
+    t2as.name='TIME'
+    t2as.index=t2.index
+    tn=t1.append(t2as)
+    df.loc[:,'TIME']=tn
+    return df
+
+
+
+
+
+def flighttime(df):
+    print("flight time = ", ( df['TIME'][len(df)-1] - df['TIME'][sv]))
+
+
+
+
+
+def x_format(df):
+    """sets x axis time format based on flight time"""
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        formatter=mplT.FuncFormatter(lambda s, x: time.strftime('%H:%M:%S', time.gmtime(s)))
+    else:
+        formatter=mplT.FuncFormatter(lambda s, x: time.strftime('%M:%S', time.gmtime(s)))
+    return formatter
+
+
+
+
+"""
+##################################################################################
+###
+###   Useful plots...
+###
+"""
+
+
+
+
+def plot_ALTSPD(df):
+    fig,axes = plt.subplots(nrows=2, ncols=1, sharex=True)
+    df.plot(y='ALTI', ax=axes[0])
+    df.plot(y='AIRSP', ax=axes[1])
+    ax=plt.gca()
+    tick_labels=ax.xaxis.major.formatter.seq
+    ax.xaxis.major.formatter.seq = [label.split()[-1] if label else "" for label in tick_labels]
+    plt.xticks(rotation=30)
+
+
+def plot_ALTI(df):
+    # RPM
+    fig, ax = plt.subplots()
+    t=df.index.seconds[sv:]
+    #df.plot(ax=ax, y="RPM", xlim=(sv+50,len(df)), color='b')
+    ax.plot(t,df['ALTI'][sv:],'-',label="ALTI")
+    ax.xaxis.set_major_formatter(x_format(df))
+
+    ax.set_ylabel('Feet')
+    ax.legend(loc='lower center')
+    ax=plt.gca()
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elasped Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+
+
+
+def plot_AIRSP(df):
+    # RPM
+    fig, ax = plt.subplots()
+    t=df.index.seconds[sv:]
+    #df.plot(ax=ax, y="RPM", xlim=(sv+50,len(df)), color='b')
+    ax.plot(t,df['AIRSP'][sv:],'-',label="AIRSP")
+    ax.xaxis.set_major_formatter(x_format(df))
+
+    ax.set_ylabel('Kts')
+    ax.legend(loc='lower center')
+    ax=plt.gca()
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elasped Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+
+
+
+def plot_ROLL(df):
+    fig, ax = plt.subplots()
+    t=df.index.seconds[sv:]
+
+    ax.plot(t,df['ROLL'][sv:],'-',label="ROLL")
+    ax.xaxis.set_major_formatter(x_format(df))
+
+    ax.set_ylabel('Degree')
+    ax.legend(loc='lower center')
+    ax=plt.gca()
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elasped Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+
+
+
+def plot_AOA(df):
+    # RPM
+    fig, ax = plt.subplots()
+    t=df.index.seconds[sv:]
+    #df.plot(ax=ax, y="RPM", xlim=(sv+50,len(df)), color='b')
+    ax.plot(t,df['AOA'][sv:],'-',label="AOA")
+    ax.xaxis.set_major_formatter(x_format(df))
+
+    ax.set_ylabel('Deg')
+    ax.legend(loc='lower center')
+    ax=plt.gca()
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elasped Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+
+
+def plot_PITCH(df):
+    # RPM
+    fig, ax = plt.subplots()
+    t=df.index.seconds[sv:]
+    #df.plot(ax=ax, y="RPM", xlim=(sv+50,len(df)), color='b')
+    ax.plot(t,df['PITCH'][sv:],'-',label="PITCH")
+    ax.xaxis.set_major_formatter(x_format(df))
+
+    ax.set_ylabel('Deg')
+    ax.legend(loc='lower center')
+    ax=plt.gca()
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elasped Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+

Analysis_Tools/programs/ems2csv.py

@@ -0,0 +1,146 @@
+"""
+This program converts an EMS file to csv
+
+index, width, description, comment
+
+0, 2, Hour, 00 to 23, zulu hour
+2, 2, Minute, 00 to 59, zulu minutes
+4, 2, Second, 00 to 59, zulu seconds
+6, 2, Fractions, 00 to 63, counter for 1/64 second. Data output frequency.
+8, 4, manifold pressure, inHg * 100
+12, 3, oil temp, deg F
+15, 3, oil pressure, psi
+18, 3, fuel pressure, psi * 10
+21, 3, voltage, volts * 10
+24, 3, current, amps
+27, 3, rpm, rpm/10
+30, 3, fuel flow, gph * 10
+33, 4, fuel remaining, gallons * 10
+37, 3, fuel left tank, gallons * 10
+40, 3, fuel right tank, gallons * 10
+43, 8, GP-1
+51, 8, GP-2
+59, 8, GP-3
+67, 4, GP thremocouple
+71, 4, EGT_1
+75, 4, EGT_2
+79, 4, EGT_3
+83, 4, EGT_4
+87, 4, EGT_5
+91, 4, EGT_6
+95, 3, CHT_1
+98, 3, CHT_2
+101, 3, CHT_3
+104, 3, CHT_4
+107, 3, CHT_5
+110, 3, CHT_6
+113, 1, contact-1
+114, 1, contact-2
+115, 2, Product ID, Ascii-hex Dynon product ID: 01=EFIS-D10A, 10=EFIS-D10, 03=EMS-D10
+117, 2, Checksum, The ascii-hex 2 byte sum of all 49 preceding bytes
+119, 2, CR/LF, Carriage Return, Linefeed = 0x0D, 0x0A
+
+"""
+
+import argparse
+import struct
+
+"""
+open file and read a byte at a time into a buffer
+when two headers have been found, process bytes inbetween
+then copy second header into the beginning of a new buffer and throw away old buffer
+lines are only valid if surrounded by headers, otherwise they may be incomplete
+"""
+
+
+def emstime(buf):
+    hrs=int(buf[0:2])
+    min=int(buf[2:4])
+    sec=int(buf[4:6])
+    return(hrs,min,sec)
+
+def manp(buf):
+    return(int(buf[8:12])/100.0)
+
+def oilt(buf):
+    return(int(buf[12:15]))
+
+def oilp(buf):
+    return(int(buf[15:18]))
+
+def fuelp(buf):
+    return(int(buf[18:21])/10.0)
+
+def volts(buf):
+    return(int(buf[21:24])/10.0)
+
+def amps(buf):
+    return(int(buf[24:27]))
+
+def rpm(buf):
+    return(int(buf[27:30])*10.0)
+
+def fflow(buf):
+    return(int(buf[30:33])/10.0)
+
+def frem(buf):
+    return(int(buf[33:37])/10.0)
+
+def ful(buf):
+    return(int(buf[37:40])/10.0)
+
+def fur(buf):
+    return(int(buf[40:43])/10.0)
+
+def egt(buf):
+    e1=int(buf[71:75])
+    e2=int(buf[75:79])
+    e3=int(buf[79:83])
+    e4=int(buf[83:87])
+    return(e1,e2,e3,e4)
+
+def cht(buf):
+    c1=int(buf[95:98])
+    c2=int(buf[98:101])
+    c3=int(buf[101:104])
+    c4=int(buf[104:107])
+    return(c1,c2,c3,c4)
+
+
+
+def output_values(h,m,s,manp,oilt,oilp,fuelp,volts,amps,rpm,fflow,frem,ful,fur,egt1,egt2,egt3,egt4,cht1,cht2,cht3,cht4):
+    print(("%02d:%02d:%02d, " # h,m,s
+          "%4.2f, %d, %d, %3.1f, %3.1f, %d, " # manp,oilt,oilp,fuelp,volts,amps
+          "%d, %3.1f, %3.1f, %3.1f, %3.1f, " # rpm,fflow,frem,ful,fur
+          "%d, %d, %d, %d, " # egt[1..4]
+          "%d, %d, %d, %d") % (h,m,s,manp,oilt,oilp,fuelp,volts,amps,rpm,fflow,frem,ful,fur,egt1,egt2,egt3,egt4,cht1,cht2,cht3,cht4))
+
+
+def read_data(name):
+    with open(name, "r") as f:
+        buf1=[] # start with buffer clear
+        l1=f.readline()
+        while l1 != "": # not EOF
+            #print(len(l1))
+            #print(l1)
+            if (len(l1)==120): # was 121 with openlog
+                b=l1
+                output_values(emstime(b)[0],emstime(b)[1],emstime(b)[2],manp(b),oilt(b),oilp(b),fuelp(b),volts(b),amps(b),rpm(b),fflow(b),frem(b),ful(b),fur(b),egt(b)[0],egt(b)[1],egt(b)[2],egt(b)[3],cht(b)[0],cht(b)[1],cht(b)[2],cht(b)[3])
+            l1=f.readline()
+    return 0
+
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='EMS csv converter',
+        epilog="./%(prog)s [options] > outputfile.csv to save the result")
+    parser.add_argument('fname',
+        help='file name to convert with path')
+    args = parser.parse_args()
+    data=read_data(args.fname)
+    return data
+
+
+if __name__=="__main__":
+    d=main()

Analysis_Tools/programs/ems_plot_util_nb.py

@@ -0,0 +1,454 @@
+# %pylab makes the following imports::
+#
+#import numpy
+#import matplotlib
+#from matplotlib import pylab, mlab, pyplot
+#np = numpy
+#plt = pyplot
+#
+#from IPython.display import display
+#from IPython.core.pylabtools import figsize, getfigs
+#
+#from pylab import *
+#from numpy import *
+
+import pandas as pd
+from matplotlib.dates import date2num, DateFormatter
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mplT
+import datetime
+import time
+import subprocess
+
+# GLOBAL SAVEPLOTS
+SAVEPLOTS = False
+SAVEPATH = "."
+
+# GLOBAL start value sv:
+sv=0
+
+
+
+
+def get_df(datafile):
+    """read csv datafile; return dataframe"""
+    
+    global sv
+
+    df=pd.read_csv(datafile, comment="#", names=['TIME','MP','OILT','OILP', 'FUELP','VBAT','IBAT','RPM','FFLOW','Frem','FL','FR','EGT1','EGT2','EGT3','EGT4','CHT1','CHT2','CHT3','CHT4'])
+
+    # convert time data to datetime format
+    df['TIME']=pd.to_datetime(df['TIME'], format='%H:%M:%S')
+
+    sv=1
+    # ignore data before clock is set from EFIS
+    # set GLOBAL sv
+    for i in range(1,len(df)):
+        if ((df['TIME'][i]-df['TIME'][i-1]).value > 1000000000):
+            sv=i
+            print("start %d" %(sv))
+            break
+    return df
+
+
+
+
+def add_elapsed(df):
+    st=df['TIME'][sv] # start time
+    et=[] # elapsed time
+
+    # add elapsed time column
+    for i in range(0,len(df)):
+        et.append(df['TIME'][i]-st)
+    df2=pd.DataFrame(et)
+    df2.columns=['ELAPSED']
+    #s=pd.Series([val.time() for val in df['TIME']])
+    #df2.merge(s.to_frame(), left_index=True, right_index=True)
+    ndf=df.join(df2)
+
+    # make flight duration (elapsed time) the index
+    #ndf.index=ndf['ELAPSED']
+    #del ndf['ELAPSED'] # delete the column
+    ndf = ndf.set_index('ELAPSED')
+    #ndf = ndf.set_index('TIME')
+    #ndf.drop(['TIME'], axis=1, inplace=True)
+
+    return ndf
+
+ 
+
+
+
+def log_number(date):
+    """get number of logs on date and return size + name"""
+    result=[]
+    rv=subprocess.check_output("python "+"flight_index.py "+date, shell=True)
+    rvl=rv.split('\n')
+    for i in range(0,len(rvl)-1): 
+        result.append(rvl[i].lstrip(" ").split(" "))
+    return result
+
+
+
+
+
+def check_newday(df):
+    """returns boolean and the index of the last time before 00:00:00"""
+    newday=False
+    idx=0
+    # quick check if we need to search for the idx
+    r=df['TIME'][sv]-df['TIME'][len(df)-1]
+    if (r.days == -1):
+        return newday,0
+    # skip ['TIME'][0] since includes index name...
+    for i in range(1,len(df)-1):
+        td=date2num(df['TIME'][i+1])-date2num(df['TIME'][i])
+        if td < 0: 
+            print("day wrap @ ", end=' ')
+            print(i)
+            idx=i
+            newday=True
+    return newday,idx
+
+
+
+
+
+def fix_newday(df,idx):
+    # fix new day in-line so we don't parameterize or globalize the df
+    t1=df['TIME'][:idx+1]
+    t2=df['TIME'][idx+1:]
+    t2a=pd.DataFrame([x+datetime.timedelta(days=1) for x in t2])
+    t2as=t2a.iloc[:,0] # get series
+    t2as.name='TIME'
+    t2as.index=t2.index
+    tn=t1.append(t2as)
+    df.loc[:,'TIME']=tn
+    return df
+
+
+
+
+
+def flighttime(df):
+    """returns flight time of current dataframe"""
+    #print "flight time = ", ( df['TIME'][len(df)-1] - df['TIME'][sv])
+    # use elapsed time to avoid day change
+    print("flight time = ", ( df.index[len(df)-1] - df.index[sv]))
+
+
+
+
+
+def x_format(df):
+    """sets x axis time format based on flight time"""
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        formatter=mplT.FuncFormatter(lambda s, x: time.strftime('%H:%M:%S', time.gmtime(s)))
+    else:
+        formatter=mplT.FuncFormatter(lambda s, x: time.strftime('%M:%S', time.gmtime(s)))
+    return formatter
+
+
+
+
+
+"""
+##################################################################################
+###
+###   Useful plots...
+###
+"""
+
+
+def plot_all_df(df):
+    df[sv:].plot(rot=30)
+    pass
+
+
+
+def plot_EGT(df):
+    # Zulu time
+    fig, ax = plt.subplots() 
+    ax.xaxis_date()
+    ax.xaxis.set_major_formatter(DateFormatter('%H:%M'))
+    x=df['TIME'][sv:]
+    
+    t=list(map(date2num, x))
+    
+    y1=df['EGT1'][sv:]
+    y2=df['EGT2'][sv:]
+    y3=df['EGT3'][sv:]
+    y4=df['EGT4'][sv:]
+
+    ax.plot_date(t,y1,'-',label="EGT1")
+    ax.plot_date(t,y2,'-',label="EGT2")
+    ax.plot_date(t,y3,'-',label="EGT3")
+    ax.plot_date(t,y4,'-',label="EGT4")
+
+    plt.legend(loc='lower center')
+    plt.xlabel("Time")
+    plt.ylabel("Temp (F)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_EGT.png")
+ 
+
+def plot_EGTdt(df):
+    # Elapsed time
+    # index set to 'ELAPSED'
+    fig, ax = plt.subplots() 
+    t=df.index.seconds[sv:]
+
+    y1=df['EGT1'][sv:]
+    y2=df['EGT2'][sv:]
+    y3=df['EGT3'][sv:]
+    y4=df['EGT4'][sv:]
+    
+    ax.plot(t,y1,'-',label="EGT1")
+    ax.plot(t,y2,'-',label="EGT2")
+    ax.plot(t,y3,'-',label="EGT3")
+    ax.plot(t,y4,'-',label="EGT4")
+    
+    ax.xaxis.set_major_formatter(x_format(df))    
+    plt.legend(loc='lower center')
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elapsed Time (MM:SS)")
+    plt.ylabel("Temp (F)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_EGTe.png")
+
+
+
+def plot_CHT(df):
+    fig, ax = plt.subplots()
+    ax.xaxis_date()
+    ax.xaxis.set_major_formatter(DateFormatter('%H:%M'))
+    x=df['TIME'][sv:]
+    t=list(map(date2num, x))
+
+    y1=df['CHT1'][sv:]
+    y2=df['CHT2'][sv:]
+    y3=df['CHT3'][sv:]
+    y4=df['CHT4'][sv:]
+    
+    ax.plot(t,y1,'-',label="CHT1")
+    ax.plot(t,y2,'-',label="CHT2")
+    ax.plot(t,y3,'-',label="CHT3")
+    ax.plot(t,y4,'-',label="CHT4")
+    
+    plt.legend(loc='lower center')
+    plt.xlabel("Time")
+    plt.ylabel("Temp (F)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_CHT.png")
+
+
+
+
+def plot_CHTdt(df):
+    # Elapsed time
+    # index set to 'ELAPSED'
+    fig, ax = plt.subplots() 
+    t=df.index.seconds[sv:]
+
+    y1=df['CHT1'][sv:]
+    y2=df['CHT2'][sv:]
+    y3=df['CHT3'][sv:]
+    y4=df['CHT4'][sv:]
+    
+    ax.plot(t,y1,'-',label="CHT1")
+    ax.plot(t,y2,'-',label="CHT2")
+    ax.plot(t,y3,'-',label="CHT3")
+    ax.plot(t,y4,'-',label="CHT4")
+    
+    ax.xaxis.set_major_formatter(x_format(df))    
+    plt.legend(loc='lower center')
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elapsed Time (MM:SS)")
+    plt.ylabel("Temp (F)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_CHTe.png")
+
+
+
+
+
+def plot_OIL(df):
+    fig,axs = plt.subplots(nrows=2, ncols=1, sharex=True)
+    df.plot(y='RPM', xlim=(sv+50,len(df)), ax=axs[0])
+    df.plot(y='OILP', xlim=(sv+50,len(df)), ax=axs[1])
+    plt.ylabel("PSI")
+    ax=plt.gca()
+    tick_labels=ax.xaxis.major.formatter.seq
+    ax.xaxis.major.formatter.seq = [label.split()[-1] if label else "" for label in tick_labels]
+    plt.xticks(rotation=30)
+    plt.xlabel("Time (MM:SS)")
+    axs[0].set_title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_OIL.png")
+
+"""
+def plot_BAT(df):
+    # VBAT, IBAT
+    fig, ax = plt.subplots()
+    ax.xaxis_date()
+    ax.xaxis.set_major_formatter(DateFormatter('%H:%M'))
+    x=df['TIME'][sv:]
+    t=list(map(date2num, x))
+
+    df.plot(ax=ax, y="VBAT", xlim=(sv+50,len(df)), color='b')
+    ax.set_ylabel('Volts')
+    ax.legend(loc='lower center')
+    ax2=ax.twinx()
+    df.plot(ax=ax2, y="IBAT", xlim=(sv+50,len(df)), color='g')
+    ax2.set_ylabel("Amps")
+    ax2.legend(loc='lower right')
+    ax=plt.gca()
+    #tick_labels=ax.xaxis.major.formatter.seq
+    #ax.xaxis.major.formatter.seq = [label.split()[-1] if label else "" for label in tick_labels]
+    plt.xticks(rotation=30)
+    plt.xlabel("Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_BAT.png")
+"""    
+
+def plot_BAT(df):
+    # VBAT, IBAT
+    t=df.index.seconds[sv:]
+
+    fig,axs = plt.subplots(nrows=2, ncols=1, sharex=True)
+    #axs[1].xaxis_date()
+    axs[1].xaxis.set_major_formatter(DateFormatter('%H:%M'))
+    df.plot(ax=axs[0],y='VBAT', xlim=(sv,len(df)))
+    axs[0].set_ylabel("Volts")
+    axs[0].set_ylim(10.0,14.0)
+    axs[0].grid()
+    #axs[0].legend(loc='lower center')
+    #
+    df.plot(ax=axs[1],y='IBAT', xlim=(sv,len(df)))
+    axs[1].set_ylabel("Amps")
+    axs[1].set_ylim(-25.0,25)
+    axs[1].grid()
+    #
+    ax=plt.gca()
+    ax.xaxis.set_major_formatter(x_format(df))
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elapsed Time (MM:SS)")
+    axs[0].set_title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_VI.png")
+
+
+
+def plot_FUEL(df):
+    #  FL, FR
+    t=df.index.seconds[sv:]
+
+    fig,axs = plt.subplots(nrows=2, ncols=1, sharex=True)
+    #axs[1].xaxis_date()
+    axs[1].xaxis.set_major_formatter(DateFormatter('%H:%M'))
+    axs[0].plot(t,df['FL'][sv:],'-',label="FuelLeft")
+    axs[1].plot(t,df['FR'][sv:],'-',label="FuelRight")
+    axs[0].set_ylim(0,18)
+    axs[1].set_ylim(0,18)
+    axs[0].set_ylabel("Gal")
+    axs[1].set_ylabel("Gal")
+    axs[0].legend(loc='lower center')
+    axs[1].legend(loc='lower center')
+    ax=plt.gca()
+    ax.xaxis.set_major_formatter(x_format(df))
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elapsed Time (MM:SS)")
+    axs[0].set_title(FLIGHTDATE)
+    axs[0].grid()
+    axs[1].grid()
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_FuelLR.png")
+ 
+    # FFLOW, Frem
+    # see https://matplotlib.org/2.0.2/examples/color/named_colors.html
+    fig, ax = plt.subplots()
+    plt.xticks(rotation=30)
+    ax.plot(t,df["Frem"][sv:],'-',label="Frem", color='b')
+    ax.set_ylabel('Gal')
+    ax.legend(loc='lower center')
+
+    ax2=ax.twinx()
+    ax2.plot(t,df["FFLOW"][sv:],'-',label="FFLOW", color='g')
+    ax2.set_ylabel("GPH")
+    ax2.legend(loc='lower right')
+
+    ax.xaxis.set_major_formatter(x_format(df))
+    ax2.xaxis.set_major_formatter(x_format(df))
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        ax.set_xlabel("Elapsed Time (HH:MM:SS)")
+        ax2.set_xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        ax.set_xlabel("Elapsed Time (MM:SS)")
+        ax2.set_xlabel("Elapsed Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+    ax.grid(axis='y',color='lightblue')
+    ax2.grid(axis='y',color='lightgreen')
+    ax.grid(axis='x')
+    ax2.grid(axis='x')
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_Fuel.png")
+
+
+
+def plot_PWR(df):
+    # MP, RPM
+    fig, ax = plt.subplots()
+    df.plot(ax=ax, y="MP", xlim=(sv+50,len(df)), color='b')
+    ax.set_ylabel('PSI')
+    ax.legend(loc='lower center')
+    ax2=ax.twinx()
+    df.plot(ax=ax2, y="RPM", xlim=(sv+50,len(df)), color='g')
+    ax2.set_ylabel("rpm")
+    ax2.legend(loc='lower right')
+    ax=plt.gca()
+    tick_labels=ax.xaxis.major.formatter.seq
+    ax.xaxis.major.formatter.seq = [label.split()[-1] if label else "" for label in tick_labels]
+    plt.xticks(rotation=30)
+    plt.xlabel("Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_PWR.png")
+    
+
+def plot_RPM(df):
+    # RPM
+    fig, ax = plt.subplots()
+    t=df.index.seconds[sv:]
+    #df.plot(ax=ax, y="RPM", xlim=(sv+50,len(df)), color='b')
+    ax.plot(t,df['RPM'][sv:],'-',label="RPM")
+    ax.xaxis.set_major_formatter(x_format(df))
+
+    ax.set_ylabel('RPM')
+    ax.legend(loc='lower center')
+    ax=plt.gca()
+    plt.xticks(rotation=30)
+    if df.index[-1] > pd.Timedelta("01:00:00"):
+        plt.xlabel("Elapsed Time (HH:MM:SS)")
+    else:
+        plt.xlabel("Elasped Time (MM:SS)")
+    plt.title(FLIGHTDATE)
+    if SAVEPLOTS: savefig(SAVEPATH+FLIGHTDATE+"/"+FLIGHTDATE+"_PWR.png")
+
+
+def plot_ALL(df):
+    plot_PWR(df)
+    plot_FUEL(df)
+    plot_OIL(df)
+    plot_BAT(df)
+    plot_CHT(df)
+    plot_EGT(df)
+
+
+

Analysis_Tools/programs/gps_convert.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+# $1 is .csv input $2 is .kml output
+gpsbabel -t -i unicsv -f "$1" -x transform,trk=wpt  -o kml,floating=1 -F "$2"
+#echo "$1"
+#echo "$2"
+

Analysis_Tools/programs/gps_formatter.py

@@ -0,0 +1,183 @@
+# IPython log file
+"""
+This program converts a Garmin aviation GPS log to csv
+"""
+
+"""
+FIXME:
+need to handle FlightPlan data. see 30oct2017 LOG005N_G.TXT 
+search KBFL
+
+"""
+
+import argparse
+
+"""
+start with simple lat/lon & altitude
+
+STX (0x02 ^B)
+type-1 sentence
+type-2 sentence(s)
+ETX (0x03 ^C)
+
+type-1
+------
+
+id-char (alphabetic)
+dd..dd data 1-10 chars
+CR
+LF
+
+id=z
+aaaaa altitude in feet
+xcsv format ALT_FEET
+
+id=A
+s dd mmhh latitude: s{N,S} deg min hundredths of min
+min to decimal = mmhh/60
+xcsv format: N=+,S=- deg.min
+xcsv format LAT_DIRDECIMAL
+
+id=B
+s ddd mmhh longitude: s{E,W} deg min hundredths of min
+min to decimal = mmhh/60
+xcsv format: E=+,W=- deg.min
+xcsv format LON_DIRDECIMAL
+
+id=C
+ddd track in degrees
+
+id=D
+sss ground speed in knots
+xcsv format PATH_SPEED_KNOTS
+unicsv format in m/s
+1mps =  1.94384617178935 knot
+1knot = 0.5144440mps
+
+
+type-2
+------
+
+id 3 chars: w nn nn{01..31}
+seq
+wpt
+lat
+lon
+mvar
+CR
+LF
+
+"""
+
+"""
+drop chars until first ETX
+keep chars STX until ETX
+trim STX and ETX
+read lines searching for id
+
+example:
+z-0000
+AN 37 3053
+BW 122 1492
+C346
+D000
+
+
+"""
+
+def openfile(fname):
+    f=open(fname, "r")
+    return f
+
+
+def format_data(buf):
+    """process 1 record"""
+    s=''.join(buf)
+    ss=s.replace('\r','').split("\n")
+    # don't process when A- B- = missing lat/lon
+    if s.find("A-") > 0 or s.find("B-") > 0: return
+    for i in range(0,len(ss)):
+        if ss[i][0]=='z': 
+            #print ss[i][1:]+", ",
+            s2=ss[i][1:]
+            try:
+                int(s2)
+                isint=True
+            except ValueError:
+                isint=False
+            if isint == True: print(s2+", ", end=' ')
+        if ss[i][0]=='A': 
+            s1=ss[i][1:].split()
+            if s1[0]=='N': sign=1
+            else: sign=-1
+            try:
+                s2=sign*(int(s1[1])+(int(s1[2])/100.0)/60.0)
+                print("%0.5f"%(s2)+", ", end=' ')
+            except:
+                #print "ERROR!!"
+                #print s1
+                pass 
+        if ss[i][0]=='B': 
+            s1=ss[i][1:].split()
+            if s1[0]=='E': sign=1
+            else: sign=-1
+            try:
+                s2=sign*(int(s1[1])+(int(s1[2])/100.0)/60.0)
+                print("%0.5f"%(s2)+", ", end=' ')
+            except:
+                #print "ERROR!!"
+                #print s1
+                pass
+        if ss[i][0]=='D': 
+            try:
+                s1=int(ss[i][1:])*0.5144440
+                print("%0.2f"%(s1)+", ", end=' ')
+            except:
+                pass
+
+
+
+def read_data(f):
+    """read gps records"""
+    rec_num=0
+    # search for begin record in GPS.TXT using vim with :/^v^b
+    # skip to the end of record 
+    # in case we started logging in the middle of a record
+    byte=f.read(1)
+    if byte == '' : return
+    while (ord(byte) != 3): # ETX (0x03 ^C)
+        byte=f.read(1)
+        if byte == '' : return
+    # now process records until the end of file
+    byte=f.read(1)
+    while byte != '':
+        buf1=[] # start with buffer clear
+        while ord(byte) != 3: # ETX (0x03 ^C)
+            #print "%02X " % ord(byte),
+            byte = f.read(1)
+            if byte == '': break
+            buf1.append(byte)
+        else:  # ETX, so process this record
+            rec_num+=1
+            byte = f.read(1)
+            #print buf1
+            format_data(buf1)
+            print(" ") # essentially new line for the record
+            #print rec_num
+
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Garmin Aviaiton GPS format converter')
+    parser.add_argument('file',
+        help='name of file to convert')
+    args = parser.parse_args()
+    fh=openfile(args.file)
+    #print "ALT_FEET, LAT_DIRDECIMAL, LON_DIRDECIMAL, PATH_SPEED_KNOTS"
+    print("altfeet, lat, lon, speed")
+    data=read_data(fh)
+
+
+if __name__=="__main__":
+    main()
+

Analysis_Tools/programs/log_splitter.py

@@ -0,0 +1,68 @@
+# IPython log file
+"""
+This program splits a combo log into separate log files
+The combo log has a header on each sentence.
+E: for EMS
+F: for EFIS
+G: for GPS
+
+There appears to be a bug with the MBed GPS logger where the end of record
+^C is dropped.
+
+The MBed logger also uses \r as line end
+use
+LC_CTYPE=C tr -s '\r' '\n' < Log001n.txt > tmp
+so we can process with readline()
+EFIS and EMS have \r line ends
+GPS has \r\n line ends
+
+"""
+
+
+import argparse
+import os
+
+
+
+def process_data(fname):
+    EOF=False
+    f=open(fname, encoding='utf-8', errors='ignore')
+    fname=os.path.splitext(fname)[0]
+    #fe=open(fname+"_F.txt", "w") # E <-> F label reversed in MBED!! or miswired @ MBED!!
+    #ff=open(fname+"_E.txt", "w")
+    fe=open(fname+"_E.txt", "w") # Teensy
+    ff=open(fname+"_F.txt", "w")
+    fg=open(fname+"_G.txt", "w")
+    while not EOF:
+        s=f.readline()
+        if s=="":
+            EOF=True
+            break
+        if s[:2] == "G:":
+            """multiple lines... """ 
+            fg.write(s[2:])
+            while not EOF:
+                s=f.readline()
+                if s=="":
+                    EOF=True
+                    break
+                if (s[:2] != "E:") and (s[:2] != "F:"): 
+                    fg.write(s)
+                else: 
+                    fg.write(chr(3))
+                    break # done with GPS record
+        if s[:2] == "E:": fe.write(s[2:])
+        if s[:2] == "F:": ff.write(s[2:])
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Log Splitter: creates separate log files for each device')
+    parser.add_argument('file',
+        help='name of file to convert')
+    args = parser.parse_args()
+    process_data(args.file)
+
+
+if __name__=="__main__":
+    main()
+

Analysis_Tools/programs/walk_logs.py

@@ -0,0 +1,87 @@
+"""
+run from Airplane_Data_logger/programs/
+
+processes all LOG0*.TXT files
+~/Documents/Projects/Airplane_Data_Logger/N72KH Data/flights/-date-/
+~/Documents/Projects/Airplane_Data_Logger/programs/
+
+"""
+
+import glob
+import os
+import argparse
+
+
+def prepare(date):
+    print("splitting logs")
+    sp="../N72KH Data/flights/%s/N72KH_combined_%s/" %(date,date) # combined log
+    fl=glob.glob(sp+"L*")
+    for f in fl: # file in filelist
+        os.rename(f, f.upper())
+        fn=os.path.splitext(f.upper())[0] # fn = filename
+        # tr bsd command = translate characters \r -> \n
+        c="LC_CTYPE=C tr -s '\r' '\n' < '%s' > '%s'" % (f.upper(),fn+"N.TXT")
+        os.system(c)
+        # run log_splitter.py on the translated file (fn+"N")
+        c="python ./log_splitter.py '%s.TXT'" % (fn+"N")
+        os.system(c)
+        # NOW MOVE TO CORRECT DIRECTORY
+        pi=[i for i, c in enumerate(f) if c =='/']
+        c="mv '%sN_E.TXT' '%s/N72KH_EMS_%s/'" % (fn,f[:pi[-2]],date)
+        os.system(c)
+        c="mv '%sN_F.TXT' '%s/N72KH_EFIS_%s/'" % (fn,f[:pi[-2]],date)
+        os.system(c)
+        c="mv '%sN_G.TXT' '%s/N72KH_GPS_%s/'" % (fn,f[:pi[-2]],date)
+        os.system(c)
+ 
+def process(date,logs):
+    if logs=='all' or logs=='ems':
+        print("processing EMS")
+        #sp="../N72KH Data/flights/%s/N72KH_EIS_%s/" %(date,date) # GRT EIS
+        sp="../N72KH Data/flights/%s/N72KH_EMS_%s/" %(date,date) # Dynon EMS
+        fl=glob.glob(sp+"LOG0*.TXT")
+        for f in fl:
+            fn=f.split("/")[-1].split(".")[0] # filename
+            # s="python ./eis_bin2csv.py '%s%s.TXT' > '%s%s.csv'" % (sp,fn,sp,fn) # GRT EIS
+            s="python ./ems2csv.py '%s%s.TXT' > '%s%s.csv'" % (sp,fn,sp,fn) # Dynon EMS
+            os.system(s)
+
+    if logs=='all' or logs=='efis':
+        print("processing EFIS")
+        sp="../N72KH Data/flights/%s/N72KH_EFIS_%s/" %(date,date)
+        fl=glob.glob(sp+"LOG0*.TXT")
+        for f in fl:
+            fn=f.split("/")[-1].split(".")[0] # filename
+            s="python ./efis2csv.py '%s%s.TXT' > '%s%s.csv'" % (sp,fn,sp,fn)
+            os.system(s)
+
+    if logs=='all' or logs=='gps':
+        print("processing GPS")
+        sp="../N72KH Data/flights/%s/N72KH_GPS_%s/" %(date,date) # path
+        fl=glob.glob(sp+"LOG0*.TXT") # file list
+        for f in fl:
+            fn=f.split("/")[-1].split(".")[0] # filename
+            s="python ./gps_formatter.py '%s%s.TXT' > '%s%s.csv'" % (sp,fn,sp,fn)
+            os.system(s)
+            s="./gps_convert.sh \"%s%s.csv\"  \"%s%s.kml\"" % (sp,fn,sp,fn)
+            #print s
+            os.system(s)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Process all log files')
+    parser.add_argument('date',
+                       help='folder date (e.g. 2nov2017)')
+    parser.add_argument('logname', nargs='?', default='all',
+                       help='{gps | ems | efis | all}')
+
+    args = parser.parse_args()
+
+    prepare(args.date)
+    process(args.date,args.logname)
+
+
+
+if __name__=="__main__":
+    main()
+