#!/usr/bin/python VERSION = "2005.05.20a" ################################################################################ # Balloon TX UnPack And PreProcess # written by Michael Stock for Langmuir Labs # # Reads in data from John Battle's programs (prom_adtest and balloon_tx) # which are files in .edat format. # Average specified CHANNELS over specified intervals, # Extrapolate time information, and output data (voltages read from the # Diamond Prometheus PC 104 A2D inputs) as csv (comma separated values). # # The most significant function of this program is to take the time data that is # interlaced through the raw binary data and give each data point a time in microseconds # This program allegedly fixed the known bugs in the microsecond data and the epoch data # but this fix has not been verified as of 5/20/2005. ################################################################################ # # USAGE: # Here is an example of a Bash script that uses BalloonTX_UPAPP to convert some .edat files to some .csv files # # ./BalloonTX_UPAPP.py -v -i /DATA/20040818/RAW_DATA/Esonde03_08182004_1947_001.edat -f /DATA/20040818/ASCII_DATA/Esonde03_08182004_1947.csv # ./BalloonTX_UPAPP.py -v -i /DATA/20040818/RAW_DATA/Esonde03_08182004_1948_001.edat -f /DATA/20040818/ASCII_DATA/Esonde03_08182004_1948.csv # ################################################################################ # # TIMING INFO: (Execution times measured on Feynman -- Averaging really slows things down #%AVERAGE : [1, 1, 1, 1, 1, 300, 300, 100] # processing packet 13 of 13 Calculating Time with PPS input # 191.381574 seconds elapsed during program operation #%AVERAGE : [1, 1, 1, 1, 1, 100, 100, 100] # 146.282315 seconds elapsed during program operation #%AVERAGE : [1, 1, 1, 1, 1, 1, 1, 1] # 119.995938 seconds elapsed during program operation # ################################################################################# # OUTPUT: # Here is (the beginning of) a sample output file: # (Real output files are often 20-40 MBytes long) ################################################################################ #% File Created by BalloonTX_UPAPP, written by Michael Stock #% Version :: 2005.05.20 #% #% This file was created on Thu May 19 23:11:30 2005 (System Time) #% First packet starts on Wed Aug 18 19:48:03 2004 GMT #% 2004.08.18 19:48:03 GMT #% 1092858483 Epoch #% #% #% ERROR CODES ----------------------------------------------------------------- #% these show up in the error flag column, the last column, # # ... # comment lines omitted here # ... #%OPERATION SETTINGS #% #%POSITION_START : 0 #%POSITION_END : 0 #%A2D_OFFSET : [0, 0, 0, 0, 0, 0, 0, 0] #%AVERAGE : [1, 1, 1, 1, 1, 300, 300, 100] #%PPS_PEAK_VOLTAGE : 4.88 #% #% seconds, micro seconds, E Field 0, E Field 1, E Field 2, E Field 3, PPS, Mag X, Mag Y, Mag Z, Error Code #1092858483, 3173445, 0.039062, 0.039673, -0.082397, 0.458679, 0.019836, 0.340576, 1.009521, 1.502686, 00 #1092858483, 3173545, 0.039673, 0.038757, -0.082397, 0.459290, 0.021362, 0.341797, 1.009674, 1.502838, 00 #1092858483, 3173645, 0.040894, 0.040283, -0.081177, 0.459290, 0.021362, 0.341390, 1.009115, 1.502380, 00 # ################################################################################ # Balloon TX UnPack And PreProcess # # version history # 2004.04.18 first version, ported from unpack_data.py # bug: read() doesn't read entire file in windows, works under linux though. # 2004.04.20 the program functions # bug: no AVERAGE, filling in time doesn't work # 2004.04.23 fixed FORMAT_STRING error # added AVERAGE bug: filling in time still doesn't work # 2004.05.17 addad writing support for CHANNELS != 8 # added flag for voltage output # 2004.05.19 fixed 05.17 additions # 2004.06.03 updated configuration for balloon_tx output # 2004.06.07 minor changes to configuration # OVERFLOW_SIZE = 4 # changed some numbers to ** format # seems to correctly unpack balloon_tx output added some comments # added no longer processes pakcets with an overflow # 2004.06.10 fixed bug in john's balloon_tx, and got compatability confirmed # 2004.06.15 Started adding a UI to the program # Added: # SKIP_OVERFLOW flag # break_flag flag in main() # error handling for time header mismatches # A2D_OFFSET # Fixed: # bug in csv output that adds an extra ',' at end # ErrorOut can now terminate program # 2004.06.16 Afternoon 06.15 changes and morning 06.16 changes # option handling working for: # flags # help # 2004.06.17 More option handling for: # A2D_OFFSET (takes list) # AVERAGE (takes list) # Removed Break flag (it wasn't being used) # Added POSITION_START and POSITION_END # !!Additions seemed to be written to 06.16 file # !!06.17 file is missing them # 2004.06.18 Finished basic UI # More Option Handling for: # input file # output file # 2004.06.25 there were some changes made on 06.21 that are # unaccounted for. included error handling in # time fillin code and micro second output # cleaning up global definitions and conventions # 2004.07.07 added error grabbing in PPS time calculation # 2004.07.08 overwrites output file if it exists, instead of appending to it # fixed bug in the PPS time calculation, time now # correctly calculated off PPS (pulses come on even # seconds, and pulses between packets are separated by # even number of seconds) # 2004.07.21 not much has been done. there is now a PPS option # 2004.07.23 Mike didn't document what he changed here, but this was the standard program # used for all data up through 1/1/2005 # 2004.09.21 minor changes so that the program can be loaded as a module # 2004.09.28 adding in the ability to calculate posistion from mag data # 2004.12.03 Added in Error Codes # Increased the PPS peak Voltage so that the PPS calc # wouldn't error, which seems to have worked # this will alter the times slightly # 2004.12.03 Appended: fixed the epoch time so that it flips over # after an even second # 2004.12.09 Changed the micro second time so that it starts at the # beginning of an even second # Added some more information to the header while i was at it # 2005.05.20 R.Sonnenfeld Changed the version string # added some comments and fixed spelling of error messages. # This version was used to reprocess most of the summer04 data. ################################################################################ from struct import unpack #for unpacking the binary data import math import sys #for option handling and program termination import getopt #for option handling import time import string TRUE = 1 FALSE = 0 # File Name Variables ---------------------------------------------------------- # change these to point to the apropriate files you want the program to act on # # the input file with the binary data from battle's ad_test program LOG_FILE = 'xxx.edat' # the output file. writing the file happens in append mode, which means if the # file already exists, then the program will add more lines to the end of the # file. if you desire a new file to be made, remove or move the old one. OUT_FILE = 'xxx.csv' # Program Operation Variables -------------------------------------------------- # these are flags, set TRUE or FALSE, they may change often DEBUG = FALSE # DEBUGging, lots of information printed to std out VERBOSE = FALSE # VERBOSE, less information printed to std out SKIP_OVERFLOW = TRUE # set this to skip processing packets that have an overflow A2D_VOLTAGE_OUT = TRUE # if this is false, program outputs raw integer values # from the A2D, otherwise calculates the float votages POSITION_START = 0 # start processing position in bytes POSITION_END = 0 # end processing position in bytes, 0, 'end' or 'END' # for end of file # these are channel specific variables, taken as a list which should have # a length equal to the number of A2D CHANNELS # hopefully these won't change often A2D_OFFSET = [0,0,0,0,0,0,0,0] # description below # A2D_OFFSET[x] gets subtracted off the value of channel x's input value # before it gets converted to a voltage. (the values are in A2D counts, # not volts and should therefor be signed integers) # this should end up being prometeus board specific AVERAGE = [1,1,1,1,1,100,100,100] # description below # number of samples to average over for # sample (in a list, should be 'CHANNELS' long # 1 should turn off AVERAGE # the only CHANNELS that use AVERAGE are the # mag CHANNELS (5,6,7) # MAG VARIABLES ---------------------------------------------------------------- MAG_PHI_0 = 0.0 MAG_THETA_0 = 0.0 MAG_CHANNELS = [5,6,7] # Program Format Variables ----------------------------------------------------- # these are STATIC variables that define the format the data comes in, and # therefore SHOULDN"T change # DON"T MESS WITH THESE UNLESS YOU KNOW WHAT YOU"RE DOING # RANDOM CHANGES OF THESE VARIABLES WILL LIKELY CAUSE THIS PROGRAM TO # THROW ERRORS AND NOT UNPACK THE DATA CORRECTLY # for ad_test output #CHANNELS = 8 # the number of data CHANNELS we are unpacking #FORMAT_STRING = 'todddddddd' #'t' is a time stamp, 'd' is data, # #'o' is overflow #SPS = 10000 # samples per second #SPP = 10000 # samples per data packet (should be the same as SPS if 1 # # second of data is sent in each packet) #DATA_SIZE = 20000 # size of data for 1 channel in 1 packet #TIME_SIZE = 16 # size of a single time stamp (should be 20) #OVERFLOW_SIZE = 4 # size of overflow thing #PACKET_SIZE = 160020 # size of 1 full packet #DEL_T = 1000000000/SPS # delta t, the time change between samples # # in nano-seconds # for balloon_tx output CHANNELS = 8 # the number of data CHANNELS we are unpacking FORMAT_STRING = 'toddtddtddtdd' #'t' is a time stamp, 'd' is data, #'o' is overflow SPS = 10000 # samples per second, FAD sample speed SPP = 20000 # samples per data packet (should be the same as SPS if 1 # second of data is sent in each packet) DATA_SIZE = 2*SPP # size of data for 1 channel in 1 packet TIME_SIZE = 16 # size of a single time stamp (should be 20) OVERFLOW_SIZE = 4 # size of overflow thing PACKET_SIZE = 320068 # size of 1 full packet DEL_T = 10**9/SPS # delta t, the time change between samples # in nano-seconds # *** PPS variables *** PPS_CHANNEL = 4 # the channel that pps data comes in on (0-7) # set to 'NONE' to turn off pps time extrapolation PPS_PEAK_VOLTAGE = 4.88 # the peak voltage of the pps peak PPS_UPPER_THESH = 0.80 # fraction of the PPS_PEAK_VOLTAGE that PPS_LOWER_THESH = 0.07 # voltages below this are considered noise PPS_R = 4990 # the Resistance of the Resistor in the Filter in Ohms PPS_C = 1.5*10**-8 # the Capacitance of the Capacitor in the Filter # in Farads # Global Variable Initialization ----------------------------------------------- # these are so that these variables don't have to be passed # they are definitely extraneous, since they don't get passed to main() and so # can only be read by functions that don't define another variable with the same # name IN_DATA = '' # input data OUT_DATA = range(CHANNELS+2) # output data ERROR_CODES = \ """% % ERROR CODES ----------------------------------------------------------------- % these show up in the error flag column, the last column, % 0 no error % 1 PPS not over threshold % 2 PPS Calculation Failed % 4 overflow occured % 8 time header mismatch % the flag column has the sum of these values, so % 3 --> PPS not over threshold & PPS Calculation Failed % """ def main(POSITION_START, POSITION_END, IN_DATA): DebugOut('program start') #----------------------------------------------------------------------- # there's a bug in the program here. f.read() is supposed to read to # the EOF (end of file), but in windows it doesn't. it works fine in # linux, so after some failed attempts to get it to work, i gave in and # left it that way. there's not much incentive to get it to work in # windows anyway, due to the extreme lack of windows machines @ langmuir #----------------------------------------------------------------------- # this gets set true the first time the output file is written to created_output_file = FALSE # check packet size # if this check fails, it calls an error, and continues with the user # defined value, don't expect the program to return meaningful data if # it calls an error PACKET_SIZE_ = 0 for tag in FORMAT_STRING: if tag == 't': PACKET_SIZE_ += TIME_SIZE if tag == 'd': PACKET_SIZE_ += DATA_SIZE if tag == 'o': PACKET_SIZE_ += OVERFLOW_SIZE if PACKET_SIZE_ != PACKET_SIZE: ErrorOut('PACKET_SIZE does not matched check', 1) #load the log file output by prom_ad_test.c VerboseOut("Opening File : " + LOG_FILE) try: f = open(LOG_FILE, 'r') #opening in read mode IN_DATA = f.read() f.close() except: ErrorOut('input file doesn\'t seem to exist', 0) #determine length of file in seconds of data #len(IN_DATA) is the size of the file in bytes # integer number of packets check # if the check fails, num_packets should be the floor of the fraction if len(IN_DATA) % PACKET_SIZE != 0: ErrorOut('file size not integer number of packets', 0) num_packets = len(IN_DATA)/PACKET_SIZE DebugOut('there are %i packets of data' % (num_packets)) # POSITION_START should be an integer packet number too if POSITION_START % PACKET_SIZE != 0: ErrorOut('processing must start at the beginning of a packet',0) # initialize iterators i = POSITION_START / PACKET_SIZE # iterators for number of packets position = POSITION_START # the position in the data file, in bytes # the ending position if POSITION_END == 'end' or POSITION_END == 0: #then we goto the end of the file POSITION_END = len(IN_DATA) #this is so the times that get filled in will be always increasing #within 1 file worth of data first_packet_time = -1 #gets set to the start second of the #first packet # cycle through the packets while i < num_packets: #this will loop from i=0 to i=num_packets-1 error_code = [0] #initialize the Error code for this packet #check to see if we've completed processing if position >= POSITION_END: Program_End() VerboseOut('processing packet %i of %i' % (i+1, num_packets)) channel = 0 #the channel we're pulling data from time_struct = [] #initialize the time_struct # cycle through the format string # a string is just a list, so we can use this for construction # which is really easy to code, and should be pretty fast Over_Flow = FALSE #overflow tag, like break_flag, but only #this packet will terminate #also handled mismatched time headers #(usually due to packet corruption which #is the same as an overflow) POSITION_START = position # marks beginnign of packet incase # of overflow for tag in FORMAT_STRING: # data or time? if tag == 't': #then it's a time thing DebugOut('working on time header') #append time info to time header, #shows up at the front of the list, but #all the time headers should be the same, so #it shouldn't matter (time_struct[:0],) = \ Get_Time_Header(position, IN_DATA) position += TIME_SIZE elif tag == 'o': #then it's a overflow thing DebugOut('warning on overflow tag') (overflow,) = Get_Overflow(position, IN_DATA) #test for overflow if overflow != 0: #there's something amiss ErrorOut('overflow error at ' +\ 'position %i :: %i' % \ (position,overflow),1) if 4 not in error_code: error_code.append(4) #this will skip the rest of the #processing if enabled if SKIP_OVERFLOW: Over_Flow = TRUE #set flag break position += OVERFLOW_SIZE elif tag == 'd': #then it's a data thing DebugOut('working on unpacking channel %i' \ %(channel)) #get the channel's data OUT_DATA[channel+2] = \ Get_Channel_Data(position, IN_DATA, \ channel) #apply the AVERAGE filter OUT_DATA[channel+2] = \ Average(OUT_DATA[channel+2],channel) #increment the channel channel += 1 position += DATA_SIZE else: #then i don't know what it is ErrorOut('unknown tag type',0) #check to see if an overflow's been thrown, if so skip rest of #processing if Over_Flow: i += 1 #sets the iterator position = POSITION_START + PACKET_SIZE #gets the position back on track continue # Fill in Time Data if PPS_CHANNEL != 'NONE': #this fills in time corrected based on PPS data from #the GPS #only send the data with the pps signal in it (OUT_DATA[0],OUT_DATA[1],Over_Flow,first_packet_time, t_error)= \ Fill_In_PPS_Time(time_struct, \ OUT_DATA[PPS_CHANNEL+2], Over_Flow, \ first_packet_time) else: # this won't be accurate, but it fills in something (OUT_DATA[0],OUT_DATA[1],Over_Flow,first_packet_time, t_error)= \ Fill_In_Time_Data(time_struct, Over_Flow, \ first_packet_time,[0]) for ec in t_error: if ec not in error_code: error_code.append(ec) # this if statement skips processing this packet if the overflow # tag was set above. Overflows are common. if Over_Flow: i += 1 #sets the iterator position = POSITION_START + PACKET_SIZE #gets the position back on track continue i += 1 #incriments i #write csv data #basically write the values with comma's between them #no header is written yet #loop through the seconds of data DebugOut('writing data to %s' % (OUT_FILE)) #check if we're writing floats (voltage) or counts (raw A2D) if not created_output_file: Create_Output_File(OUT_DATA[0][0],1) created_output_file = TRUE Write_Data(OUT_DATA, error_code, 1) Program_End() #------------------------------------------------------------------------------- # Averages the data according to the AVERAGE variable (global) # # the AVERAGE variable is a list of numbers, length = CHANNELS (if there are # 5 CHANNELS of data, there needs to be 5 numbers in the list). if you do # not want to average the data on a channel, set it's value to 1. EX, if # there are 3 CHANNELS, 1 & 2 averaged over 100 samples and 3 not averaged, # then AVERAGE = [100,100,1] #------------------------------------------------------------------------------- def Average(OUT_DATA,channel): DebugOut('working on averaging channel %i' % channel) boxcar = range(AVERAGE[channel]) #check to see if AVERAGE = 1 if AVERAGE[channel] == 1: #then we don't average this channel #print channel_data return OUT_DATA else: #we need to average the data i = 0 #the first AVERAGE samples summation = 0 channel_data = [] while i < AVERAGE[channel] - 1: #add current sample to sum summation += OUT_DATA[i] #fill in the boxcar boxcar[i] = OUT_DATA[i] #divide by i to get average value = summation/(i+1) channel_data[i:] = (value ,) #increment i += 1 #the middle samples while i <= SPP - AVERAGE[channel]: #fill in apropriate term of the boxcar boxcar[(i)%AVERAGE[channel]] = OUT_DATA[i] #calculate ave value, stick in OUT_DATA value = sum(boxcar) / AVERAGE[channel] channel_data[i:] = (value ,) #increment i += 1 #the last AVERAGE samples while i < SPP: #the last samples #sum the tail of channel_data #divide sum by # iterations left mod len(boxcar) value = \ sum(OUT_DATA[i:])/((SPP-i) % AVERAGE[channel]) channel_data[i:] = (value ,) i += 1 #print OUT_DATA return channel_data #------------------------------------------------------------------------------- # fills in the values for time based on the initial time stamp by adding # DEL_T to the start time in the time header. # # also deos error checking to make sure the end time matches up and that # different time headers don't claim different times (they shouldn't) #------------------------------------------------------------------------------- def Fill_In_Time_Data(time_struct, Over_Flow, first_packet_time, pps_error): #check to make sure all the time headers are the same in this packet VerboseOut('Calculating Time without PPS input') st_time_s = time_struct[0] #seconds st_time_n = time_struct[1] #nanoseconds nd_time_s = time_struct[2] #seconds nd_time_n = time_struct[3] #nanoseconds if first_packet_time < 0: first_packet_time = st_time_s #convert the first packet time to be an even minute, and ave the old value first_packet_mod60 = first_packet_time - (first_packet_time % 60) i = 0 t_error = FALSE while i < len(time_struct)-1: if st_time_s != time_struct[i]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE if st_time_n != time_struct[i+1]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE if nd_time_s != time_struct[i+2]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE if nd_time_n != time_struct[i+3]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE i+=4 if t_error: if 8 not in pps_error: pps_error.append(8) cur_sec = st_time_s #the bit at the end sets 0 us to the time of the first packet cur_mic = st_time_n/1000 + (st_time_s-first_packet_mod60)*10**6 str_mic = (st_time_s-first_packet_mod60)*10**6 seconds = [] usecond = [] i = 0 while i < SPP: #the number of seconds that has elapsed is del_s = (cur_mic - str_mic) / 10**6 #integer math) usecond[i:] = (cur_mic,) cur_mic += DEL_T/1000 seconds[i:] = (cur_sec + del_s,) i += 1 #this is a bit of a hack, if one of the headers didn't match, #the overflow flag is set, causing the program to skip processing #the rest of the packet #but, if this happens, it does not mean there was an overflow, more #likely the packet became corrupted in transfer or the setting were #not set right if SKIP_OVERFLOW and Over_Flow: return (seconds , usecond, TRUE, first_packet_time, pps_error) else: return (seconds , usecond, FALSE, first_packet_time, pps_error) def Fill_In_PPS_Time(time_struct, pps_signal, Over_Flow, first_packet_time): VerboseOut('Calculating Time with PPS input') #check to make sure all the time headers are the same in this packet st_time_s = time_struct[0] #seconds st_time_n = time_struct[1] #nanoseconds nd_time_s = time_struct[2] #seconds nd_time_n = time_struct[3] #nanoseconds if first_packet_time < 0: first_packet_time = st_time_s #convert the fisrt packet time to be an even minute, and ave the old value first_packet_mod60 = first_packet_time - (first_packet_time % 60) i = 0 t_error = FALSE #this is the error flag pps_error = [0] #this is the code returned while i < len(time_struct)-1: if st_time_s != time_struct[i]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE if st_time_n != time_struct[i+1]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE if nd_time_s != time_struct[i+2]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE if nd_time_n != time_struct[i+3]: #there's a problem!!! ErrorOut('time headers do not match',1) Over_Flow = TRUE t_error = TRUE i+=4 if t_error and 8 not in pps_error: pps_error.append(8) #set the code t_error = FALSE #reset the flag #first, make sure the beginning of pps_signal isn't on a pulse i = 0 while i > 0.8*PPS_PEAK_VOLTAGE: i += 1 #search through the pps signal for the pulse #pulse is 100 ms wide, incriment by 75 ms #gotta reset i to 0 for more itterations #75 ms = 75*10**6 ns ms75 = int(75000000/DEL_T) GOOD_PPS = TRUE while GOOD_PPS: if i > SPP: #then there was no pulse ErrorOut('No PPS pulse detected',1) t_error = TRUE #use backup time method (seconds,usecond,Over_Flow,first_packet_time)= \ Fill_In_Time_Data(time_struct, Over_Flow, \ first_packet_time) GOOD_PPS = FALSE break elif pps_signal[i] > 0.8*PPS_PEAK_VOLTAGE: #found the pulse break else: #incriment by 75.0 ms i += ms75 if t_error: pps_error.append(1) #set the error code t_error = FALSE #reset the error flag #use Binary Search to fine the first point above 0.8*pps*voltage i_min = i - ms75 i_max = i while GOOD_PPS: if pps_signal[(i_min + i_max)/2] > 0.8*PPS_PEAK_VOLTAGE: #it's part of the pulse i_max = (i_min + i_max)/2 else: #(i_min + i_max)/2 <= 0.8*PPS_PEAK_VOLTAGE i_min = (i_min + i_max)/2 if i_min == i_max: #then we're done i = i_max break elif i_min == i_max - 1: #there might be an infinite loop here if pps_signal[i_max - 1] > 0.8* PPS_PEAK_VOLTAGE: i = i_max - 1 else: i = i_max break while GOOD_PPS: #get to the more linear part of the curve if possible if pps_signal[i-1] > 0.05*PPS_PEAK_VOLTAGE: #then we can us this point to calculate time #(above noise) i = i - 1 #calculate the time shift (see hard copy notes p 43 #time_shift *should* be positive #print pps_signal[i] try: time_shift = -PPS_R*PPS_C*math.log(1.0-pps_signal[i] \ /PPS_PEAK_VOLTAGE) except: #error, this equation didn't compute!!! GOOD_PPS = FALSE ErrorOut('PPS Calculation Failed',1) t_error = TRUE break #print time_shift #modular will = 0 if the start time is correct modular = (st_time_n + i*DEL_T - int(time_shift*(10**9))) \ % (10**9) #so, if it's correct, st_time_n will not change #we add 10**9 to get the inverse modular (if that makes sense, #see the hardcopy description on this) st_time_n = st_time_n - modular + 10**9 if st_time_n > 10**9: #it's more the 1 second st_time_s += 1 st_time_n = st_time_n - 10**9 #the actual time always comes after what the system time reads #because there is a delay, so we shouldn't have to worry about #negitive values for st_time_n but ... since there is error in #the clock of maybe a significant fraction of a second, if that #error is greater than the delay, it's possible that the data #set started !before! the time start stamp. unfortunetly, #there's no way for me to test for this. if it happens, this #data will be off by exactly 1 second #now finally fill in all the times cur_sec = st_time_s #the bit at the end sets 0 us to the time of the first packet cur_mic = st_time_n/1000 + (st_time_s-first_packet_mod60)*10**6 str_mic = (st_time_s-first_packet_mod60)*10**6 seconds = [] usecond = [] i = 0 while i < SPP: #the number of seconds that has elapsed is del_s = (cur_mic - str_mic) / 10**6 #integer math usecond[i:] = (cur_mic,) cur_mic += DEL_T/1000 seconds[i:] = (cur_sec + del_s,) i += 1 break if t_error: pps_error.append(2) if not GOOD_PPS: (seconds,usecond,Over_Flow,first_packet_time,pps_error) = \ Fill_In_Time_Data(time_struct,Over_Flow,first_packet_time,pps_error) if SKIP_OVERFLOW and Over_Flow: return (seconds , usecond, TRUE, first_packet_time,pps_error) else: return (seconds , usecond, FALSE, first_packet_time,pps_error) #------------------------------------------------------------------------------- # this is gonna calculate position from the magnetometer data # takes *1* line of data, not the whole data structure, with just 3 values # returns 3 values (hor angle, vert angle and magnitude) #------------------------------------------------------------------------------- def Calc_Orient(mag_data): x = mag_data[0] y = mag_data[1] z = mag_data[2] phi = math.acos(z) # theta_x = arccos ( x/sin(phi) ) x = x / math.sin(phi) # rounding error fix, this *should* be excedingly rare, but since # we divide by sin(phi), it's possible that a rounding error causes # x to be outside of the domain of arccos (+/- 1) if x*x > 1: # we're out of the domain if x*x - 1 < .01: DebugOut('rounding error x') # y is very close to 1, so it's probably an issue # with the rounding of floadting point if x < 0: x = -1.0 else: x = 1.0 else: # error suck suck, this means the code is really doing # something wrong. ErrorOur('x channel is out of the domain of arccos',0) if x >= 0 and y >= 0: #QUAD I theta = theta_x elif y > 0: # then x < 0 #QUAD II theta = theta_x elif x > 0: # then y < 0 #QUAD IV theta = 2 * math.pi - theta_x else: # then x.y < 0 #QUAD III theta = 2 * math.pi - theta_x #------------------------------------------------------------------------------- # the ass backwards time header. the time format is stored as 2 (32 b?) # integers instead of 1 (64 b?) integer. the first on is in seconds, the # second one is in nanoseconds (not that the prometheous has that much acuracy) # # it logs the start time, the end time and an overflow flag # # as of now, this information is not saved because it's not used, but in the # version of the program that works on real data, this will be important # # just like Get_Channel_Data, this takes the starting location of IN_DATA, # and gets the header assuming it starts from there #------------------------------------------------------------------------------- def Get_Time_Header(start_byte, IN_DATA): time_st_1 = 0 #start time part 1 time_st_2 = 0 #start time part 2 time_nd_1 = 0 #end time part 1 time_nd_2 = 0 #end time part 2 #will get the time stamp from the data #should be 4 4byte integers #this is using the unpack from the struct library #'l' stands for 'long integer' which is 4 bytes long (time_st_1,) = unpack('l', IN_DATA[start_byte:start_byte+4]) (time_st_2,) = unpack('l', IN_DATA[start_byte+4:start_byte+8]) (time_nd_1,) = unpack('l', IN_DATA[start_byte+8:start_byte+12]) (time_nd_2,) = unpack('l', IN_DATA[start_byte+12:start_byte+16]) #returns a tuple return ((time_st_1, time_st_2, time_nd_1, time_nd_2),) def Get_Overflow(start_byte, IN_DATA): #returns the overflow tag #overflow is either a 2byte # or a 4byte #, depending on version #of transmit program if OVERFLOW_SIZE == 2: return unpack('h', IN_DATA[start_byte:start_byte+2]) else: return unpack('l', IN_DATA[start_byte:start_byte+4]) #------------------------------------------------------------------------------- # the meat of the program def Get_Channel_Data(position, IN_DATA, channel): channel_data = [] k = 0 #this will get the sample values from 1 second of data #should be 10000 2 byte integers #k cycles to the decimated lines per second value while k < SPP: (value,) = \ unpack('h', IN_DATA[position:position + 2]) #the A2D_OFFSET is in A2D counts, so subtract it off now, #while it's still integer value = value - A2D_OFFSET[channel] # iterate position by size of 16b integer (in bytes) position += 2 # this converts to a voltage, it does not compenate for # voltage offsets # only converts if A2D_VOLTAGE_OUT is true if A2D_VOLTAGE_OUT: valuef = value * 1.0 / 3276.8 # if you want a2d counts, change valuef to value channel_data[k:] = (valuef,) else: #we're outputting integers (faster) channel_data[k:] = (value,) k += 1 return channel_data #------------------------------------------------------------------------------- # Creates the output file and writes the header to that file # # the mode variable is used to determine which type of file is going to be # written (in case the headers vary for certain file formats) def Create_Output_File(time_start_s, mode): """this creates the file that Mrite_Data writes to, it is in charge of writing the header for the output file""" # Clear output file, so that if the file exists, the data in the file # will be erased #this code pauses, but doesn't let you cancel, so it's commented #try: # #this checks to see if the file exists, then gives the user a # #chance to exit the program if they don't want to overwrite # #the file # f = open(OUT_FILE, 'r') # ErrorOut('output file already exists', 1) # print 'use crt-c to terminate program' # for i in range(0,5): # print 'deleting output file in %i seconds' % (5-i) # time.sleep(1) #except: # #do nothing # i = 0 #there's no except clause, if the above fails, then the file doesn't #exist (try fails) then the following creates the file, if the file #does exist (try passes), then the following erases the file f = open(OUT_FILE, 'w') date_string = str(time.gmtime(time_start_s)[0]) + '.' +\ string.zfill(str(time.gmtime(time_start_s)[1]),2)+'.'+\ string.zfill(str(time.gmtime(time_start_s)[2]),2)+' '+\ string.zfill(str(time.gmtime(time_start_s)[3]),2)+':'+\ string.zfill(str(time.gmtime(time_start_s)[4]),2)+':'+\ string.zfill(str(time.gmtime(time_start_s)[5]),2) #!!! MODE 1 HEADER # writes: # date output file was created (human readable) # date data was taken at (2x human readable forms, and epoch time) # format string for the data if mode == 1: header = \ '% File Created by BalloonTX_UPAPP, written by Michael Stock\n' +\ '% Version :: ' + VERSION +\ '\n%\n' +\ '% This file was created on \t' +\ time.asctime(time.localtime()) + '\t(System Time)\n' +\ '% First packet starts on\t' +\ time.asctime(time.gmtime(time_start_s)) + '\tGMT\n' +\ '%\t\t\t\t' + date_string + '\t\tGMT\n' +\ '%\t\t\t\t' + str(time_start_s) + '\t\t\tEpoch\n' +\ '%\n' +\ ERROR_CODES +\ """% %TIME NOTES % %Micro Second Time starts from the begining of the minute the the data began on, so if the first packet starts at 13:42:01 %the micro seconds is micro seconds since 13:42:00 THE FIRST PACKET DOES NOT ALWAYS START ON AN EVEN MINUTE % %The Seconds Time is Epoch seconds since long ago (1970?), a standard Unix date string. For the most part, it's extraneous %since the time of the first packet is recorded in the header. It's been left in for compatability with other existing scripts. % %OPERATION SETTINGS % """ +\ '%POSITION_START : ' + str(POSITION_START) + '\n' + \ '%POSITION_END : ' + str(POSITION_END) + '\n' +\ '%A2D_OFFSET : ' + str(A2D_OFFSET) + '\n' +\ '%AVERAGE : ' + str(AVERAGE) + '\n' +\ '%PPS_PEAK_VOLTAGE : ' + str(PPS_PEAK_VOLTAGE) + '\n' +\ '%\n' +\ '% seconds, micro seconds, E Field 0, E Field 1, E Field 2, E Field 3, PPS, Mag X, Mag Y, Mag Z, Error Code\n' f.write(header) f.close() def Write_Data(OUT_DATA,error_code,mode): j = 0 f = open(OUT_FILE, 'a') while j < SPP: i = 0 write_string = '' if A2D_VOLTAGE_OUT: while i < 2: #for the time entries write_string += '%i, ' % OUT_DATA[i][j] i += 1 while i < CHANNELS+1: #for the voltage entries write_string += '%f, ' % OUT_DATA[i][j] i += 1 #this is the last sample, doesn't have ', ' at the end write_string += '%f, ' % OUT_DATA[i][j] else: while i < CHANNELS+1: write_string += '%i, ' % OUT_DATA[i][j] i += 1 #this is the last sample, doesn't have ', ' at the end write_string += '%i, ' % OUT_DATA[i][j] write_string += string.zfill(str(sum(error_code)),2) write_string += '\n' f.write(write_string) j += 1 f.close() def DebugOut(string): #if DEBUGging is on, this prints the string if DEBUG: print string def VerboseOut(string): #if DEBUGging or VERBOSE is on, this prints the string if VERBOSE or DEBUG: print string def ErrorOut(string,code): #ultra simplistic Error Handling sys.stderr.write(':::\n' + string + '\n:::\n') if code == 0: #terminal code terminate program Program_End() elif code == 1: #non-terminal code code = 1 #do nothing def Program_Start(): return time.time() def Program_End(): TIME_ELAPSED_END = time.time() elapsed_time = TIME_ELAPSED_END - TIME_ELAPSED_START VerboseOut('%f seconds elapsed during program operation' % \ elapsed_time) #sys.exit() TIME_ELAPSED_START = Program_Start() help_text = """python BalloonTX_UPAPP -hdvsV Version :: """ + VERSION + """ -h Display this help screen --help VERBOSE help (more help) -v VERBOSE Mode (info displayed) -d DEBUGging mode enabled (more info displayed) -o flips SKIP_OVERFLOW flag (default TRUE) -V flips A2D_VOLTAGE_OUT flag (default TRUE) -s start processing input at (must be beginning of a packet) -e stop processing input at -i input file (log file, binary) -f output file (csv file, ascii) -O [] offsets in A2D counts, separated by ,'s -A [] lengths of AVERAGE for each channel in #samples, separated by ,'s -P <0,channel> 0 turns PPS calc off, other sets channel to turn off all default processing, add these options -A [1,1,1,1,1,1,1,1] -P 0""" try: options, xarguments = getopt.getopt(sys.argv[1:],'hvdoVO:A:s:e:i:f:P:',\ ['help','VERBOSE', 'DEBUG', 'SKIP_OVERFLOW', \ 'A2D_VOLTAGE_OUT', 'A2D_OFFSET=', 'AVERAGE=', 'start=', \ 'end=', 'POSITION_START=', 'POSITION_END=', 'input_file=', \ 'in_file=', 'file_in=', 'LOG_FILE=', 'output_file=', \ 'OUT_FILE=', 'file_out=', 'start=', 'POSITION_START=', \ 'end=', 'POSITION_END=']) except getopt.error: #something went amiss getting options, ErrorOut(help_text,0) #flags and help for option in options[:]: if option[0] == '-h' or option[0] == '--help': #print help screen and exit print help_text Program_End() sys.exit() elif option[0] == '-v' or option[0] == '--VERBOSE': #flip VERBOSE flag if VERBOSE == TRUE: VERBOSE = FALSE else: VERBOSE = TRUE options.remove(option) elif option[0] == '-d' or option[0] == '--DEBUG': #flip DEBUGging flag if DEBUG == TRUE: DEBUG = FALSE else: DEBUG = TRUE options.remove(option) elif option[0] == '--SKIP_OVERFLOW': #flip the skip overflow flag if SKIP_OVERFLOW == TRUE: SKIP_OVERFLOW = FALSE else: SKIP_OVERFLOW = TRUE options.remove(option) elif option[0] == '-V' or option[0] == '--A2D_VOLTAGE_OUT': #flip A2D voltage out flag if A2D_VOLTAGE_OUT == TRUE: A2D_VOLTAGE_OUT = FALSE else: A2D_VOLTAGE_OUT = TRUE options.remove(option) #for AVERAGE and A2D_OFFSET (which are lists) for option in options[:]: if option[0] == '-O' or option[0] == '--A2D_OFFSET': #requires a list like [x,x,x,x ... x,x] #should be channel arguments in the list if option[1][0] != '[': #it's not a list in the correct format ErrorOut('A2D_OFFSET argument invalid',0) A2D_OFFSET_string = option[1][1:len(option[1])-1] A2D_OFFSET_string = A2D_OFFSET_string.split(',') if len(A2D_OFFSET_string) == CHANNELS: #then there are the correct number of entries i = 0 for offset in A2D_OFFSET_string: A2D_OFFSET[i] = int(offset) i+=1 else: #there are an incorrect number of offsets ErrorOut('incorrect number of offsets for A2D_OFFSET',0) options.remove(option) elif option[0] in ('-A', '--AVERAGE'): #requires a list just like before. the code to see if #options are pressent slightly different above, but does #the same thing #requires a list like [x,x,x,x ... x,x] #should be channel arguments in the list if option[1][0] != '[': #it's not a list in the correct format ErrorOut('AVERAGE argument invalid',0) #take the list given drop the '[' and ']' AVERAGE_string = option[1][1:len(option[1])-1] #split the input by ',' to get the values AVERAGE_string = AVERAGE_string.split(',') #check number of entries if len(AVERAGE_string) == CHANNELS: #then there are the correct number of entries i = 0 for length in AVERAGE_string: length = int(length) if length <= 0: ErrorOut(\ 'can not average over fewer than ' +\ '1 sample',0) AVERAGE[i] = length i+=1 else: #there are an incorrect number of offsets ErrorOut('incorrect number of lengths for AVERAGE',0) options.remove(option) #things that take arguments for o in options[:]: if o[0] in ('-s', '--start', '--POSITION_START'): #set position start POSITION_START = int(o[1]) elif o[0] in ('-e', '--end', '--POSITION_END'): #set position end if o[1] == 'end': POSITION_END = o[1] else: POSITION_END = int(o[1]) elif o[0] in ('-i', '--input_file', '--file_in', '--in_file', '--LOG_FILE'): #set the LOG_FILE (input) try: open(o[1]) except: error_string = 'input log file \"%s\" ' % (o[1]) +\ 'does\'t seem to exist!' ErrorOut(error_string , 0) #only get here if the error above doesn't get called LOG_FILE = o[1] elif o[0] in ('-f', '-o' , '--OUT_FILE', '--file_out', '--output_file'): #set the OUT_FILE (output) OUT_FILE = o[1] elif o[0] in ('-P'): #set PPS if o[1] == 1: PPS_CHANNEL = 'NONE' else: PPS_CHANNEL = o[1] if __name__ == "__main__": main(POSITION_START, POSITION_END, IN_DATA)