Merge pull request #8 from nzkarit/FR24-CSV

FR24 csv

.gitignore

@@ -1,6 +1,10 @@
 *.iq8s
 *.pyc
 *.log*
+*.csv
+
+# Keep the example CSV
+!example.csv
 
 # Don't include the generated CSV related files
 generated/

ADSB_Encoder.py

@@ -60,11 +60,10 @@ def argParser():
     parser.add_argument('-s', '--surface', action='store', default=cfg.getboolean('plane', 'surface'), type=auto_bool, dest='surface', help='If the plane is on the ground or not. Default: %(default)s')
     parser.add_argument('-o', '--out', '--output', action='store', type=str, default=cfg.get('general', 'outputfilename'), dest='outputfilename', help='The iq8s output filename. This is the file which you will feed into the hackRF. Default: %(default)s')
     parser.add_argument('-r', '--repeats', action='store', dest='repeats', type=int, default=cfg.getint('general', 'repeats'), help='How many repeats of the data to perform. Default: %(default)s')
-    parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, default=cfg.get('general', 'csvfile'), dest='csvfile', help='Import a CSV file with the plane data in it. Default: %(default)s')    
+    parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str, default=cfg.get('general', 'csvfile'), dest='csvfile', help='Import a CSV file with the plane data in it. Default: %(default)s')
+    parser.add_argument('--intermessagegap', action='store', type=int, default=cfg.get('general', 'intermessagegap'), dest='intermessagegap', help='When repeating or reading a CSV the number of microseconds between messages. Default: %(default)s')
+    parser.add_argument('--realtime', action='store', default=cfg.getboolean('general', 'realtime'), type=auto_bool, dest='realtime', help='When running a CSV which has a timestamp column whether to run in realtime following the timestamp or if just follow intermessagegap. If realtime is set it will override intermessagegap. Default: %(default)s')
     # TODO Make it so it can do a static checksum
-    # TODO Get pause between messages
-    # TODO Get pause between repeats
-    # TODO Do a pause function
     return parser.parse_args()
 
 def singlePlane(arguments):
@@ -81,16 +80,21 @@ def singlePlane(arguments):
         hackrf = HackRF()
         samples_array = hackrf.hackrf_raw_IQ_format(df17_array)
         samples = samples+samples_array
+        gap_array = ppm.addGap(arguments.intermessagegap)
+        samples_array = hackrf.hackrf_raw_IQ_format(gap_array)
+        samples = samples+samples_array
     return samples
 
 def manyPlanes(arguments):
-    logger.info('Processing CSV file: %s' % (arguments.csvfile))    
+    logger.info('Processing CSV file: %s' % (arguments.csvfile))
     samples = bytearray()
     logger.info('Repeating the message %s times' % (arguments.repeats))
+    prevtimestamp = 0
     for i in range(0, arguments.repeats):
         with open(arguments.csvfile, newline='') as csvfile:
             reader = csv.DictReader(csvfile, delimiter=',')
-            for row in reader:        
+            for row in reader:
+                gap = arguments.intermessagegap
                 if not 'icao' in row.keys():
                     row['icao'] = arguments.icao
                 else:
@@ -119,6 +123,11 @@ def manyPlanes(arguments):
                     row['time'] = arguments.time
                 if not 'surface' in row.keys():
                     row['surface'] = arguments.surface
+                if 'timestamp' in row.keys():
+                    if arguments.realtime:
+                        gap = int(row['timestamp']) - prevtimestamp
+                        gap = gap * 100000
+                        prevtimestamp = int(row['timestamp'])
                 logger.debug('Row from CSV: %s' % (row))
                 modes = ModeS()
                 (df17_even, df17_odd) = modes.df17_pos_rep_encode(row['capability'], row['icao'], row['typecode'], row['surveillancestatus'], row['nicsupplementb'], row['altitude'], row['time'], row['latitude'], row['longitude'], row['surface'])
@@ -129,6 +138,9 @@ def manyPlanes(arguments):
                 hackrf = HackRF()
                 samples_array = hackrf.hackrf_raw_IQ_format(df17_array)
                 samples = samples+samples_array
+                gap_array = ppm.addGap(gap)
+                samples_array = hackrf.hackrf_raw_IQ_format(gap_array)
+                samples = samples+samples_array
     return samples
 
 def writeOutputFile(filename, data):
@@ -138,11 +150,11 @@ def writeOutputFile(filename, data):
     SamplesFile.write(data)
     SamplesFile.close()
     os.system('sync')
-    os.system('rm %s' % (filename)) 
+    os.system('rm %s' % (filename))
     logger.info('dd for file: %s' % (filename))
-    os.system("dd if=%s of=%s bs=4k seek=63 > /dev/null 2>&1" % (tmpfile, filename)) 
+    os.system("dd if=%s of=%s bs=4k seek=63 > /dev/null 2>&1" % (tmpfile, filename))
     os.system('sync')
-    os.system('rm %s'%(tmpfile))   
+    os.system('rm %s'%(tmpfile))
 
 def main():
     global cfg
@@ -188,6 +200,3 @@ def threadingCSV(csv):
 if __name__ == "__main__":
     main()
     
-
-
-

FR24csv.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+#
+# Will take a FR24 CSV and make a ADSB_Encoder CSV
+import csv
+import os
+import argparse
+import configparser
+
+def auto_int(x):
+    """Parses HEX into for argParser"""
+    return int(x, 0)
+
+def argParser():
+    description = 'This script will take a FR24 CSV file and convert it into a format for FR24csv.py'
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('-i', '--icao', action='store', type=auto_int, dest='icao', default=cfg.get('plane', 'icao'), help='The ICAO number for the plane in hex. Ensure the ICAO is prefixed with \'0x\' to ensure this is parsed as a hex number. This is 24 bits long. Default: %(default)s')
+    parser.add_argument('--csv', '--csvfile', '--in', '--input', action='store', type=str,  dest='csvfile', help='The name of the FR24 CSV file', required=True)  
+    return parser.parse_args()
+
+def reverseCSV(csvfile):
+    """Reverse a CSV. Returns a dictionary of the CSV"""
+    data = []
+    with open(csvfile, newline='') as csvfilein:
+        reader = csv.DictReader(csvfilein, delimiter=',')
+        for row in reader:
+            data.append(row)
+    csvfilein.close()
+    return reversed(data)
+
+def main():
+    global cfg
+    cfg = configparser.ConfigParser()
+    cfg.read('config.cfg')
+    
+    arguments = argParser()
+    
+    csvFilename = 'fr24.csv'
+    
+    time = 0
+    #Need to reverse the FR24 CSV as it is in reverse order i.e. the most recent record is row 2 and the first ADS-B message of the flight is the last row in the CSV
+    data = reverseCSV(arguments.csvfile)
+    with open(csvFilename, 'w', newline='') as csvfileout:
+        fieldnames = ['timestamp', 'icao', 'latitude', 'longitude', 'altitude']
+        output = csv.DictWriter(csvfileout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, fieldnames=fieldnames)
+        output.writeheader()
+        for row in data:
+            if time == 0:
+                time = int(row['Timestamp'])
+            rowtime = int(row['Timestamp']) - time
+            position = row['Position'].split(',')
+            newrow = {'timestamp':rowtime, 'icao':hex(arguments.icao), 'latitude':position[0], 'longitude':position[1], 'altitude':row['Altitude']}
+            output.writerow(newrow)
+        csvfileout.close()
+
+    
+if __name__ == "__main__":
+    main()

PPM.py

@@ -46,3 +46,16 @@ class PPM:
         #print '[{}]'.format(', '.join(hex(x) for x in ppm))
         
         return bytearray(ppm)
+        
+    def addGap(self, gap):
+        """
+        This function will add dead air as a gap between messages
+        Args:
+            gap: The number of microseconds to have as a gap
+        Returns:
+            The bytearray of the PPM data
+        """
+        ppm = [ ]
+        for i in range(gap):    # pause
+            ppm.append( 0 )
+        return bytearray(ppm)

README.md

@@ -103,6 +103,26 @@ These CSV files can be used for input into the application
 ## generateAllICAO.py
 This script will generate a CSV with all the different ICAO numbers in it.
 
+# Import FlightRadar24 CSV files
+This script will take a FR24 CSV files and convert it ready for import into ADSB_Encoder.py. It outputs a file called fr24.csv.
+```
+$ ./FR24csv.py --csv <exportFromFR24.csv>
+$ ./ADSB_Encoder.py --csv fr24.csv
+
+$ ./FR24csv.py --help
+usage: FR24csv.py [-h] [-i ICAO] --csv CSVFILE
+
+This script will take a FR24 CSV file and convert it into a format for
+FR24csv.py
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -i ICAO, --icao ICAO  The ICAO number for the plane in hex. Ensure the ICAO
+                        is prefixed with '0x' to ensure this is parsed as a
+                        hex number. This is 24 bits long. Default: 0x75008F
+  --csv CSVFILE, --csvfile CSVFILE, --in CSVFILE, --input CSVFILE
+                        The name of the FR24 CSV file
+```
 # References
 1. "*Gr-Air-Modes*", **Nick Foster**, 2012
 1. "*EXPLOITING THE AUTOMATIC DEPENDENT SURVEILLANCE BROADCAST SYSTEM VIA FALSE TARGET INJECTION*", **Domenic Magazu III**, 2012

config.cfg

@@ -1,7 +1,12 @@
 [general]
 outputfilename = Samples_256K.iq8s
 repeats = 1
-csvfile = 
+csvfile =
+# Currently in PPM.py one message is 48 dead air, 8 preamble, 112 message, 100 dead air, 8 preamble, 112 message, 48 dead air leaves us with 99564 microseconds to make a second
+intermessagegap = 99564
+# This is the message length from PPM.py
+messagelength = 436
+realtime = false
 
 [plane]
 icao = 0x75008F