Temp commit. Not yet working.

* PyQt4.QtSql's SQLite interface doesn't appear to return the same results as the SQLite browser. This is probably me depending on a bad data ordering assumption.
* QSqlQueryModel isn't set up to have the db change from underneath it, AFAIK -- have to add the ability to notify it there's new data.

apps/modes_gui

@@ -77,12 +77,12 @@ class mainwindow(QtGui.QMainWindow):
         self.queue = gr.msg_queue(10)
         self.running = False
         self.kmlgen = None #necessary bc we stop its thread in shutdown
-        self.dbname = "air_modes.db"
+        self.dbname = "adsb.db"
         self.num_reports = 0
         self.last_report = 0
         self.context = zmq.Context(1)
 
-        self.datamodel = dashboard_data_model(None)
+        self.datamodel = dashboard_sql_model(None)
         self.ui.list_aircraft.setModel(self.datamodel)
         self.ui.list_aircraft.setModelColumn(0)
 
@@ -96,15 +96,16 @@ class mainwindow(QtGui.QMainWindow):
         self.dashboard_mapper.setModel(self.datamodel)
         self.dashboard_mapper.addMapping(self.ui.line_icao, 0)
         #self.dashboard_mapper.addMapping(self.ui.prog_rssi, 2)
-        self.dashboard_mapper.addMapping(self.ui.line_latitude, 3)
-        self.dashboard_mapper.addMapping(self.ui.line_longitude, 4)
-        self.dashboard_mapper.addMapping(self.ui.line_alt, 5)
-        self.dashboard_mapper.addMapping(self.ui.line_speed, 6)
-        #self.dashboard_mapper.addMapping(self.ui.compass_heading, 7)
-        self.dashboard_mapper.addMapping(self.ui.line_climb, 8)
-        self.dashboard_mapper.addMapping(self.ui.line_ident, 9)
-        self.dashboard_mapper.addMapping(self.ui.line_type, 10)
-        self.dashboard_mapper.addMapping(self.ui.line_range, 11)
+        self.dashboard_mapper.addMapping(self.ui.line_latitude, 2)
+        self.dashboard_mapper.addMapping(self.ui.line_longitude, 3)
+        self.dashboard_mapper.addMapping(self.ui.line_alt, 4)
+        self.dashboard_mapper.addMapping(self.ui.line_speed, 5)
+        #self.dashboard_mapper.addMapping(self.ui.compass_heading, 6)
+        self.dashboard_mapper.addMapping(self.ui.line_climb, 7)
+        self.dashboard_mapper.addMapping(self.ui.line_ident, 8)
+        self.dashboard_mapper.addMapping(self.ui.line_type, 9)
+        #self.dashboard_mapper.addMapping(self.ui.line_range, 11)
+        #self.dashboard_mapper.addMapping(self.ui.compass_bearing, 12)
 
         compass_palette = QtGui.QPalette()
         compass_palette.setColor(QtGui.QPalette.Foreground, QtCore.Qt.white)
@@ -114,7 +115,8 @@ class mainwindow(QtGui.QMainWindow):
         self.ui.compass_heading.setNeedle(Qwt.QwtDialSimpleNeedle(Qwt.QwtDialSimpleNeedle.Ray, False, QtCore.Qt.black))
         self.ui.compass_bearing.setNeedle(Qwt.QwtDialSimpleNeedle(Qwt.QwtDialSimpleNeedle.Ray, False, QtCore.Qt.black))
 
-        #hook up the update signal
+        #hook up the update signals which are explicitly necessary
+        #most of the dashboard_mapper and list_aircraft stuff is implicitly done already
         self.ui.list_aircraft.selectionModel().currentRowChanged.connect(self.dashboard_mapper.setCurrentModelIndex)
         self.ui.list_aircraft.selectionModel().currentRowChanged.connect(self.update_heading_widget)
         self.ui.list_aircraft.selectionModel().currentRowChanged.connect(self.update_bearing_widget)
@@ -128,27 +130,27 @@ class mainwindow(QtGui.QMainWindow):
 ############ widget update functions for non-mapped widgets ############
     def update_heading_widget(self, index):
         if index.model() is not None:
-            heading = index.model().data(index.model().index(index.row(), self.datamodel._colnames.index("heading"))).toDouble()[0]
+            heading = index.model().data(index.model().index(index.row(), 6)).toDouble()[0]
             self.ui.compass_heading.setValue(heading)
 
     def update_bearing_widget(self, index):
         if index.model() is not None:
-            bearing = index.model().data(index.model().index(index.row(), self.datamodel._colnames.index("bearing"))).toDouble()[0]
+            bearing = 0#index.model().data(index.model().index(index.row(), 12)).toDouble()[0]
             self.ui.compass_bearing.setValue(bearing)
 
     def unmapped_widgets_dataChanged(self, startIndex, endIndex):
         index = self.ui.list_aircraft.selectionModel().currentIndex()
         if index.row() in range(startIndex.row(), endIndex.row()+1): #the current aircraft was affected
-            if self.datamodel._colnames.index("heading") in range(startIndex.column(), endIndex.column()+1):
+            if 6 in range(startIndex.column(), endIndex.column()+1):
                 self.update_heading_widget(index)
-            if self.datamodel._colnames.index("bearing") in range(startIndex.column(), endIndex.column()+1):
+            if 12 in range(startIndex.column(), endIndex.column()+1):
                 self.update_bearing_widget(index)
-            if self.datamodel._colnames.index("rssi") in range(startIndex.column(), endIndex.column()+1):
+            if 2 in range(startIndex.column(), endIndex.column()+1):
                 self.update_rssi_widget(index)
 
     def update_rssi_widget(self, index):
         if index.model() is not None:
-            rssi = index.model().data(index.model().index(index.row(), 2)).toDouble()[0]
+            rssi = 0#index.model().data(index.model().index(index.row(), 2)).toDouble()[0]
             self.ui.prog_rssi.setValue(rssi)
 
     def increment_reportspersec(self, msg):
@@ -277,8 +279,7 @@ class mainwindow(QtGui.QMainWindow):
                 my_position = None
 
             self._cpr_dec = air_modes.cpr_decoder(my_position)
-
-            self.datamodelout = dashboard_output(self._cpr_dec, self.datamodel, self._publisher)
+#            self.datamodelout = dashboard_output(self._cpr_dec, self.datamodel, self._publisher)
 
             self.lock = threading.Lock() #grab a lock to ensure sql and kml don't step on each other
 

python/gui_model.py

@@ -22,10 +22,11 @@
 # This file contains data models, view delegates, and associated classes
 # for handling the GUI back end data model.
 
-from PyQt4 import QtCore, QtGui
+from PyQt4 import QtCore, QtGui, QtSql
 import air_modes
 import threading, math, time
 from air_modes.exceptions import *
+from gnuradio.gr.pubsub import pubsub
 
 #fades the ICAOs out as their last report gets older,
 #and display ident if available, ICAO otherwise
@@ -37,12 +38,16 @@ class ICAOViewDelegate(QtGui.QStyledItemDelegate):
             painter.drawRect(option.rect)
 
         #if there's an ident available, use it. otherwise print the ICAO
-        if index.model().data(index.model().index(index.row(), 9)) != QtCore.QVariant():
-            paintstr = index.model().data(index.model().index(index.row(), 9)).toString()
+        if index.model().data(index.model().index(index.row(), 8)) != QtCore.QVariant():
+            paintstr = index.model().data(index.model().index(index.row(), 8)).toString()
         else:
             paintstr = index.model().data(index.model().index(index.row(), 0)).toString()
-        last_report = index.model().data(index.model().index(index.row(), 1)).toDouble()[0]
-        age = (time.time() - last_report)
+
+        #FIXME this is kind of heinous, find out how you got int data out of it last time
+        last_report = time.strptime(str(index.model().data(index.model().index(index.row(), 1)).toString()), "%Y-%m-%d %H:%M:%S")
+        age = (time.mktime(time.gmtime()) - time.mktime(last_report)) - 3600.*time.daylight
+        print age
+
         max_age = 60. #age at which it grays out
         #minimum alpha is 0x40 (oldest), max is 0xFF (newest)
         age = min(age, max_age)
@@ -50,152 +55,36 @@ class ICAOViewDelegate(QtGui.QStyledItemDelegate):
         painter.setPen(QtGui.QColor(0, 0, 0, alpha))
         painter.drawText(option.rect.left()+3, option.rect.top(), option.rect.width(), option.rect.height(), option.displayAlignment, paintstr)
 
-#the data model used to display dashboard data.
-class dashboard_data_model(QtCore.QAbstractTableModel):
+#class dashboard_sql_model(QtCore.QAbstractTableModel):
+#    def __init__(self, parent):
+#        QtCore.QAbstractTableModel.__init__(self, parent)
+
+#    def update(self, icao):
+
+#TODO must add libqt4-sql, libqt4-sql-sqlite, python-qt4-sql to dependencies
+#TODO looks like you're going to have to either integrate this into sql.py (ugh!) or find a way to keep it in sync
+#seems like it wants to have control over maintaining data currency
+#worst case is you make your own damn SQL query model based on abstracttablemodel.
+class dashboard_sql_model(QtSql.QSqlQueryModel):
     def __init__(self, parent):
-        QtCore.QAbstractTableModel.__init__(self, parent)
-        self._data = []
-        self.lock = threading.Lock()
-        self._colnames = ["icao", "seen", "rssi", "latitude", "longitude", "altitude", "speed", "heading", "vertical", "ident", "type", "range", "bearing"]
-        #custom precision limits for display
-        self._precisions = [None, None, None, 6, 6, 0, 0, 0, 0, None, None, 2, 0]
-        for field in self._colnames:
-            self.setHeaderData(self._colnames.index(field), QtCore.Qt.Horizontal, field)
-    def rowCount(self, parent=QtCore.QVariant()):
-        return len(self._data)
-    def columnCount(self, parent=QtCore.QVariant()):
-        return len(self._colnames)
-    def data(self, index, role=QtCore.Qt.DisplayRole):
-        if not index.isValid():
-            return QtCore.QVariant()
-        if index.row() >= self.rowCount():
-            return QtCore.QVariant()
-        if index.column() >= self.columnCount():
-            return QtCore.QVariant()
-        if (role != QtCore.Qt.DisplayRole) and (role != QtCore.Qt.EditRole):
-            return QtCore.QVariant()
-        if self._data[index.row()][index.column()] is None:
-            return QtCore.QVariant()
-        else:
-            #if there's a dedicated precision for that column, print it out with the specified precision.
-            #this only works well if you DON'T have other views/widgets that depend on numeric data coming out.
-            #i don't like this, but it works for now. unfortunately it seems like Qt doesn't give you a
-            #good alternative.
-            if self._precisions[index.column()] is not None:
-                return QtCore.QVariant("%.*f" % (self._precisions[index.column()], self._data[index.row()][index.column()]))
-            else:
-                if self._colnames[index.column()] == "icao":
-                    return QtCore.QVariant("%06x" % self._data[index.row()][index.column()]) #return as hex string
-                else:
-                    return QtCore.QVariant(self._data[index.row()][index.column()])
-
-    def setData(self, index, value, role=QtCore.Qt.EditRole):
-        self.lock.acquire()
-        if not index.isValid():
-            return False
-        if index.row() >= self.rowCount():
-            return False
-        if index.column >= self.columnCount():
-            return False
-        if role != QtCore.Qt.EditRole:
-            return False
-        self._data[index.row()][index.column()] = value
-        self.lock.release()
-
-    #addRecord implements an upsert on self._data; that is,
-    #it updates the row if the ICAO exists, or else it creates a new row.
-    def addRecord(self, record):
-        self.lock.acquire()
-        icaos = [x[0] for x in self._data]
-        if record["icao"] in icaos:
-            row = icaos.index(record["icao"])
-            for column in record:
-                self._data[row][self._colnames.index(column)] = record[column]
-            #create index to existing row and tell the model everything's changed in this row
-            #or inside the for loop, use dataChanged on each changed field (might be better)
-            self.dataChanged.emit(self.createIndex(row, 0), self.createIndex(row, len(self._colnames)-1))
-
-        #only create records for ICAOs with ADS-B reports
-        elif ("latitude" or "speed" or "ident") in record:
-            #find new inserted row number
-            icaos.append(record["icao"])
-            newrowoffset = sorted(icaos).index(record["icao"])
-            self.beginInsertRows(QtCore.QModelIndex(), newrowoffset, newrowoffset)
-            newrecord = [None for x in xrange(len(self._colnames))]
-            for col in xrange(0, len(self._colnames)):
-                if self._colnames[col] in record:
-                    newrecord[col] = record[self._colnames[col]]
-            self._data.append(newrecord)
-            self._data = sorted(self._data, key = lambda x: x[0]) #sort by icao
-            self.endInsertRows()
-        self.lock.release()
-        self.prune()
-
-    #weeds out ICAOs older than 5 minutes
-    def prune(self):
-        self.lock.acquire()
-        for (index,row) in enumerate(self._data):
-            if time.time() - row[1] >= 60:
-                self.beginRemoveRows(QtCore.QModelIndex(), index, index)
-                self._data.pop(index)
-                self.endRemoveRows()
-        self.lock.release()
-                
-class dashboard_output:
-    def __init__(self, cprdec, model, pub):
-        self.model = model
-        self._cpr = cprdec
-        pub.subscribe("modes_dl", self.output)
-    def output(self, msg):
-        try:
-            msgtype = msg.data["df"]
-            now = time.time()
-            newrow = {"rssi": msg.rssi, "seen": now}
-            if msgtype in [0, 4, 20]:
-                newrow["altitude"] = air_modes.altitude.decode_alt(msg.data["ac"], True)
-                newrow["icao"] = msg.ecc
-                self.model.addRecord(newrow)
-            
-            elif msgtype == 17:
-                icao = msg.data["aa"]
-                newrow["icao"] = icao
-                subtype = msg.data["ftc"]
-                if subtype == 4:
-                    (ident, actype) = air_modes.parseBDS08(msg.data)
-                    newrow["ident"] = ident
-                    newrow["type"] = actype
-                elif 5 <= subtype <= 8:
-                    (ground_track, decoded_lat, decoded_lon, rnge, bearing) = air_modes.parseBDS06(msg.data, self._cpr)
-                    newrow["heading"] = ground_track
-                    newrow["latitude"] = decoded_lat
-                    newrow["longitude"] = decoded_lon
-                    newrow["altitude"] = 0
-                    if rnge is not None:
-                        newrow["range"] = rnge
-                        newrow["bearing"] = bearing
-                elif 9 <= subtype <= 18:
-                    (altitude, decoded_lat, decoded_lon, rnge, bearing) = air_modes.parseBDS05(msg.data, self._cpr)
-                    newrow["altitude"] = altitude
-                    newrow["latitude"] = decoded_lat
-                    newrow["longitude"] = decoded_lon
-                    if rnge is not None:
-                        newrow["range"] = rnge
-                        newrow["bearing"] = bearing
-                elif subtype == 19:
-                    subsubtype = msg.data["sub"]
-                    velocity = None
-                    heading = None
-                    vert_spd = None
-                    if subsubtype == 0:
-                        (velocity, heading, vert_spd) = air_modes.parseBDS09_0(msg.data)
-                    elif 1 <= subsubtype <= 2:
-                        (velocity, heading, vert_spd) = air_modes.parseBDS09_1(msg.data)
-                    newrow["speed"] = velocity
-                    newrow["heading"] = heading
-                    newrow["vertical"] = vert_spd
-    
-                self.model.addRecord(newrow)
-
-        except ADSBError:
-            return
+        QtSql.QSqlQueryModel.__init__(self, parent)
+        self._query = """select tab1.icao, tab1.seen, tab1.lat, tab1.lon, tab1.alt, speed, heading, vertical, ident, type
+                         from (select * from (select * from positions order by seen desc) group by icao) tab1
+                    left join (select * from (select * from vectors order by seen desc)   group by icao) tab2
+                        on tab1.icao=tab2.icao
+                    left join (select * from (select * from ident)) tab3
+                        on tab1.icao=tab3.icao
+                    where tab1.seen > datetime('now', '-1 minute')"""
+        self._sql = None
+        self._db = QtSql.QSqlDatabase("QSQLITE")
+        self._db.setDatabaseName("adsb.db") #TODO specify this elsewhere
+        self._db.open()
+        #what is this i don't even
+        #fetches the combined data of all three tables for all ICAOs seen in the last minute.
+        #FIXME PyQt's SQLite gives you different results than the SQLite browser
+        self.setQuery(self._query, self._db)
 
+    #the big club
+    def update_all(self, icao):
+        self.setQuery(self._query, self._db)
+        #self.dataChanged.emit(self.createIndex(0, 0), self.createIndex(self.rowCount(), self.columnCount()))

python/sql.py

@@ -38,7 +38,7 @@ class output_sql:
     c = self._db.cursor()
     query = """CREATE TABLE IF NOT EXISTS "positions" (
               "icao" INTEGER KEY NOT NULL,
-              "seen" TEXT NOT NULL,
+              "seen" DATETIME NOT NULL,
               "alt"  INTEGER,
               "lat"  REAL,
               "lon"  REAL
@@ -46,7 +46,7 @@ class output_sql:
     c.execute(query)
     query = """CREATE TABLE IF NOT EXISTS "vectors" (
               "icao"     INTEGER KEY NOT NULL,
-              "seen"     TEXT NOT NULL,
+              "seen"     DATETIME NOT NULL,
               "speed"    REAL,
               "heading"  REAL,
               "vertical" REAL
@@ -96,42 +96,34 @@ class output_sql:
 
     return query
 
-#TODO: if there's a way to publish selective reports on upsert to distinguish,
-#for instance, between a new ICAO that's just been heard, and a refresh of an
-#existing ICAO, both of those would be useful publishers for the GUI model.
-#otherwise, worst-case you can just refresh everything every time a report
-#comes in, but it's going to use more CPU. Not likely a problem if you're only
-#looking at ADS-B (no mode S) data.
-#It's probably time to look back at the Qt SQL table model and see if it can be
-#bent into shape for you.
   def sql17(self, data):
     icao24 = data["aa"]
     bdsreg = data["me"].get_type()
     #self["bds%.2i" % bdsreg] = icao24 #publish under "bds08", "bds06", etc.
 
     if bdsreg == 0x08:
-      (msg, typename) = air_modes.parseBDS08(data)
-      return "INSERT OR REPLACE INTO ident (icao, ident, type) VALUES (" + "%i" % icao24 + ", '" + msg + "', '" + typename + "')"
+      (msg, typename) = self.parseBDS08(data)
+      return "INSERT OR REPLACE INTO ident (icao, ident, type) VALUES (%i, '%s', '%s')" % (icao24, msg, typename)
     elif bdsreg == 0x06:
       [ground_track, decoded_lat, decoded_lon, rnge, bearing] = air_modes.parseBDS06(data, self._cpr)
       altitude = 0
       if decoded_lat is None: #no unambiguously valid position available
         raise CPRNoPositionError
       else:
-        return "INSERT INTO positions (icao, seen, alt, lat, lon) VALUES (" + "%i" % icao24 + ", datetime('now'), " + str(altitude) + ", " + "%.6f" % decoded_lat + ", " + "%.6f" % decoded_lon + ")"
+        return "INSERT INTO positions (icao, seen, alt, lat, lon) VALUES (%i, datetime('now'), %i, %.6f, %.6f)" % (icao24, int(altitude), decoded_lat, decoded_lon)
     elif bdsreg == 0x05:
       [altitude, decoded_lat, decoded_lon, rnge, bearing] = air_modes.parseBDS05(data, self._cpr)
       if decoded_lat is None: #no unambiguously valid position available
         raise CPRNoPositionError
       else:
-        return "INSERT INTO positions (icao, seen, alt, lat, lon) VALUES (" + "%i" % icao24 + ", datetime('now'), " + str(altitude) + ", " + "%.6f" % decoded_lat + ", " + "%.6f" % decoded_lon + ")"
+        return "INSERT INTO positions (icao, seen, alt, lat, lon) VALUES (%i, datetime('now'), %i, %.6f, %.6f)" % (icao24, int(altitude), decoded_lat, decoded_lon)
     elif bdsreg == 0x09:
       subtype = data["bds09"].get_type()
       if subtype == 0:
-        [velocity, heading, vert_spd, turnrate] = air_modes.parseBDS09_0(data)
-        return "INSERT INTO vectors (icao, seen, speed, heading, vertical) VALUES (" + "%i" % icao24 + ", datetime('now'), " + "%.0f" % velocity + ", " + "%.0f" % heading + ", " + "%.0f" % vert_spd + ")"
+        [velocity, heading, vert_spd, turnrate] = self.parseBDS09_0(data)
+        return "INSERT INTO vectors (icao, seen, speed, heading, vertical) VALUES (%i, datetime('now'), %.0f, %.0f, %.0f)" % (icao24, velocity, heading, vert_spd)
       elif subtype == 1:
-        [velocity, heading, vert_spd] = air_modes.parseBDS09_1(data)
-        return "INSERT INTO vectors (icao, seen, speed, heading, vertical) VALUES (" + "%i" % icao24 + ", datetime('now'), " + "%.0f" % velocity + ", " + "%.0f" % heading + ", " + "%.0f" % vert_spd + ")"
+        [velocity, heading, vert_spd] = self.parseBDS09_1(data)
+        return "INSERT INTO vectors (icao, seen, speed, heading, vertical) VALUES (%i, datetime('now'), %.0f, %.0f, %.0f)" % (icao24, velocity, heading, vert_spd)
       else:
         raise NoHandlerError