--- a/scripts/process.py	Sun Jul 15 22:52:26 2018 -0400
+++ b/scripts/process.py	Mon Jul 16 00:05:17 2018 -0400
@@ -89,6 +89,7 @@
             videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)])
         else:
             videoSequences.append(session.query(VideoSequence).get(int(videoId)))
+    videoSequences = [vs for vs in videoSequences if vs is not None]
     sites = set([vs.cameraView.site for vs in videoSequences])
 elif args.siteIds is not None:
     for siteId in args.siteIds:
@@ -96,11 +97,12 @@
             sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)])
         else:
             sites.append(session.query(Site).get(int(siteId)))
+    sites = [s for s in sites if s is not None]
     for site in sites:
         videoSequences.extend(getSiteVideoSequences(site))
 else:
     print('No video/site to process')
-sys.exit()
+
 if args.nProcesses > 1:
     pool = Pool(args.nProcesses)
 
@@ -109,6 +111,28 @@
 #################################
 if args.progress:
     print('Providing information on data progress')
+    headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification
+    data = []
+    for site in sites:
+        unprocessedVideoSequences = []
+        for vs in getSiteVideoSequences(site):
+            if (parentPath/vs.getDatabaseFilename()).is_file():
+                tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename()))
+                data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames])
+            else:
+                unprocessedVideoSequences.append(vs)
+                data.append([site.name, vs.idx, False, False, False])
+        #if len(unprocessedVideoSequences):
+        #    print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences)))
+    data = pd.DataFrame(data, columns = headers)
+    print('-'*80)
+    print('\t'+' '.join(headers[2:]))
+    print('-'*80)
+    for name, group in data.groupby(['site']): #.agg({'vs': 'count'}))
+        n = group.vs.count()
+        print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n))
+    print('-'*80)
+    print(data)
 
 #################################
 # Delete
@@ -131,7 +155,7 @@
 if args.process in ['feature', 'object']: # tracking
     if args.nProcesses == 1:
         for vs in videoSequences:
-            if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object':
+            if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
                 if args.configFilename is None:
                     configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
                 else:
@@ -144,7 +168,7 @@
                 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
     else:
         for vs in videoSequences:
-            if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object':
+            if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
                 if args.configFilename is None:
                     configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
                 else:

--- a/trafficintelligence/storage.py	Sun Jul 15 22:52:26 2018 -0400
+++ b/trafficintelligence/storage.py	Mon Jul 16 00:05:17 2018 -0400
@@ -70,6 +70,18 @@
     except sqlite3.OperationalError as error:
         printDBError(error)        
 
+def tableNames(filename):
+    'Lists the names of the tables in the SQLite file'
+    if Path(filename).is_file():
+        with sqlite3.connect(filename) as connection:
+            try:
+                cursor = connection.cursor()
+                cursor.execute('SELECT name FROM sqlite_master WHERE type = \'table\'')
+                return [row[0] for row in cursor]
+            except sqlite3.OperationalError as error:
+                printDBError(error)
+    return []
+        
 def createTrajectoryTable(cursor, tableName):
     if tableName.endswith('positions') or tableName.endswith('velocities'):
         cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (trajectory_id INTEGER, frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))")
@@ -259,7 +271,7 @@
     The number loaded is either the first objectNumbers objects,
     or the indices in objectNumbers from the database'''
     objects = []
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             objects = loadTrajectoriesFromTable(connection, 'positions', trajectoryType, objectNumbers, timeStep)
             objectVelocities = loadTrajectoriesFromTable(connection, 'velocities', trajectoryType, objectNumbers, timeStep)
@@ -452,7 +464,7 @@
 
     Load descriptions?'''
     objects = []
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             if objectType == 'bb':
                 topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep)
@@ -511,7 +523,7 @@
     
     TODO choose the interactions to load'''
     interactions = []
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             cursor = connection.cursor()
             try:
@@ -557,7 +569,7 @@
 def loadBoundingBoxTableForDisplay(filename):
     '''Loads bounding boxes from bounding_boxes table for display over trajectories'''
     boundingBoxes = {} # list of bounding boxes for each instant
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             cursor = connection.cursor()
             try:
@@ -628,7 +640,7 @@
 
 def loadPrototypeAssignmentsFromSqlite(filename, objectType):
     prototypeAssignments = {}
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             cursor = connection.cursor()
             try:
@@ -648,7 +660,7 @@
 def loadPrototypesFromSqlite(filename, withTrajectories = True):
     'Loads prototype ids and matchings (if stored)'
     prototypes = []
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             cursor = connection.cursor()
             objects = []
@@ -709,7 +721,7 @@
     from sklearn import mixture # todo if not avalaible, load data in duck-typed class with same fields
     from ast import literal_eval
     pois = []
-    if Path(filename).exists():
+    if Path(filename).is_file():
         with sqlite3.connect(filename) as connection:
             cursor = connection.cursor()
             try:
@@ -1334,7 +1346,7 @@
 
     def __init__(self, filename = None):
         self.configFilename = filename
-        if filename is not None and Path(filename).exists():
+        if filename is not None and Path(filename).is_file():
             self.loadConfigFile(filename)
         else:
             print('Configuration filename {} could not be loaded.'.format(filename))
@@ -1424,7 +1436,7 @@
             parentPath = Path(filename).parent
             self.sectionHeader = config.sections()[0]
             self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename'))
-            if Path(self.homographyFilename).exists():
+            if Path(self.homographyFilename).is_file():
                 self.homography = loadtxt(self.homographyFilename)
             else:
                 self.homography = None
@@ -1443,7 +1455,7 @@
             parentPath = Path(filename).parent
             self.sectionHeader = config.sections()[0]
             self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename'))
-            if Path(self.intrinsicCameraFilename).exists():
+            if Path(self.intrinsicCameraFilename).is_file():
                 self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename)
             else:
                 self.intrinsicCameraMatrix = None
@@ -1465,12 +1477,12 @@
         self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename'))
         self.databaseFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'database-filename'))
         self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename'))
-        if Path(self.homographyFilename).exists():
+        if Path(self.homographyFilename).is_file():
             self.homography = loadtxt(self.homographyFilename)
         else:
             self.homography = None
         self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename'))
-        if Path(self.intrinsicCameraFilename).exists():
+        if Path(self.intrinsicCameraFilename).is_file():
             self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename)
         else:
             self.intrinsicCameraMatrix = None
@@ -1505,7 +1517,7 @@
 
     def __init__(self, filename = None):
         self.configFilename = filename
-        if filename is not None and Path(filename).exists():
+        if filename is not None and Path(filename).is_file():
             self.loadConfigFile(filename)
         else:
             print('Configuration filename {} could not be loaded.'.format(filename))