Mercurial Hosting > traffic-intelligence
changeset 1065:d4d052a05337
added progress report functionality
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Mon, 16 Jul 2018 00:05:17 -0400 |
parents | cbc026dacf0b |
children | 862b55a87e63 |
files | scripts/process.py trafficintelligence/storage.py |
diffstat | 2 files changed, 52 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/scripts/process.py Sun Jul 15 22:52:26 2018 -0400 +++ b/scripts/process.py Mon Jul 16 00:05:17 2018 -0400 @@ -89,6 +89,7 @@ videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)]) else: videoSequences.append(session.query(VideoSequence).get(int(videoId))) + videoSequences = [vs for vs in videoSequences if vs is not None] sites = set([vs.cameraView.site for vs in videoSequences]) elif args.siteIds is not None: for siteId in args.siteIds: @@ -96,11 +97,12 @@ sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)]) else: sites.append(session.query(Site).get(int(siteId))) + sites = [s for s in sites if s is not None] for site in sites: videoSequences.extend(getSiteVideoSequences(site)) else: print('No video/site to process') -sys.exit() + if args.nProcesses > 1: pool = Pool(args.nProcesses) @@ -109,6 +111,28 @@ ################################# if args.progress: print('Providing information on data progress') + headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification + data = [] + for site in sites: + unprocessedVideoSequences = [] + for vs in getSiteVideoSequences(site): + if (parentPath/vs.getDatabaseFilename()).is_file(): + tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename())) + data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames]) + else: + unprocessedVideoSequences.append(vs) + data.append([site.name, vs.idx, False, False, False]) + #if len(unprocessedVideoSequences): + # print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences))) + data = pd.DataFrame(data, columns = headers) + print('-'*80) + print('\t'+' '.join(headers[2:])) + print('-'*80) + for name, group in data.groupby(['site']): #.agg({'vs': 'count'})) + n = group.vs.count() + print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n)) + print('-'*80) + print(data) ################################# # Delete @@ -131,7 +155,7 @@ if args.process in ['feature', 'object']: # tracking if args.nProcesses == 1: for vs in videoSequences: - if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object': + if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object': if args.configFilename is None: configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) else: @@ -144,7 +168,7 @@ print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename())) else: for vs in videoSequences: - if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object': + if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object': if args.configFilename is None: configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) else:
--- a/trafficintelligence/storage.py Sun Jul 15 22:52:26 2018 -0400 +++ b/trafficintelligence/storage.py Mon Jul 16 00:05:17 2018 -0400 @@ -70,6 +70,18 @@ except sqlite3.OperationalError as error: printDBError(error) +def tableNames(filename): + 'Lists the names of the tables in the SQLite file' + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + try: + cursor = connection.cursor() + cursor.execute('SELECT name FROM sqlite_master WHERE type = \'table\'') + return [row[0] for row in cursor] + except sqlite3.OperationalError as error: + printDBError(error) + return [] + def createTrajectoryTable(cursor, tableName): if tableName.endswith('positions') or tableName.endswith('velocities'): cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (trajectory_id INTEGER, frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") @@ -259,7 +271,7 @@ The number loaded is either the first objectNumbers objects, or the indices in objectNumbers from the database''' objects = [] - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: objects = loadTrajectoriesFromTable(connection, 'positions', trajectoryType, objectNumbers, timeStep) objectVelocities = loadTrajectoriesFromTable(connection, 'velocities', trajectoryType, objectNumbers, timeStep) @@ -452,7 +464,7 @@ Load descriptions?''' objects = [] - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: if objectType == 'bb': topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep) @@ -511,7 +523,7 @@ TODO choose the interactions to load''' interactions = [] - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: cursor = connection.cursor() try: @@ -557,7 +569,7 @@ def loadBoundingBoxTableForDisplay(filename): '''Loads bounding boxes from bounding_boxes table for display over trajectories''' boundingBoxes = {} # list of bounding boxes for each instant - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: cursor = connection.cursor() try: @@ -628,7 +640,7 @@ def loadPrototypeAssignmentsFromSqlite(filename, objectType): prototypeAssignments = {} - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: cursor = connection.cursor() try: @@ -648,7 +660,7 @@ def loadPrototypesFromSqlite(filename, withTrajectories = True): 'Loads prototype ids and matchings (if stored)' prototypes = [] - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: cursor = connection.cursor() objects = [] @@ -709,7 +721,7 @@ from sklearn import mixture # todo if not avalaible, load data in duck-typed class with same fields from ast import literal_eval pois = [] - if Path(filename).exists(): + if Path(filename).is_file(): with sqlite3.connect(filename) as connection: cursor = connection.cursor() try: @@ -1334,7 +1346,7 @@ def __init__(self, filename = None): self.configFilename = filename - if filename is not None and Path(filename).exists(): + if filename is not None and Path(filename).is_file(): self.loadConfigFile(filename) else: print('Configuration filename {} could not be loaded.'.format(filename)) @@ -1424,7 +1436,7 @@ parentPath = Path(filename).parent self.sectionHeader = config.sections()[0] self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename')) - if Path(self.homographyFilename).exists(): + if Path(self.homographyFilename).is_file(): self.homography = loadtxt(self.homographyFilename) else: self.homography = None @@ -1443,7 +1455,7 @@ parentPath = Path(filename).parent self.sectionHeader = config.sections()[0] self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename')) - if Path(self.intrinsicCameraFilename).exists(): + if Path(self.intrinsicCameraFilename).is_file(): self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename) else: self.intrinsicCameraMatrix = None @@ -1465,12 +1477,12 @@ self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) self.databaseFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'database-filename')) self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename')) - if Path(self.homographyFilename).exists(): + if Path(self.homographyFilename).is_file(): self.homography = loadtxt(self.homographyFilename) else: self.homography = None self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename')) - if Path(self.intrinsicCameraFilename).exists(): + if Path(self.intrinsicCameraFilename).is_file(): self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename) else: self.intrinsicCameraMatrix = None @@ -1505,7 +1517,7 @@ def __init__(self, filename = None): self.configFilename = filename - if filename is not None and Path(filename).exists(): + if filename is not None and Path(filename).is_file(): self.loadConfigFile(filename) else: print('Configuration filename {} could not be loaded.'.format(filename))