Mercurial Hosting > traffic-intelligence
changeset 1015:cf9d29de3dbf
merge With Pr Saunier's code
author | Wendlasida |
---|---|
date | Mon, 04 Jun 2018 11:25:49 -0400 |
parents | 026f6b3b122c (current diff) 0d29b75f74ea (diff) |
children | 9b53be469a36 |
files | |
diffstat | 4 files changed, 91 insertions(+), 39 deletions(-) [+] |
line wrap: on
line diff
--- a/python/cvutils.py Fri Jun 01 17:32:52 2018 -0400 +++ b/python/cvutils.py Mon Jun 04 11:25:49 2018 -0400 @@ -18,7 +18,7 @@ from sys import stdout from os import listdir -from subprocess import check_call +from subprocess import run from math import floor, log10, ceil from numpy import dot, array, append, float32, loadtxt, savetxt, append, zeros, ones, identity, abs as npabs, logical_and, unravel_index, sum as npsum, isnan, mgrid, median, floor as npfloor, ceil as npceil @@ -282,7 +282,7 @@ else: return None - def tracking(configFilename, grouping, videoFilename = None, dbFilename = None, homographyFilename = None, maskFilename = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None): + def tracking(configFilename, grouping, videoFilename = None, dbFilename = None, homographyFilename = None, maskFilename = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, dryRun = False): '''Runs the tracker in a subprocess if grouping is True, it is feature grouping otherwise it is feature tracking''' @@ -303,14 +303,16 @@ if undistort: cmd += ['--undistort', 'true'] if intrinsicCameraMatrix is not None: # we currently have to save a file - pass#from time import time - #savetxt - #cmd += [] + from time import time + intrinsicCameraFilename = '/tmp/intrinsic-{}.txt'.format(time()) + savetxt(intrinsicCameraFilename, intrinsicCameraMatrix) + cmd += ['--intrinsic-camera-filename', intrinsicCameraFilename] if distortionCoefficients is not None: - cmd += ['--distortion-coefficients', ' '.join([str(x) for x in distortionCoefficients])] - - #check_call([trackerExe, configFilename, trackingMode]) # , stderr = out, shell = True - print(cmd) # , stderr = out, shell = True + cmd += ['--distortion-coefficients '+' '.join([str(x) for x in distortionCoefficients])] + if dryRun: + print(cmd) + else: + run(cmd) def displayTrajectories(videoFilename, objects, boundingBoxes = {}, homography = None, firstFrameNum = 0, lastFrameNumArg = None, printFrames = True, rescale = 1., nFramesStep = 1, saveAllImages = False, nZerosFilenameArg = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., annotations = [], gtMatches = {}, toMatches = {}, colorBlind = False): '''Displays the objects overlaid frame by frame over the video '''
--- a/python/metadata.py Fri Jun 01 17:32:52 2018 -0400 +++ b/python/metadata.py Mon Jun 04 11:25:49 2018 -0400 @@ -350,12 +350,17 @@ Session = sessionmaker(bind=engine) return Session() -def getSite(session, siteId): +def getSite(session, siteId = None, name = None, description = None): 'Returns the site(s) matching the index or the name' - if str.isdigit(siteId): + if siteId is not None: return session.query(Site).filter(Site.idx == int(siteId)).all() + elif name is not None: + return session.query(Site).filter(Site.description.like('%'+name+'%')).all() + elif description is not None: + return session.query(Site).filter(Site.description.like('%'+description+'%')).all() else: - return session.query(Site).filter(Site.description.like('%'+siteId+'%')).all() + print('No siteId, name or description have been provided to the function') + return [] def getCameraView(session, viewId): 'Returns the site(s) matching the index' @@ -405,3 +410,6 @@ startTime += duration session.add_all(videoSequences) session.commit() + +# management +# TODO need to be able to copy everything from a site from one sqlite to another, and delete everything attached to a site
--- a/python/storage.py Fri Jun 01 17:32:52 2018 -0400 +++ b/python/storage.py Mon Jun 04 11:25:49 2018 -0400 @@ -966,9 +966,9 @@ "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") out.close() # system call - from subprocess import check_call + from subprocess import run out = openCheck("err.log", "w") - check_call("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out, shell = True) + run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) out.close() shutil.os.remove(sqlScriptFilename)
--- a/scripts/process.py Fri Jun 01 17:32:52 2018 -0400 +++ b/scripts/process.py Mon Jun 04 11:25:49 2018 -0400 @@ -2,18 +2,20 @@ import sys, argparse from pathlib import Path +from multiprocessing.pool import Pool import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt from numpy import percentile -import storage, events, prediction +import storage, events, prediction, cvutils from metadata import * parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int) +parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int) parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int) parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp']) @@ -23,46 +25,85 @@ parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction']) parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction']) +parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true') +parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) # need way of selecting sites as similar as possible to sql alchemy syntax # override tracking.cfg from db # manage cfg files, overwrite them (or a subset of parameters) # delete sqlite files - # info of metadata -parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) - args = parser.parse_args() -# files are relative to metadata location - -session = connectDatabase(args.metadataFilename) -parentDir = Path(args.metadataFilename).parent +################################# +# Data preparation +################################# +session = connectDatabase(args.metadataFilename) +parentDir = Path(args.metadataFilename).parent # files are relative to metadata location +videoSequences = [] +if args.videoIds is not None: + videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds] +elif args.siteIds is not None: + for siteId in args.siteIds: + for site in getSite(session, siteId): + for cv in site.cameraViews: + videoSequences += cv.videoSequences +else: + print('No video/site to process') + +################################# +# Delete +################################# if args.delete is not None: - if args.delete in ['object', 'interaction']: + if args.delete == 'feature': + pass + elif args.delete in ['object', 'interaction']: #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype']) - for videoId in args.videoIds: - vs = session.query(VideoSequence).get(videoId) + for vs in videoSequences: storage.deleteFromSqlite(str(parentDir/vs.getDatabaseFilename()), args.delete) +################################# +# Process +################################# if args.process in ['feature', 'object']: # tracking - for videoId in args.videoIds: - vs = session.query(VideoSequence).get(videoId) - if args.configFilename is None: - configFilename = vs.cameraView.getTrackingConfigurationFilename() - else: - configFilename = args.configFilename - #todo cvutils.tracking(configFilename, args.process == 'object', str(parentDir/vs.getVideoSequenceFilename(), str(parentDir/vs.getDatabaseFilename(), configFilename = vs.cameraView.getHomographyFilename()) - + if args.nProcesses == 1: + for vs in videoSequences: + if not (parentDir/vs.getDatabaseFilename()).exists() or args.process == 'object': + if args.configFilename is None: + configFilename = str(parentDir/vs.cameraView.getTrackingConfigurationFilename()) + else: + configFilename = args.configFilename + if vs.cameraView.cameraType is None: + cvutils.tracking(configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun) + else: + cvutils.tracking(configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun) + else: + print('SQLite already exists: {}'.format(parentDir/vs.getDatabaseFilename())) + else: + pool = Pool(args.nProcesses) + for vs in videoSequences: + if not (parentDir/vs.getDatabaseFilename()).exists() or args.process == 'object': + if args.configFilename is None: + configFilename = str(parentDir/vs.cameraView.getTrackingConfigurationFilename()) + else: + configFilename = args.configFilename + if vs.cameraView.cameraType is None: + pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)) + else: + pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)) + else: + print('SQLite already exists: {}'.format(parentDir/vs.getDatabaseFilename())) + pool.close() + pool.join() + elif args.process == 'interaction': # safety analysis TODO make function in safety analysis script if args.predictionMethod == 'cvd': predictionParameters = prediction.CVDirectPredictionParameters() if args.predictionMethod == 'cve': predictionParameters = prediction.CVExactPredictionParameters() - for videoId in args.videoIds: - vs = session.query(VideoSequence).get(videoId) + for vs in videoSequences: print('Processing '+vs.getDatabaseFilename()) objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp')) interactions = events.createInteractions(objects) @@ -81,12 +122,14 @@ # processed += job.get() # pool.close() +################################# +# Analyze +################################# if args.analyze == 'object': # user speed for now medianSpeeds = {} speeds85 = {} minLength = 2*30 - for videoId in args.videoIds: - vs = session.query(VideoSequence).get(videoId) + for vs in videoSequences: if not vs.cameraView.siteIdx in medianSpeeds: medianSpeeds[vs.cameraView.siteIdx] = [] speeds85[vs.cameraView.siteIdx] = [] @@ -111,8 +154,7 @@ maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} indicators = {} interactions = {} - for videoId in args.videoIds: - vs = session.query(VideoSequence).get(videoId) + for vs in videoSequences: if not vs.cameraView.siteIdx in interactions: interactions[vs.cameraView.siteIdx] = [] indicators[vs.cameraView.siteIdx] = {}