Mercurial Hosting > traffic-intelligence

comparison scripts/process.py @ 1054:d13f9bfbf3ff

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Retry
author Wendlasida
date Fri, 06 Jul 2018 18:42:58 -0400
parents c9c03c97ed9f
children 9d4a06f49cb8
comparison
equal deleted inserted replaced
1053:60cc87e824c4 1054:d13f9bfbf3ff
5 from multiprocessing.pool import Pool 5 from multiprocessing.pool import Pool
6 6
7 #import matplotlib 7 #import matplotlib
8 #atplotlib.use('Agg') 8 #atplotlib.use('Agg')
9 import matplotlib.pyplot as plt 9 import matplotlib.pyplot as plt
10 from numpy import percentile 10 import numpy as np
11 from pandas import DataFrame 11 from pandas import DataFrame
12 12
13 from trafficintelligence import storage, events, prediction, cvutils, utils 13 from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml
14 from trafficintelligence.metadata import * 14 from trafficintelligence.metadata import *
15 15
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') 16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
17 # input 17 # input
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) 18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int) 19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int) 20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*')
21 21
22 # main function 22 # main function
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) 23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction']) 24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction'])
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) 25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
26 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction']) 26 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction'])
27 27
28 # common options 28 # common options
29 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') 29 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
30 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int) 30 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
31 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature')
31 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true') 32 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
32 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) 33 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
34 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int)
35
36 ### process options
37 # motion pattern learning and assignment
38 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes', default = 'prototypes.sqlite')
39 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
40 parser.add_argument('--nobjects-mp', dest = 'nMPObjects', help = 'number of objects/interactions to process', type = int)
41 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
42 parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float)
43 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
44 parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float)
45 parser.add_argument('--min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
46 #parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
47 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
48 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
49 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
50 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
51 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
52 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true')
53
54 # safety analysis
55 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
56 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
57 # override other tracking config, erase sqlite?
58
33 59
34 # analysis options 60 # analysis options
35 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) 61 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
36 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1) 62 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1)
37 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = float, default = 15.) 63 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = float, default = 15.)
38 parser.add_argument('--aggregation', dest = 'aggMethod', help = 'aggregation method per user/event and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) 64 parser.add_argument('--aggregation', dest = 'aggMethod', help = 'aggregation method per user/event and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
39 parser.add_argument('--aggregation-centile', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) 65 parser.add_argument('--aggregation-centile', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
40 dpi = 150 66 dpi = 150
41 # unit of analysis: site or video sequence? 67 # unit of analysis: site or video sequence?
42 68
43 # safety analysis
44 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
45 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
46 # override other tracking config, erase sqlite?
47
48 # need way of selecting sites as similar as possible to sql alchemy syntax 69 # need way of selecting sites as similar as possible to sql alchemy syntax
49 # override tracking.cfg from db 70 # override tracking.cfg from db
50 # manage cfg files, overwrite them (or a subset of parameters) 71 # manage cfg files, overwrite them (or a subset of parameters)
51 # delete sqlite files 72 # delete sqlite files
52 # info of metadata 73 # info of metadata
57 # Data preparation 78 # Data preparation
58 ################################# 79 #################################
59 session = connectDatabase(args.metadataFilename) 80 session = connectDatabase(args.metadataFilename)
60 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location 81 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
61 videoSequences = [] 82 videoSequences = []
83 sites = []
62 if args.videoIds is not None: 84 if args.videoIds is not None:
63 videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds] 85 videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds]
86 siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
64 elif args.siteIds is not None: 87 elif args.siteIds is not None:
65 for siteId in args.siteIds: 88 siteIds = set(args.siteIds)
66 for site in getSite(session, siteId): 89 for siteId in siteIds:
90 tmpsites = getSite(session, siteId)
91 sites.extend(tmpsites)
92 for site in tmpsites:
67 for cv in site.cameraViews: 93 for cv in site.cameraViews:
68 videoSequences += cv.videoSequences 94 videoSequences.extend(cv.videoSequences)
69 else: 95 else:
70 print('No video/site to process') 96 print('No video/site to process')
71 97
72 if args.nProcesses > 1: 98 if args.nProcesses > 1:
73 pool = Pool(args.nProcesses) 99 pool = Pool(args.nProcesses)
119 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename())) 145 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
120 pool.close() 146 pool.close()
121 pool.join() 147 pool.join()
122 148
123 elif args.process == 'prototype': # motion pattern learning 149 elif args.process == 'prototype': # motion pattern learning
124 pass 150 # learn by site by default -> group videos by site (or by camera view? TODO add cameraviews)
151 # by default, load all objects, learn and then assign (BUT not save the assignments)
152 for site in sites:
153 print('Learning motion patterns for site {} ({})'.format(site.idx, site.name))
154 objects = {}
155 object2VideoSequences = {}
156 for cv in site.cameraViews:
157 for vs in cv.videoSequences:
158 print('Loading '+vs.getDatabaseFilename())
159 objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
160 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
161 objectsWithFeatures = objects[vs.idx]
162 objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()]
163 prototypeType = 'feature'
164 else:
165 prototypeType = args.trajectoryType
166 for obj in objects[vs.idx]:
167 object2VideoSequences[obj] = vs
168 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
169 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
170 trainingObjects = [o for tmpobjects in objects.values() for o in tmpobjects]
171 if args.nMPObjects is not None and args.nMPObjects < len(trainingObjects):
172 m = int(np.floor(float(len(trainingObjects))/args.nMPObjects))
173 trainingObjects = trainingObjects[::m]
174 similarities = -np.ones((len(trainingObjects), len(trainingObjects)))
175 prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, trainingObjects, similarities, args.minSimilarity, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, [])
176 if args.outputPrototypeDatabaseFilename is None:
177 outputPrototypeDatabaseFilename = args.databaseFilename
178 else:
179 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
180 # TODO maintain mapping from object prototype to db filename + compute nmatchings before
181 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
182 storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices])
183
125 184
126 elif args.process == 'interaction': 185 elif args.process == 'interaction':
127 # safety analysis TODO make function in safety analysis script 186 # safety analysis TODO make function in safety analysis script
128 if args.predictionMethod == 'cvd': 187 if args.predictionMethod == 'cvd':
129 predictionParameters = prediction.CVDirectPredictionParameters() 188 predictionParameters = prediction.CVDirectPredictionParameters()
181 row += aggSpeeds.tolist() 240 row += aggSpeeds.tolist()
182 else: 241 else:
183 row.append(aggSpeeds) 242 row.append(aggSpeeds)
184 data.append(row) 243 data.append(row)
185 data = DataFrame(data, columns = headers) 244 data = DataFrame(data, columns = headers)
186 if args.siteIds is None:
187 siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
188 else:
189 siteIds = set(args.siteIds)
190 if args.output == 'figure': 245 if args.output == 'figure':
191 for name in headers[4:]: 246 for name in headers[4:]:
192 plt.ioff() 247 plt.ioff()
193 plt.figure() 248 plt.figure()
194 plt.boxplot([data.loc[data['sites']==siteId, name] for siteId in siteIds], labels = [session.query(Site).get(siteId).name for siteId in siteIds]) 249 plt.boxplot([data.loc[data['sites']==siteId, name] for siteId in siteIds], labels = [session.query(Site).get(siteId).name for siteId in siteIds])