Mercurial Hosting > traffic-intelligence

annotate scripts/process.py @ 986:3be8aaa47651

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work in progress
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Tue, 06 Mar 2018 23:54:10 -0500
parents 668a85c963c3
children f026ce2af637
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
1 #! /usr/bin/env python
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
2
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
3 import sys, argparse
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
4 from pathlib2 import Path
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
5
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
6 import matplotlib
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
7 matplotlib.use('Agg')
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
8 import matplotlib.pyplot as plt
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
9
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
10 import storage, events, prediction
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
11 from metadata import *
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
12
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
13 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
14 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
15 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
16 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
17 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
18 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction'])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
19 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction'])
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
20
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
21 # need way of selecting sites as similar as possible to sql alchemy syntax
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
22 # override tracking.cfg from db
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
23 # manage cfg files, overwrite them (or a subset of parameters)
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
24 # delete sqlite files
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
25
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
26 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
27
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
28 args = parser.parse_args()
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
29 # files are relative to metadata location
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
30
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
31 session = connectDatabase(args.metadataFilename)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
32 parentDir = Path(args.metadataFilename).parent
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
33
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
34 if args.delete is not None:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
35 if args.delete in ['object', 'interaction']:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
36 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype'])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
37 for videoId in args.videoIds:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
38 vs = session.query(VideoSequence).get(videoId)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
39 storage.deleteFromSqlite(str(parentDir/vs.getDatabaseFilename()), args.delete)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
40
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
41 if args.process == 'interaction':
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
42 # safety analysis TODO make function in safety analysis script
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
43 predictionParameters = prediction.CVDirectPredictionParameters()#prediction.CVExactPredictionParameters()
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
44 for videoId in args.videoIds:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
45 vs = session.query(VideoSequence).get(videoId)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
46 print('Processing '+vs.getDatabaseFilename())
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
47 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
48 interactions = events.createInteractions(objects)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
49 #if args.nProcesses == 1:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
50 #print(str(parentDir/vs.cameraView.getTrackingConfigurationFilename()))
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
51 params = storage.ProcessParameters(str(parentDir/vs.cameraView.getTrackingConfigurationFilename()))
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
52 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
53 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
54 storage.saveIndicatorsToSqlite(str(parentDir/vs.getDatabaseFilename()), processed)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
55 # else:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
56 # pool = Pool(processes = args.nProcesses)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
57 # nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses)))
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
58 # jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], not args.noMotionPrediction, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)) for i in range(args.nProcesses)]
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
59 # processed = []
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
60 # for job in jobs:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
61 # processed += job.get()
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
62 # pool.close()
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
63
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
64 if args.analyze == 'interaction':
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
65 indicatorIds = [2,5,7,10]
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
66 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
67 indicators = {}
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
68 interactions = {}
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
69 for videoId in args.videoIds:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
70 vs = session.query(VideoSequence).get(videoId)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
71 if not vs.cameraView.siteIdx in interactions:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
72 interactions[vs.cameraView.siteIdx] = []
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
73 indicators[vs.cameraView.siteIdx] = {}
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
74 for i in indicatorIds:
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
75 indicators[vs.cameraView.siteIdx][i] = []
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
76 interactions[vs.cameraView.siteIdx] += storage.loadInteractionsFromSqlite(str(parentDir/vs.getDatabaseFilename()))
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
77 print(vs.getDatabaseFilename(), len(interactions[vs.cameraView.siteIdx]))
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
78 for inter in interactions[vs.cameraView.siteIdx]:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
79 for i in indicatorIds:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
80 indic = inter.getIndicator(events.Interaction.indicatorNames[i])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
81 if indic is not None:
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
82 indicators[vs.cameraView.siteIdx][i].append(indic.getMostSevereValue()*conversionFactors[i])
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
83
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
84 for i in indicatorIds:
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
85 tmp = [indicators[siteId][i] for siteId in indicators]
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
86 plt.ioff()
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
87 plt.figure()
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
88 plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
89 plt.title(events.Interaction.indicatorNames[i])
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
90 plt.savefig(events.Interaction.indicatorNames[i]+'.png')
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
91 plt.close()