Mercurial Hosting > traffic-intelligence

annotate scripts/process.py @ 984:a69695d14e59

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work on script for large datasets
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Tue, 06 Mar 2018 08:26:13 -0500
parents 7463c9bc846b
children 668a85c963c3
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
1 #! /usr/bin/env python
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
2
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
3 import sys, argparse
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
4 from pathlib2 import Path
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
5
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
6 import storage, events, prediction
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
7 from metadata import *
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
8
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
9 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
10 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
11 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
12 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
13
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
14 # need way of selecting sites as similar as possible to sql alchemy syntax
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
15 # override tracking.cfg from db
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
16 # manage cfg files, overwrite them (or a subset of parameters)
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
17 # delete sqlite files
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
18
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
19 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
20
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
21 args = parser.parse_args()
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
22 # files are relative to metadata location
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
23
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
24 session = createDatabase(args.metadataFilename)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
25 parentDir = Path(args.metadataFilename).parent
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
26
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
27 # todo change prediction parameters
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
28 predictionParameters = prediction.CVExactPredictionParameters()
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
29
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
30 for videoId in args.videoIds:
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
31 vs = session.query(VideoSequence).get(videoId)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
32 print(vs.getDatabaseFilename())
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
33 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
34 interactions = events.createInteractions(objects)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
35 #if args.nProcesses == 1:
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
36 params = storage.ProcessParameters(str(parentDir/vs.cameraView.getTrackingConfigurationFilename()))
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
37 #print(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
38 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
39 storage.saveIndicatorsToSqlite(str(parentDir/vs.getDatabaseFilename()), processed)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
40
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
41 # else:
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
42 # pool = Pool(processes = args.nProcesses)
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
43 # nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses)))
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
44 # jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], not args.noMotionPrediction, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)) for i in range(args.nProcesses)]
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
45 # processed = []
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
46 # for job in jobs:
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
47 # processed += job.get()
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
48 # pool.close()