Mercurial Hosting > traffic-intelligence
annotate scripts/process.py @ 986:3be8aaa47651
work in progress
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Tue, 06 Mar 2018 23:54:10 -0500 |
parents | 668a85c963c3 |
children | f026ce2af637 |
rev | line source |
---|---|
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
1 #! /usr/bin/env python |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
2 |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
3 import sys, argparse |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
4 from pathlib2 import Path |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
5 |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
6 import matplotlib |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
7 matplotlib.use('Agg') |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
8 import matplotlib.pyplot as plt |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
9 |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
10 import storage, events, prediction |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
11 from metadata import * |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
12 |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
13 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
14 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
15 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int) |
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
16 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true') |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
17 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
18 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction']) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
19 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction']) |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
20 |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
21 # need way of selecting sites as similar as possible to sql alchemy syntax |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
22 # override tracking.cfg from db |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
23 # manage cfg files, overwrite them (or a subset of parameters) |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
24 # delete sqlite files |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
25 |
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
26 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
27 |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
28 args = parser.parse_args() |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
29 # files are relative to metadata location |
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
30 |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
31 session = connectDatabase(args.metadataFilename) |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
32 parentDir = Path(args.metadataFilename).parent |
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
33 |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
34 if args.delete is not None: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
35 if args.delete in ['object', 'interaction']: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
36 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype']) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
37 for videoId in args.videoIds: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
38 vs = session.query(VideoSequence).get(videoId) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
39 storage.deleteFromSqlite(str(parentDir/vs.getDatabaseFilename()), args.delete) |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
40 |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
41 if args.process == 'interaction': |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
42 # safety analysis TODO make function in safety analysis script |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
43 predictionParameters = prediction.CVDirectPredictionParameters()#prediction.CVExactPredictionParameters() |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
44 for videoId in args.videoIds: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
45 vs = session.query(VideoSequence).get(videoId) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
46 print('Processing '+vs.getDatabaseFilename()) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
47 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp')) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
48 interactions = events.createInteractions(objects) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
49 #if args.nProcesses == 1: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
50 #print(str(parentDir/vs.cameraView.getTrackingConfigurationFilename())) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
51 params = storage.ProcessParameters(str(parentDir/vs.cameraView.getTrackingConfigurationFilename())) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
52 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
53 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
54 storage.saveIndicatorsToSqlite(str(parentDir/vs.getDatabaseFilename()), processed) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
55 # else: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
56 # pool = Pool(processes = args.nProcesses) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
57 # nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses))) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
58 # jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], not args.noMotionPrediction, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)) for i in range(args.nProcesses)] |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
59 # processed = [] |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
60 # for job in jobs: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
61 # processed += job.get() |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
62 # pool.close() |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
63 |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
64 if args.analyze == 'interaction': |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
65 indicatorIds = [2,5,7,10] |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
66 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
67 indicators = {} |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
68 interactions = {} |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
69 for videoId in args.videoIds: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
70 vs = session.query(VideoSequence).get(videoId) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
71 if not vs.cameraView.siteIdx in interactions: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
72 interactions[vs.cameraView.siteIdx] = [] |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
73 indicators[vs.cameraView.siteIdx] = {} |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
74 for i in indicatorIds: |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
75 indicators[vs.cameraView.siteIdx][i] = [] |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
76 interactions[vs.cameraView.siteIdx] += storage.loadInteractionsFromSqlite(str(parentDir/vs.getDatabaseFilename())) |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
77 print(vs.getDatabaseFilename(), len(interactions[vs.cameraView.siteIdx])) |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
78 for inter in interactions[vs.cameraView.siteIdx]: |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
79 for i in indicatorIds: |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
80 indic = inter.getIndicator(events.Interaction.indicatorNames[i]) |
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
81 if indic is not None: |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
82 indicators[vs.cameraView.siteIdx][i].append(indic.getMostSevereValue()*conversionFactors[i]) |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
83 |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
84 for i in indicatorIds: |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
85 tmp = [indicators[siteId][i] for siteId in indicators] |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
86 plt.ioff() |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
87 plt.figure() |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
88 plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators]) |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
89 plt.title(events.Interaction.indicatorNames[i]) |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
90 plt.savefig(events.Interaction.indicatorNames[i]+'.png') |
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
91 plt.close() |