Mercurial Hosting > traffic-intelligence

annotate scripts/process.py @ 1023:a13f47c8931d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work on processing large datasets (generate speed data)
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Wed, 06 Jun 2018 16:51:15 -0400
parents 16932cefabc1
children 73b124160911
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
998
933670761a57 updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 988
diff changeset
1 #! /usr/bin/env python3
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
2
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
3 import sys, argparse
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
4 from pathlib import Path
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
5 from multiprocessing.pool import Pool
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
6
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
7 #import matplotlib
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
8 #atplotlib.use('Agg')
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
9 import matplotlib.pyplot as plt
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
10 from numpy import percentile
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
11 from pandas import DataFrame
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
12
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
13 import storage, events, prediction, cvutils, utils
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
14 from metadata import *
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
15
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
17 # input
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
21
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
22 # main function
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction'])
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
26 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction'])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
27
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
28 # common options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
29 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
30 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
1007
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
31 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
32 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
33
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
34 # analysis options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
35 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
36 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
37 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = float, default = 15.)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
38 parser.add_argument('--aggregation', dest = 'aggMethod', help = 'aggregation method per user/event and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
39 parser.add_argument('--aggregation-centile', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
40 dpi = 150
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
41 # unit of analysis: site or video sequence?
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
42
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
43 # safety analysis
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
44 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
45 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
46 # override other tracking config, erase sqlite?
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
47
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
48 # need way of selecting sites as similar as possible to sql alchemy syntax
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
49 # override tracking.cfg from db
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
50 # manage cfg files, overwrite them (or a subset of parameters)
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
51 # delete sqlite files
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
52 # info of metadata
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
53
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
54 args = parser.parse_args()
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
55
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
56 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
57 # Data preparation
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
58 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
59 session = connectDatabase(args.metadataFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
60 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
61 videoSequences = []
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
62 if args.videoIds is not None:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
63 videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds]
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
64 elif args.siteIds is not None:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
65 for siteId in args.siteIds:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
66 for site in getSite(session, siteId):
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
67 for cv in site.cameraViews:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
68 videoSequences += cv.videoSequences
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
69 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
70 print('No video/site to process')
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
71
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
72 if args.nProcesses > 1:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
73 pool = Pool(args.nProcesses)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
74
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
75 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
76 # Delete
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
77 #################################
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
78 if args.delete is not None:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
79 if args.delete == 'feature':
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
80 pass
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
81 elif args.delete in ['object', 'interaction']:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
82 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype'])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
83 for vs in videoSequences:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
84 storage.deleteFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.delete)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
85
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
86 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
87 # Process
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
88 #################################
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
89 if args.process in ['feature', 'object']: # tracking
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
90 if args.nProcesses == 1:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
91 for vs in videoSequences:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
92 if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object':
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
93 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
94 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
95 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
96 configFilename = args.configFilename
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
97 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
98 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
99 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
100 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
101 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
102 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
103 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
104 for vs in videoSequences:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
105 if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object':
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
106 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
107 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
108 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
109 configFilename = args.configFilename
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
110 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
111 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
112 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
113 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
114 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
115 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
116 pool.close()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
117 pool.join()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
118
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
119 elif args.process == 'interaction':
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
120 # safety analysis TODO make function in safety analysis script
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
121 if args.predictionMethod == 'cvd':
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
122 predictionParameters = prediction.CVDirectPredictionParameters()
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
123 if args.predictionMethod == 'cve':
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
124 predictionParameters = prediction.CVExactPredictionParameters()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
125 for vs in videoSequences:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
126 print('Processing '+vs.getDatabaseFilename())
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
127 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
128 interactions = events.createInteractions(objects)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
129 #if args.nProcesses == 1:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
130 #print(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
131 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
132 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
133 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
134 storage.saveIndicatorsToSqlite(str(parentPath/vs.getDatabaseFilename()), processed)
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
135 # else:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
136 # pool = Pool(processes = args.nProcesses)
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
137 # nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses)))
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
138 # jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], not args.noMotionPrediction, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones, False, None)) for i in range(args.nProcesses)]
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
139 # processed = []
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
140 # for job in jobs:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
141 # processed += job.get()
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
142 # pool.close()
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
143
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
144 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
145 # Analyze
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
146 #################################
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
147 if args.analyze == 'object':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
148 # user speeds, accelerations
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
149 # aggregation per site
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
150 data = [] # list of observation per site-user with time
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
151 headers = ['sites', 'date', 'time', 'user_type']
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
152 aggFunctions = {}
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
153 for method in args.aggMethod:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
154 if method == 'centile':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
155 aggFunctions[method] = utils.aggregationFunction(method, args.aggCentiles)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
156 for c in args.aggCentiles:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
157 headers.append('{}{}'.format(method,c))
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
158 else:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
159 aggFunctions[method] = utils.aggregationFunction(method)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
160 headers.append(method)
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
161 for vs in videoSequences:
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
162 d = vs.startTime.date()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
163 t1 = vs.startTime.time()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
164 minUserDuration = args.minUserDuration*vs.cameraView.cameraType.frameRate
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
165 print('Extracting speed from '+vs.getDatabaseFilename())
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
166 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object', args.nObjects)
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
167 for o in objects:
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
168 if o.length() > minUserDuration:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
169 row = [vs.cameraView.siteIdx, d, utils.framesToTime(o.getFirstInstant(), vs.cameraView.cameraType.frameRate, t1), o.getUserType()]
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
170 tmp = o.getSpeeds()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
171 for method,func in aggFunctions.items():
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
172 aggSpeeds = vs.cameraView.cameraType.frameRate*3.6*func(tmp)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
173 if method == 'centile':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
174 row += aggSpeeds.tolist()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
175 else:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
176 row.append(aggSpeeds)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
177 data.append(row)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
178 data = DataFrame(data, columns = headers)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
179 if args.siteIds is None:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
180 siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
181 else:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
182 siteIds = set(args.siteIds)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
183 if args.output == 'figure':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
184 for name in headers[4:]:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
185 plt.ioff()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
186 plt.figure()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
187 plt.boxplot([data.loc[data['sites']==siteId, name] for siteId in siteIds], labels = [session.query(Site).get(siteId).name for siteId in siteIds])
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
188 plt.ylabel(name+' Speeds (km/h)')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
189 plt.savefig(name.lower()+'-speeds.png', dpi=dpi)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
190 plt.close()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
191 elif args.output == 'event':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
192 data.to_csv('speeds.csv', index = False)
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
193 if args.analyze == 'interaction':
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
194 indicatorIds = [2,5,7,10]
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
195 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
196 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
197 indicators = {}
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
198 interactions = {}
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
199 for vs in videoSequences:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
200 if not vs.cameraView.siteIdx in interactions:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
201 interactions[vs.cameraView.siteIdx] = []
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
202 indicators[vs.cameraView.siteIdx] = {}
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
203 for i in indicatorIds:
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
204 indicators[vs.cameraView.siteIdx][i] = []
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
205 interactions[vs.cameraView.siteIdx] += storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
206 print(vs.getDatabaseFilename(), len(interactions[vs.cameraView.siteIdx]))
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
207 for inter in interactions[vs.cameraView.siteIdx]:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
208 for i in indicatorIds:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
209 indic = inter.getIndicator(events.Interaction.indicatorNames[i])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
210 if indic is not None:
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
211 v = indic.getMostSevereValue()*conversionFactors[i]
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
212 if v < maxIndicatorValue[i]:
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
213 indicators[vs.cameraView.siteIdx][i].append(v)
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
214
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
215 for i in indicatorIds:
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
216 tmp = [indicators[siteId][i] for siteId in indicators]
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
217 plt.ioff()
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
218 plt.figure()
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
219 plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
220 plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')')
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
221 plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150)
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
222 plt.close()