Mercurial Hosting > traffic-intelligence

annotate scripts/process.py @ 1278:8e61ff3cd503 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
correct bug to take into account first frame num in config, and other related bugs in dltrack.py
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Thu, 27 Jun 2024 15:31:36 -0400
parents b968c33f8c2f
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
998
933670761a57 updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 988
diff changeset
1 #! /usr/bin/env python3
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
2
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
3 import sys, argparse
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
4 from pathlib import Path
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
5 from multiprocessing.pool import Pool
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
6
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
7 #import matplotlib
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
8 #atplotlib.use('Agg')
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
9 import matplotlib.pyplot as plt
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
10 import numpy as np
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
11 import pandas as pd
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
12
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
13 from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml
1028
cc5cb04b04b0 major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1026
diff changeset
14 from trafficintelligence.metadata import *
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
15
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
17 # input
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*')
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
21
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
22 # main function
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
1036
0d7e5e290ea3 upload to pypi
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1028
diff changeset
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction'])
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
26 parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true')
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'collision-map', 'event-speed', 'event-interaction'])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
28
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
29 # common options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
30 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
31 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
1044
75a6ad604cc5 work on motion patterns
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1043
diff changeset
32 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature')
1007
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
33 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
34 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
35 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int)
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
36
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
37 ### process options
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
38 # motion pattern learning and assignment
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
39 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes', default = 'prototypes.sqlite')
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
40 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
41 parser.add_argument('--nobjects-mp', dest = 'nMPObjects', help = 'number of objects/interactions to process', type = int)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
42 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
43 parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float)
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
44 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
45 parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float)
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
46 parser.add_argument('--min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
47 #parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
48 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
49 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
50 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
51 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
52 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
53 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
54
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
55 # safety analysis
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
56 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
57 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
58 # override other tracking config, erase sqlite?
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
59
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
60
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
61 # analysis options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float)
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
64 parser.add_argument('--max-time-indicator-value', dest = 'maxTimeIndicatorValue', help = 'maximum indicator value for time indicators like PET and TTC (s)', type = float)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
65 parser.add_argument('--max-speed-indicator-value', dest = 'maxSpeedIndicatorValue', help = 'maximum indicator value for speed indicators like individual speed statistics and speed differential (km/h)', type = float)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
66 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
67 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
68 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
69 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
70 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
71 dpi = 150
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
72 # unit of analysis: site - camera-view
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
73
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
74 # need way of selecting sites as similar as possible to sql alchemy syntax
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
75 # override tracking.cfg from db
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
76 # manage cfg files, overwrite them (or a subset of parameters)
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
77 # delete sqlite files
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
78 # info of metadata
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
79
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
80 args = parser.parse_args()
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
81
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
82 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
83 # Data preparation
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
84 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
85 session = connectDatabase(args.metadataFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
86 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
87 videoSequences = []
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
88 sites = []
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
89 if args.videoIds is not None:
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
90 for videoId in args.videoIds:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
91 if '-' in videoId:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
92 videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)])
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
93 else:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
94 videoSequences.append(session.query(VideoSequence).get(int(videoId)))
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
95 videoSequences = [vs for vs in videoSequences if vs is not None]
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
96 sites = set([vs.cameraView.site for vs in videoSequences])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
97 elif args.siteIds is not None:
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
98 for siteId in args.siteIds:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
99 if '-' in siteId:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
100 sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)])
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
101 else:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
102 sites.append(session.query(Site).get(int(siteId)))
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
103 sites = [s for s in sites if s is not None]
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
104 for site in sites:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
105 videoSequences.extend(getSiteVideoSequences(site))
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
106 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
107 print('No video/site to process')
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
108
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
109 if args.nProcesses > 1:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
110 pool = Pool(args.nProcesses)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
111
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
112 #################################
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
113 # Report progress in the processing
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
114 #################################
1073
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
115 if args.progress: # TODO find video sequences that have null camera view, to work with them
1082
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
116 print('Providing information on progress of data processing')
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
117 headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
118 data = []
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
119 for site in sites:
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
120 unprocessedVideoSequences = []
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
121 for vs in getSiteVideoSequences(site):
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
122 if (parentPath/vs.getDatabaseFilename()).is_file(): # TODO check time of file?
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
123 tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename()))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
124 data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames])
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
125 else:
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
126 unprocessedVideoSequences.append(vs)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
127 data.append([site.name, vs.idx, False, False, False])
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
128 #if len(unprocessedVideoSequences):
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
129 # print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences)))
1082
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
130 videoSequences = session.query(VideoSequence).filter(VideoSequence.cameraViewIdx.is_(None)).all()
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
131 data = pd.DataFrame(data, columns = headers)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
132 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
133 print('\t'+' '.join(headers[2:]))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
134 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
135 for name, group in data.groupby(['site']): #.agg({'vs': 'count'}))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
136 n = group.vs.count()
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
137 print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
138 print('-'*80)
1082
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
139 if len(videoSequences) > 0:
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
140 print('{} video sequences without a camera view:'.format(len(videoSequences)))
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
141 print([vs.idx for vs in videoSequences])
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
142 print('-'*80)
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
143 with pd.option_context('display.max_rows', None, 'display.max_columns', None):
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
144 print(data)
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
145
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
146 #################################
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
147 # Delete
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
148 #################################
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
149 if args.delete is not None:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
150 if args.delete == 'feature':
1026
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
151 response = input('Are you sure you want to delete the tracking results (SQLite files) of all these sites (y/n)?')
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
152 if response == 'y':
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
153 for vs in videoSequences:
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
154 p = parentPath.absolute()/vs.getDatabaseFilename()
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
155 p.unlink()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
156 elif args.delete in ['object', 'interaction']:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
157 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype'])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
158 for vs in videoSequences:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
159 storage.deleteFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.delete)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
160
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
161 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
162 # Process
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
163 #################################
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
164 if args.process in ['feature', 'object']: # tracking
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
165 if args.nProcesses == 1:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
166 for vs in videoSequences:
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
167 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
168 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
169 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
170 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
171 configFilename = args.configFilename
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
172 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
173 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
174 else: #caution: cameratype can be not none, but without parameters for undistortion
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
175 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
176 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
177 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
178 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
179 for vs in videoSequences:
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
180 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
181 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
182 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
183 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
184 configFilename = args.configFilename
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
185 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
186 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
187 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
188 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
189 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
190 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
191 pool.close()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
192 pool.join()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
193
1039
5621e4ad2428 removing prefix option to loadtrajectories from SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1036
diff changeset
194 elif args.process == 'prototype': # motion pattern learning
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
195 # learn by site by default -> group videos by camera view TODO
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
196 # by default, load all objects, learn and then assign (BUT not save the assignments)
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
197 for site in sites:
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
198 print('Learning motion patterns for site {} ({})'.format(site.idx, site.name))
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
199 objects = {}
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
200 object2VideoSequences = {}
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
201 for cv in site.cameraViews:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
202 for vs in cv.videoSequences:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
203 print('Loading '+vs.getDatabaseFilename())
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
204 objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
205 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
206 objectsWithFeatures = objects[vs.idx]
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
207 objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()]
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
208 prototypeType = 'feature'
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
209 else:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
210 prototypeType = args.trajectoryType
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
211 for obj in objects[vs.idx]:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
212 object2VideoSequences[obj] = vs
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
213 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
214 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
215 trainingObjects = [o for tmpobjects in objects.values() for o in tmpobjects]
1049
c9c03c97ed9f bug fix to store numpy integers in SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1048
diff changeset
216 if args.nMPObjects is not None and args.nMPObjects < len(trainingObjects):
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
217 m = int(np.floor(float(len(trainingObjects))/args.nMPObjects))
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
218 trainingObjects = trainingObjects[::m]
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
219 similarities = -np.ones((len(trainingObjects), len(trainingObjects)))
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
220 prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, trainingObjects, similarities, args.minSimilarity, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, [])
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
221 if args.outputPrototypeDatabaseFilename is None:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
222 outputPrototypeDatabaseFilename = args.databaseFilename
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
223 else:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
224 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
225 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
226 storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices])
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
227
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
228 elif args.process == 'interaction':
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
229 # safety analysis TODO make function in safety analysis script
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
230 if args.predictionMethod == 'cvd':
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
231 predictionParameters = prediction.CVDirectPredictionParameters()
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
232 elif args.predictionMethod == 'cve':
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
233 predictionParameters = prediction.CVExactPredictionParameters()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
234 for vs in videoSequences:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
235 print('Processing '+vs.getDatabaseFilename())
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
236 if args.configFilename is None:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
237 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
238 else:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
239 params = storage.ProcessParameters(args.configFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
240 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
241 interactions = events.createInteractions(objects)
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
242 if args.nProcesses == 1:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
243 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones)
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
244 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None) # params.crossingZones
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
245 else:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
246 #pool = Pool(processes = args.nProcesses)
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
247 nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses)))
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
248 jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None)) for i in range(args.nProcesses)] # params.crossingZones
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
249 processed = []
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
250 for job in jobs:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
251 processed += job.get()
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
252 #pool.close()
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
253 storage.saveIndicatorsToSqlite(str(parentPath/vs.getDatabaseFilename()), processed)
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
254
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
255 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
256 # Analyze
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
257 #################################
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
258 if args.analyze == 'object':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
259 # user speeds, accelerations
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
260 # aggregation per site
1068
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
261 if args.eventFilename is None:
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
262 print('Missing output filename (event-filename). Exiting')
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
263 sys.exit(0)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
264 data = [] # list of observation per site-user with time
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
265 headers = ['site', 'date', 'time', 'user_type']
1062
a2e20aba0740 bug correctiong
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1061
diff changeset
266 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
267 headers.extend(tmpheaders)
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
268 if args.nProcesses == 1:
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
269 for vs in videoSequences:
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
270 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles))
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
271 else:
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
272 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences]
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
273 for job in jobs:
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
274 data.extend(job.get())
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
275 pool.close()
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
276 data = pd.DataFrame(data, columns = headers)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
277 if args.output == 'figure':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
278 for name in headers[4:]:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
279 plt.ioff()
1068
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
280 plt.figure()
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
281 plt.boxplot([data.loc[data['site']==site.name, name] for site in sites], labels = [site.name for site in sites])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
282 plt.ylabel(name+' Speeds (km/h)')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
283 plt.savefig(name.lower()+'-speeds.png', dpi=dpi)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
284 plt.close()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
285 elif args.output == 'event':
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
286 data.to_csv(args.eventFilename, index = False)
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
287
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
288 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
289 indicatorIds = [2,5,7,10]
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
290 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
291 data = [] # list of observation per site-user with time
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
292 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types?
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
293 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
294 nAggFunctions = len(tmpheaders)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
295 indicatorUnits = [events.Interaction.indicatorUnits[10]] # for PET above
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
296 for i in indicatorIds[:3]:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
297 for h in tmpheaders:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
298 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h)
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
299 indicatorUnits.append(events.Interaction.indicatorUnits[i])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
300 for vs in videoSequences:
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
301 print('Extracting SMoS from '+vs.getDatabaseFilename())
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
302 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
303 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
304 conversionFactors = {2: 1., 5: 3.6*vs.cameraView.cameraType.frameRate, 7:1./vs.cameraView.cameraType.frameRate, 10:1./vs.cameraView.cameraType.frameRate}
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
305 for inter in interactions:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
306 if inter.length() > minDuration:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
307 d = vs.startTime.date()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
308 t = vs.startTime.time()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
309 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)]
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
310 pet = inter.getIndicator(events.Interaction.indicatorNames[10])
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
311 if pet is None:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
312 row.append(None)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
313 else:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
314 row.append(conversionFactors[10]*pet.getValues()[0])
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
315 for i in indicatorIds[:3]:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
316 indic = inter.getIndicator(events.Interaction.indicatorNames[i])
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
317 if indic is not None:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
318 #v = indic.getMostSevereValue()*
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
319 tmp = list(indic.values.values())
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
320 for method,func in aggFunctions.items():
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
321 agg = conversionFactors[i]*func(tmp)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
322 if method == 'centile':
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
323 row.extend(agg.tolist())
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
324 else:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
325 row.append(agg)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
326 else:
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
327 row.extend([None]*nAggFunctions)
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
328 data.append(row)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
329 data = pd.DataFrame(data, columns = headers)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
330 if args.output == 'figure':
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
331 plt.ioff()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
332 for i, indic in enumerate(headers[3:]):
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
333 if 'Time' in indic and args.maxTimeIndicatorValue is not None:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
334 tmp = data.loc[data[indic] < args.maxTimeIndicatorValue, ['site', indic]]
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
335 elif 'Speed' in indic and args.maxSpeedIndicatorValue is not None:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
336 tmp = data.loc[data[indic] < args.maxSpeedIndicatorValue, ['site', indic]]
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
337 else:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
338 tmp = data[['site', indic]]
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
339 plt.figure()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
340 tmp.boxplot(indic, 'site')
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
341 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
342 plt.ylabel(indic+' ('+indicatorUnits[i]+')')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
343 plt.savefig('boxplot-sites-'+indic+'.pdf')#, dpi=150)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
344 plt.close()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
345 plt.figure()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
346 for site in sorted(tmp.site.unique()):
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
347 x = sorted(tmp.loc[tmp.site == site, indic])
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
348 plt.plot(x, np.arange(1,len(x)+1)/len(x), label=site)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
349 plt.legend()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
350 plt.title('Cumulative Distribution Function by Site')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
351 plt.xlabel(indic+' ('+indicatorUnits[i]+')')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
352 plt.savefig('cdf-sites-'+indic+'.pdf')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
353 plt.close()
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
354 elif args.output == 'event':
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
355 data.to_csv(args.eventFilename, index = False)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
356
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
357 if args.analyze == 'collision-map':
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
358 predictionParameters = prediction.CVExactPredictionParameters()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
359 data = []
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
360 for vs in videoSequences:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
361 print('Extracting potential collision points from '+vs.getDatabaseFilename())
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
362 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
363 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
364 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
365 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
366 maxTimeIndicatorValue = vs.cameraView.cameraType.frameRate*args.maxTimeIndicatorValue
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
367 for inter in interactions:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
368 if inter.length() > minDuration:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
369 ttc = inter.getIndicator(events.Interaction.indicatorNames[7])
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
370 if ttc is not None:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
371 t = min(ttc.values, key = ttc.values.get)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
372 if args.maxTimeIndicatorValue is None or ttc.values[t] < maxTimeIndicatorValue:
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
373 inter.setRoadUsers(objects)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
374 cps, _ = predictionParameters.computeCrossingsCollisionsAtInstant(t, inter.roadUser1, inter.roadUser2, params.collisionDistance, params.predictionTimeHorizon)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
375 data.append([vs.cameraView.site.name, cps[0].x, cps[0].y, cps[0].indicator])
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
376 data = pd.DataFrame(data, columns = ['site', 'x', 'y', 'ttc'])
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
377 margin = 0.1
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
378 for site in data.site.unique():
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
379 s = session.query(Site).filter(Site.name.like('%'+site+'%')).first()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
380 img = plt.imread(str(parentPath/s.getMapImageFilename()))
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
381 tmp = data[data.site == site].copy()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
382 tmp.x = tmp.x/s.nUnitsPerPixel
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
383 tmp.y = tmp.y/s.nUnitsPerPixel
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
384 h, w, _ = img.shape
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
385 tmp = tmp[(tmp.x>-margin*w) & (tmp.x < (1+margin)*w) & (tmp.y > -margin*h) & (tmp.y < (1+margin)*h)]
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
386 plt.figure()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
387 plt.imshow(img)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
388 plt.hexbin(tmp.x, tmp.y, alpha = 0.5, edgecolors = 'face', mincnt=1, gridsize=50)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
389 plt.title('Density of Potential Collision Points at Site '+site)
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
390 plt.colorbar()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
391 plt.axis('equal')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
392 plt.savefig('collision-map-'+site+'.pdf')
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
393 #plt.close()
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
394
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
395 if args.analyze == 'event-speed': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
1084
1a7e0b2c858b remove debugging
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1083
diff changeset
396 data = pd.read_csv(args.eventFilename, parse_dates = [2])
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
397 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
398 # create time for end of each 15 min, then group by, using the agg method for each data column
1061
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1059
diff changeset
399 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
400 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
401 dataColumns = list(data.columns[4:])
1086
8734742c08c0 major refactoring of curvilinear trajectory projections
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1084
diff changeset
402 print(dataColumns)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
403 for h in dataColumns:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
404 for h2 in tmpheaders:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
405 headers.append(h+'-'+h2)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
406 if args.eventThresholds is not None:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
407 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
408 for t in args.eventThresholds:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
409 headers.append('n-{}-{}'.format(h, t))
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
410 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time())
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
411 outputData = []
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
412 for name, group in data.groupby(['site', 'date', 'intervalend15']):
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
413 row = []
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
414 row.extend(name)
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
415 groupStartTime = group.time.min()
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
416 groupEndTime = group.time.max()
1070
0154133e77df corrected bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1069
diff changeset
417 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
418 row.append(len(group))
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
419 for h in dataColumns:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
420 for method,func in aggFunctions.items():
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
421 aggregated = func(group[h])
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
422 if method == 'centile':
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
423 row.extend(aggregated)
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
424 else:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
425 row.append(aggregated)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
426 if args.eventThresholds is not None:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
427 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
428 for t in args.eventThresholds:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
429 row.append((group[h] > t).sum())
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
430 outputData.append(row)
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
431 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
432
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
433 elif args.analyze == 'event-interaction': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
1084
1a7e0b2c858b remove debugging
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1083
diff changeset
434 data = pd.read_csv(args.eventFilename, parse_dates = [2])
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
435 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
436 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
437 nAggFunctions = len(tmpheaders)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
438 dataColumns = list(data.columns[3:])
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
439 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
440 if not 'speed' in h.lower(): # proximity indicators are reversed, taking 85th centile of this column will yield the 15th centile (which we have to take the opposite again)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
441 data[h] = -data[h]
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
442 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
443 for h2 in tmpheaders:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
444 headers.append(h+'-'+h2)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
445 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
446 headers.append('n-{}-{}'.format(h, t))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
447 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time())
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
448 outputData = []
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
449 for name, group in data.groupby(['site', 'date', 'intervalend15']):
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
450 row = []
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
451 row.extend(name)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
452 groupStartTime = group.time.min()
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
453 groupEndTime = group.time.max()
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
454 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
455 row.append(len(group))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
456 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
457 for method,func in aggFunctions.items():
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
458 tmp = group.loc[~group[h].isna(), h]
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
459 if len(tmp)>0:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
460 aggregated = func(tmp) # todo invert if the resulting stat is negative
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
461 if method == 'centile':
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
462 row.extend(np.abs(aggregated))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
463 else:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
464 row.append(np.abs(aggregated))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
465 else:
1161
b968c33f8c2f work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1086
diff changeset
466 row.extend([None]*nAggFunctions)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
467 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
468 if 'speed' in h.lower():
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
469 row.append((group[h] > t).sum())
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
470 else:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
471 row.append((group[h] > -t).sum()) # take larger than than negative threshold for proximity indicators
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
472 outputData.append(row)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
473 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False)