Mercurial Hosting > traffic-intelligence
annotate scripts/process.py @ 1253:ef68d4ba7dae
added loading ego vehicle in kitti 2D format and method to plot outline
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Mon, 25 Mar 2024 17:05:20 -0400 |
parents | b968c33f8c2f |
children |
rev | line source |
---|---|
998
933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
988
diff
changeset
|
1 #! /usr/bin/env python3 |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
2 |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
3 import sys, argparse |
1003
75af46516b2b
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
4 from pathlib import Path |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
5 from multiprocessing.pool import Pool |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
6 |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
7 #import matplotlib |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
8 #atplotlib.use('Agg') |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
9 import matplotlib.pyplot as plt |
1048
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
10 import numpy as np |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
11 import pandas as pd |
986
3be8aaa47651
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
12 |
1046
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
13 from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml |
1028
cc5cb04b04b0
major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1026
diff
changeset
|
14 from trafficintelligence.metadata import * |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
15 |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
17 # input |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*') |
1046
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*') |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
21 |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
22 # main function |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) |
1036
0d7e5e290ea3
upload to pypi
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1028
diff
changeset
|
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction']) |
1003
75af46516b2b
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) |
1063
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
26 parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true') |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'collision-map', 'event-speed', 'event-interaction']) |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
28 |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
29 # common options |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
30 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
31 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int) |
1044
75a6ad604cc5
work on motion patterns
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1043
diff
changeset
|
32 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature') |
1007
192de96e5255
solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1006
diff
changeset
|
33 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true') |
192de96e5255
solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1006
diff
changeset
|
34 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) |
1043
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
35 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int) |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
36 |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
37 ### process options |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
38 # motion pattern learning and assignment |
1046
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
39 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes', default = 'prototypes.sqlite') |
1043
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
40 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with') |
1048
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
41 parser.add_argument('--nobjects-mp', dest = 'nMPObjects', help = 'number of objects/interactions to process', type = int) |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
42 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int) |
1046
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
43 parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float) |
1043
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
44 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance |
1046
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
45 parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float) |
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
46 parser.add_argument('--min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0) |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
47 #parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true') |
1043
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
48 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
49 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
50 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
51 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
52 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
53 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
54 |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
55 # safety analysis |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
56 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp']) |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
57 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true') |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
58 # override other tracking config, erase sqlite? |
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
59 |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
60 |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
61 # analysis options |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float) |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
64 parser.add_argument('--max-time-indicator-value', dest = 'maxTimeIndicatorValue', help = 'maximum indicator value for time indicators like PET and TTC (s)', type = float) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
65 parser.add_argument('--max-speed-indicator-value', dest = 'maxSpeedIndicatorValue', help = 'maximum indicator value for speed indicators like individual speed statistics and speed differential (km/h)', type = float) |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
66 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15) |
1058
16575ca4537d
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1050
diff
changeset
|
67 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) |
1069
9ee5c7636640
added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1068
diff
changeset
|
68 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) |
9ee5c7636640
added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1068
diff
changeset
|
69 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float) |
1058
16575ca4537d
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1050
diff
changeset
|
70 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data') |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
71 dpi = 150 |
1050
9d4a06f49cb8
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1049
diff
changeset
|
72 # unit of analysis: site - camera-view |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
73 |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
74 # need way of selecting sites as similar as possible to sql alchemy syntax |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
75 # override tracking.cfg from db |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
76 # manage cfg files, overwrite them (or a subset of parameters) |
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
77 # delete sqlite files |
1003
75af46516b2b
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
78 # info of metadata |
75af46516b2b
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
79 |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
80 args = parser.parse_args() |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
81 |
1009 | 82 ################################# |
83 # Data preparation | |
84 ################################# | |
85 session = connectDatabase(args.metadataFilename) | |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
86 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location |
1009 | 87 videoSequences = [] |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
88 sites = [] |
1009 | 89 if args.videoIds is not None: |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
90 for videoId in args.videoIds: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
91 if '-' in videoId: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
92 videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)]) |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
93 else: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
94 videoSequences.append(session.query(VideoSequence).get(int(videoId))) |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
95 videoSequences = [vs for vs in videoSequences if vs is not None] |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
96 sites = set([vs.cameraView.site for vs in videoSequences]) |
1009 | 97 elif args.siteIds is not None: |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
98 for siteId in args.siteIds: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
99 if '-' in siteId: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
100 sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)]) |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
101 else: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
102 sites.append(session.query(Site).get(int(siteId))) |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
103 sites = [s for s in sites if s is not None] |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
104 for site in sites: |
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
105 videoSequences.extend(getSiteVideoSequences(site)) |
1009 | 106 else: |
107 print('No video/site to process') | |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
108 |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
109 if args.nProcesses > 1: |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
110 pool = Pool(args.nProcesses) |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
111 |
1009 | 112 ################################# |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
113 # Report progress in the processing |
1009 | 114 ################################# |
1073 | 115 if args.progress: # TODO find video sequences that have null camera view, to work with them |
1082
706034a4c6cd
more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1074
diff
changeset
|
116 print('Providing information on progress of data processing') |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
117 headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
118 data = [] |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
119 for site in sites: |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
120 unprocessedVideoSequences = [] |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
121 for vs in getSiteVideoSequences(site): |
1066
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
122 if (parentPath/vs.getDatabaseFilename()).is_file(): # TODO check time of file? |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
123 tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename())) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
124 data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames]) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
125 else: |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
126 unprocessedVideoSequences.append(vs) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
127 data.append([site.name, vs.idx, False, False, False]) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
128 #if len(unprocessedVideoSequences): |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
129 # print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences))) |
1082
706034a4c6cd
more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1074
diff
changeset
|
130 videoSequences = session.query(VideoSequence).filter(VideoSequence.cameraViewIdx.is_(None)).all() |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
131 data = pd.DataFrame(data, columns = headers) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
132 print('-'*80) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
133 print('\t'+' '.join(headers[2:])) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
134 print('-'*80) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
135 for name, group in data.groupby(['site']): #.agg({'vs': 'count'})) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
136 n = group.vs.count() |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
137 print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n)) |
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
138 print('-'*80) |
1082
706034a4c6cd
more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1074
diff
changeset
|
139 if len(videoSequences) > 0: |
706034a4c6cd
more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1074
diff
changeset
|
140 print('{} video sequences without a camera view:'.format(len(videoSequences))) |
706034a4c6cd
more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1074
diff
changeset
|
141 print([vs.idx for vs in videoSequences]) |
706034a4c6cd
more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1074
diff
changeset
|
142 print('-'*80) |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
143 with pd.option_context('display.max_rows', None, 'display.max_columns', None): |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
144 print(data) |
1063
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
145 |
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
146 ################################# |
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
147 # Delete |
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
148 ################################# |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
149 if args.delete is not None: |
1009 | 150 if args.delete == 'feature': |
1026
73b124160911
more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1023
diff
changeset
|
151 response = input('Are you sure you want to delete the tracking results (SQLite files) of all these sites (y/n)?') |
73b124160911
more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1023
diff
changeset
|
152 if response == 'y': |
73b124160911
more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1023
diff
changeset
|
153 for vs in videoSequences: |
73b124160911
more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1023
diff
changeset
|
154 p = parentPath.absolute()/vs.getDatabaseFilename() |
73b124160911
more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1023
diff
changeset
|
155 p.unlink() |
1009 | 156 elif args.delete in ['object', 'interaction']: |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
157 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype']) |
1009 | 158 for vs in videoSequences: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
159 storage.deleteFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.delete) |
984
a69695d14e59
work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
160 |
1009 | 161 ################################# |
162 # Process | |
163 ################################# | |
1003
75af46516b2b
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
164 if args.process in ['feature', 'object']: # tracking |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
165 if args.nProcesses == 1: |
1009 | 166 for vs in videoSequences: |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
167 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object': |
1009 | 168 if args.configFilename is None: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
169 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) |
1009 | 170 else: |
171 configFilename = args.configFilename | |
172 if vs.cameraView.cameraType is None: | |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
173 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun) |
1066
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
174 else: #caution: cameratype can be not none, but without parameters for undistortion |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
175 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun) |
1009 | 176 else: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
177 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename())) |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
178 else: |
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
179 for vs in videoSequences: |
1065
d4d052a05337
added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1064
diff
changeset
|
180 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object': |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
181 if args.configFilename is None: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
182 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
183 else: |
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
184 configFilename = args.configFilename |
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
185 if vs.cameraView.cameraType is None: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
186 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)) |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
187 else: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
188 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)) |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
189 else: |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
190 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename())) |
1004
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
191 pool.close() |
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
192 pool.join() |
75601be6019f
work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1003
diff
changeset
|
193 |
1039
5621e4ad2428
removing prefix option to loadtrajectories from SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1036
diff
changeset
|
194 elif args.process == 'prototype': # motion pattern learning |
1050
9d4a06f49cb8
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1049
diff
changeset
|
195 # learn by site by default -> group videos by camera view TODO |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
196 # by default, load all objects, learn and then assign (BUT not save the assignments) |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
197 for site in sites: |
1048
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
198 print('Learning motion patterns for site {} ({})'.format(site.idx, site.name)) |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
199 objects = {} |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
200 object2VideoSequences = {} |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
201 for cv in site.cameraViews: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
202 for vs in cv.videoSequences: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
203 print('Loading '+vs.getDatabaseFilename()) |
1046
f2ba9858e6c6
motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1045
diff
changeset
|
204 objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject) |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
205 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
206 objectsWithFeatures = objects[vs.idx] |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
207 objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()] |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
208 prototypeType = 'feature' |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
209 else: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
210 prototypeType = args.trajectoryType |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
211 for obj in objects[vs.idx]: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
212 object2VideoSequences[obj] = vs |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
213 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon) |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
214 similarityFunc = lambda x,y : lcss.computeNormalized(x, y) |
1048
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
215 trainingObjects = [o for tmpobjects in objects.values() for o in tmpobjects] |
1049
c9c03c97ed9f
bug fix to store numpy integers in SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1048
diff
changeset
|
216 if args.nMPObjects is not None and args.nMPObjects < len(trainingObjects): |
1048
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
217 m = int(np.floor(float(len(trainingObjects))/args.nMPObjects)) |
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
218 trainingObjects = trainingObjects[::m] |
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
219 similarities = -np.ones((len(trainingObjects), len(trainingObjects))) |
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
220 prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, trainingObjects, similarities, args.minSimilarity, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, []) |
1045
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
221 if args.outputPrototypeDatabaseFilename is None: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
222 outputPrototypeDatabaseFilename = args.databaseFilename |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
223 else: |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
224 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename |
25db2383e7ae
work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1044
diff
changeset
|
225 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1) |
1048
27a822922cb0
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1047
diff
changeset
|
226 storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices]) |
1043
b735895c8815
work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1039
diff
changeset
|
227 |
1003
75af46516b2b
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
228 elif args.process == 'interaction': |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
229 # safety analysis TODO make function in safety analysis script |
987
f026ce2af637
found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
986
diff
changeset
|
230 if args.predictionMethod == 'cvd': |
f026ce2af637
found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
986
diff
changeset
|
231 predictionParameters = prediction.CVDirectPredictionParameters() |
1071
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
232 elif args.predictionMethod == 'cve': |
987
f026ce2af637
found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
986
diff
changeset
|
233 predictionParameters = prediction.CVExactPredictionParameters() |
1009 | 234 for vs in videoSequences: |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
235 print('Processing '+vs.getDatabaseFilename()) |
1071
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
236 if args.configFilename is None: |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
237 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename())) |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
238 else: |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
239 params = storage.ProcessParameters(args.configFilename) |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
240 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp')) |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
241 interactions = events.createInteractions(objects) |
1071
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
242 if args.nProcesses == 1: |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
243 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones) |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
244 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None) # params.crossingZones |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
245 else: |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
246 #pool = Pool(processes = args.nProcesses) |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
247 nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses))) |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
248 jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None)) for i in range(args.nProcesses)] # params.crossingZones |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
249 processed = [] |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
250 for job in jobs: |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
251 processed += job.get() |
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
252 #pool.close() |
1021
16932cefabc1
work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1009
diff
changeset
|
253 storage.saveIndicatorsToSqlite(str(parentPath/vs.getDatabaseFilename()), processed) |
1071
58994b08be42
added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1070
diff
changeset
|
254 |
1009 | 255 ################################# |
256 # Analyze | |
257 ################################# | |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
258 if args.analyze == 'object': |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
259 # user speeds, accelerations |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
260 # aggregation per site |
1068 | 261 if args.eventFilename is None: |
262 print('Missing output filename (event-filename). Exiting') | |
263 sys.exit(0) | |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
264 data = [] # list of observation per site-user with time |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
265 headers = ['site', 'date', 'time', 'user_type'] |
1062
a2e20aba0740
bug correctiong
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1061
diff
changeset
|
266 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) |
1058
16575ca4537d
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1050
diff
changeset
|
267 headers.extend(tmpheaders) |
1066
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
268 if args.nProcesses == 1: |
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
269 for vs in videoSequences: |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
270 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) |
1066
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
271 else: |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
272 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences] |
1066
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
273 for job in jobs: |
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
274 data.extend(job.get()) |
862b55a87e63
work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1065
diff
changeset
|
275 pool.close() |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
276 data = pd.DataFrame(data, columns = headers) |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
277 if args.output == 'figure': |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
278 for name in headers[4:]: |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
279 plt.ioff() |
1068 | 280 plt.figure() |
1064
cbc026dacf0b
changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1063
diff
changeset
|
281 plt.boxplot([data.loc[data['site']==site.name, name] for site in sites], labels = [site.name for site in sites]) |
1023
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
282 plt.ylabel(name+' Speeds (km/h)') |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
283 plt.savefig(name.lower()+'-speeds.png', dpi=dpi) |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
284 plt.close() |
a13f47c8931d
work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1021
diff
changeset
|
285 elif args.output == 'event': |
1063
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
286 data.to_csv(args.eventFilename, index = False) |
1050
9d4a06f49cb8
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1049
diff
changeset
|
287 |
9d4a06f49cb8
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1049
diff
changeset
|
288 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
984
diff
changeset
|
289 indicatorIds = [2,5,7,10] |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
290 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
291 data = [] # list of observation per site-user with time |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
292 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types? |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
293 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
294 nAggFunctions = len(tmpheaders) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
295 indicatorUnits = [events.Interaction.indicatorUnits[10]] # for PET above |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
296 for i in indicatorIds[:3]: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
297 for h in tmpheaders: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
298 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h) |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
299 indicatorUnits.append(events.Interaction.indicatorUnits[i]) |
1009 | 300 for vs in videoSequences: |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
301 print('Extracting SMoS from '+vs.getDatabaseFilename()) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
302 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename())) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
303 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
304 conversionFactors = {2: 1., 5: 3.6*vs.cameraView.cameraType.frameRate, 7:1./vs.cameraView.cameraType.frameRate, 10:1./vs.cameraView.cameraType.frameRate} |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
305 for inter in interactions: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
306 if inter.length() > minDuration: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
307 d = vs.startTime.date() |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
308 t = vs.startTime.time() |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
309 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)] |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
310 pet = inter.getIndicator(events.Interaction.indicatorNames[10]) |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
311 if pet is None: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
312 row.append(None) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
313 else: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
314 row.append(conversionFactors[10]*pet.getValues()[0]) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
315 for i in indicatorIds[:3]: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
316 indic = inter.getIndicator(events.Interaction.indicatorNames[i]) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
317 if indic is not None: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
318 #v = indic.getMostSevereValue()* |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
319 tmp = list(indic.values.values()) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
320 for method,func in aggFunctions.items(): |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
321 agg = conversionFactors[i]*func(tmp) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
322 if method == 'centile': |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
323 row.extend(agg.tolist()) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
324 else: |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
325 row.append(agg) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
326 else: |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
327 row.extend([None]*nAggFunctions) |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
328 data.append(row) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
329 data = pd.DataFrame(data, columns = headers) |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
330 if args.output == 'figure': |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
331 plt.ioff() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
332 for i, indic in enumerate(headers[3:]): |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
333 if 'Time' in indic and args.maxTimeIndicatorValue is not None: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
334 tmp = data.loc[data[indic] < args.maxTimeIndicatorValue, ['site', indic]] |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
335 elif 'Speed' in indic and args.maxSpeedIndicatorValue is not None: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
336 tmp = data.loc[data[indic] < args.maxSpeedIndicatorValue, ['site', indic]] |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
337 else: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
338 tmp = data[['site', indic]] |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
339 plt.figure() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
340 tmp.boxplot(indic, 'site') |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
341 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators]) |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
342 plt.ylabel(indic+' ('+indicatorUnits[i]+')') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
343 plt.savefig('boxplot-sites-'+indic+'.pdf')#, dpi=150) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
344 plt.close() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
345 plt.figure() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
346 for site in sorted(tmp.site.unique()): |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
347 x = sorted(tmp.loc[tmp.site == site, indic]) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
348 plt.plot(x, np.arange(1,len(x)+1)/len(x), label=site) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
349 plt.legend() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
350 plt.title('Cumulative Distribution Function by Site') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
351 plt.xlabel(indic+' ('+indicatorUnits[i]+')') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
352 plt.savefig('cdf-sites-'+indic+'.pdf') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
353 plt.close() |
1072
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
354 elif args.output == 'event': |
c67f8c36ebc7
interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1071
diff
changeset
|
355 data.to_csv(args.eventFilename, index = False) |
1058
16575ca4537d
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1050
diff
changeset
|
356 |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
357 if args.analyze == 'collision-map': |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
358 predictionParameters = prediction.CVExactPredictionParameters() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
359 data = [] |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
360 for vs in videoSequences: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
361 print('Extracting potential collision points from '+vs.getDatabaseFilename()) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
362 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename())) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
363 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
364 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename())) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
365 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
366 maxTimeIndicatorValue = vs.cameraView.cameraType.frameRate*args.maxTimeIndicatorValue |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
367 for inter in interactions: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
368 if inter.length() > minDuration: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
369 ttc = inter.getIndicator(events.Interaction.indicatorNames[7]) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
370 if ttc is not None: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
371 t = min(ttc.values, key = ttc.values.get) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
372 if args.maxTimeIndicatorValue is None or ttc.values[t] < maxTimeIndicatorValue: |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
373 inter.setRoadUsers(objects) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
374 cps, _ = predictionParameters.computeCrossingsCollisionsAtInstant(t, inter.roadUser1, inter.roadUser2, params.collisionDistance, params.predictionTimeHorizon) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
375 data.append([vs.cameraView.site.name, cps[0].x, cps[0].y, cps[0].indicator]) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
376 data = pd.DataFrame(data, columns = ['site', 'x', 'y', 'ttc']) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
377 margin = 0.1 |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
378 for site in data.site.unique(): |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
379 s = session.query(Site).filter(Site.name.like('%'+site+'%')).first() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
380 img = plt.imread(str(parentPath/s.getMapImageFilename())) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
381 tmp = data[data.site == site].copy() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
382 tmp.x = tmp.x/s.nUnitsPerPixel |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
383 tmp.y = tmp.y/s.nUnitsPerPixel |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
384 h, w, _ = img.shape |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
385 tmp = tmp[(tmp.x>-margin*w) & (tmp.x < (1+margin)*w) & (tmp.y > -margin*h) & (tmp.y < (1+margin)*h)] |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
386 plt.figure() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
387 plt.imshow(img) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
388 plt.hexbin(tmp.x, tmp.y, alpha = 0.5, edgecolors = 'face', mincnt=1, gridsize=50) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
389 plt.title('Density of Potential Collision Points at Site '+site) |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
390 plt.colorbar() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
391 plt.axis('equal') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
392 plt.savefig('collision-map-'+site+'.pdf') |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
393 #plt.close() |
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
394 |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
395 if args.analyze == 'event-speed': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds |
1084
1a7e0b2c858b
remove debugging
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1083
diff
changeset
|
396 data = pd.read_csv(args.eventFilename, parse_dates = [2]) |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
397 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000) |
1058
16575ca4537d
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1050
diff
changeset
|
398 # create time for end of each 15 min, then group by, using the agg method for each data column |
1061 | 399 headers = ['site', 'date', 'intervalend15', 'duration', 'count'] |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
400 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
401 dataColumns = list(data.columns[4:]) |
1086
8734742c08c0
major refactoring of curvilinear trajectory projections
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1084
diff
changeset
|
402 print(dataColumns) |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
403 for h in dataColumns: |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
404 for h2 in tmpheaders: |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
405 headers.append(h+'-'+h2) |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
406 if args.eventThresholds is not None: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
407 for h in dataColumns: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
408 for t in args.eventThresholds: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
409 headers.append('n-{}-{}'.format(h, t)) |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
410 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time()) |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
411 outputData = [] |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
412 for name, group in data.groupby(['site', 'date', 'intervalend15']): |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
413 row = [] |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
414 row.extend(name) |
1069
9ee5c7636640
added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1068
diff
changeset
|
415 groupStartTime = group.time.min() |
9ee5c7636640
added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1068
diff
changeset
|
416 groupEndTime = group.time.max() |
1070
0154133e77df
corrected bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1069
diff
changeset
|
417 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600) |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
418 row.append(len(group)) |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
419 for h in dataColumns: |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
420 for method,func in aggFunctions.items(): |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
421 aggregated = func(group[h]) |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
422 if method == 'centile': |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
423 row.extend(aggregated) |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
424 else: |
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
425 row.append(aggregated) |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
426 if args.eventThresholds is not None: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
427 for h in dataColumns: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
428 for t in args.eventThresholds: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
429 row.append((group[h] > t).sum()) |
1059
a87b3072bd26
working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1058
diff
changeset
|
430 outputData.append(row) |
1063
3c37d8d20e97
minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1062
diff
changeset
|
431 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False) |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
432 |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
433 elif args.analyze == 'event-interaction': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds |
1084
1a7e0b2c858b
remove debugging
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1083
diff
changeset
|
434 data = pd.read_csv(args.eventFilename, parse_dates = [2]) |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
435 headers = ['site', 'date', 'intervalend15', 'duration', 'count'] |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
436 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
437 nAggFunctions = len(tmpheaders) |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
438 dataColumns = list(data.columns[3:]) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
439 for h in dataColumns: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
440 if not 'speed' in h.lower(): # proximity indicators are reversed, taking 85th centile of this column will yield the 15th centile (which we have to take the opposite again) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
441 data[h] = -data[h] |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
442 for h in dataColumns: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
443 for h2 in tmpheaders: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
444 headers.append(h+'-'+h2) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
445 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
446 headers.append('n-{}-{}'.format(h, t)) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
447 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time()) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
448 outputData = [] |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
449 for name, group in data.groupby(['site', 'date', 'intervalend15']): |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
450 row = [] |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
451 row.extend(name) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
452 groupStartTime = group.time.min() |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
453 groupEndTime = group.time.max() |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
454 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
455 row.append(len(group)) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
456 for h in dataColumns: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
457 for method,func in aggFunctions.items(): |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
458 tmp = group.loc[~group[h].isna(), h] |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
459 if len(tmp)>0: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
460 aggregated = func(tmp) # todo invert if the resulting stat is negative |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
461 if method == 'centile': |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
462 row.extend(np.abs(aggregated)) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
463 else: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
464 row.append(np.abs(aggregated)) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
465 else: |
1161
b968c33f8c2f
work on producing figures and collision maps for large datasets with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1086
diff
changeset
|
466 row.extend([None]*nAggFunctions) |
1083
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
467 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
468 if 'speed' in h.lower(): |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
469 row.append((group[h] > t).sum()) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
470 else: |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
471 row.append((group[h] > -t).sum()) # take larger than than negative threshold for proximity indicators |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
472 outputData.append(row) |
5b597b021aed
added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
1082
diff
changeset
|
473 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False) |