Mercurial Hosting > traffic-intelligence

annotate scripts/process.py @ 1108:77ce1cb3c868

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
uploading minor version 0.2.4
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Fri, 15 Mar 2019 16:10:08 -0400
parents 8734742c08c0
children b968c33f8c2f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
998
933670761a57 updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 988
diff changeset
1 #! /usr/bin/env python3
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
2
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
3 import sys, argparse
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
4 from pathlib import Path
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
5 from multiprocessing.pool import Pool
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
6
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
7 #import matplotlib
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
8 #atplotlib.use('Agg')
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
9 import matplotlib.pyplot as plt
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
10 import numpy as np
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
11 import pandas as pd
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
12
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
13 from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml
1028
cc5cb04b04b0 major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1026
diff changeset
14 from trafficintelligence.metadata import *
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
15
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
17 # input
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*')
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
21
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
22 # main function
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
1036
0d7e5e290ea3 upload to pypi
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1028
diff changeset
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction'])
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
26 parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true')
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'event-speed', 'event-interaction'])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
28
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
29 # common options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
30 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
31 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
1044
75a6ad604cc5 work on motion patterns
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1043
diff changeset
32 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature')
1007
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
33 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
34 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
35 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int)
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
36
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
37 ### process options
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
38 # motion pattern learning and assignment
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
39 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes', default = 'prototypes.sqlite')
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
40 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
41 parser.add_argument('--nobjects-mp', dest = 'nMPObjects', help = 'number of objects/interactions to process', type = int)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
42 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
43 parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float)
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
44 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
45 parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float)
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
46 parser.add_argument('--min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
47 #parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
48 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
49 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
50 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
51 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
52 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
53 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
54
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
55 # safety analysis
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
56 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
57 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
58 # override other tracking config, erase sqlite?
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
59
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
60
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
61 # analysis options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
69 dpi = 150
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
70 # unit of analysis: site - camera-view
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
71
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
72 # need way of selecting sites as similar as possible to sql alchemy syntax
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
73 # override tracking.cfg from db
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
74 # manage cfg files, overwrite them (or a subset of parameters)
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
75 # delete sqlite files
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
76 # info of metadata
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
77
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
78 args = parser.parse_args()
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
79
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
80 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
81 # Data preparation
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
82 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
83 session = connectDatabase(args.metadataFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
84 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
85 videoSequences = []
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
86 sites = []
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
87 if args.videoIds is not None:
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
88 for videoId in args.videoIds:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
89 if '-' in videoId:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
90 videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)])
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
91 else:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
92 videoSequences.append(session.query(VideoSequence).get(int(videoId)))
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
93 videoSequences = [vs for vs in videoSequences if vs is not None]
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
94 sites = set([vs.cameraView.site for vs in videoSequences])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
95 elif args.siteIds is not None:
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
96 for siteId in args.siteIds:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
97 if '-' in siteId:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
98 sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)])
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
99 else:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
100 sites.append(session.query(Site).get(int(siteId)))
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
101 sites = [s for s in sites if s is not None]
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
102 for site in sites:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
103 videoSequences.extend(getSiteVideoSequences(site))
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
104 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
105 print('No video/site to process')
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
106
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
107 if args.nProcesses > 1:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
108 pool = Pool(args.nProcesses)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
109
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
110 #################################
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
111 # Report progress in the processing
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
112 #################################
1073
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
113 if args.progress: # TODO find video sequences that have null camera view, to work with them
1082
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
114 print('Providing information on progress of data processing')
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
115 headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
116 data = []
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
117 for site in sites:
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
118 unprocessedVideoSequences = []
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
119 for vs in getSiteVideoSequences(site):
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
120 if (parentPath/vs.getDatabaseFilename()).is_file(): # TODO check time of file?
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
121 tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename()))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
122 data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames])
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
123 else:
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
124 unprocessedVideoSequences.append(vs)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
125 data.append([site.name, vs.idx, False, False, False])
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
126 #if len(unprocessedVideoSequences):
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
127 # print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences)))
1082
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
128 videoSequences = session.query(VideoSequence).filter(VideoSequence.cameraViewIdx.is_(None)).all()
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
129 data = pd.DataFrame(data, columns = headers)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
130 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
131 print('\t'+' '.join(headers[2:]))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
132 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
133 for name, group in data.groupby(['site']): #.agg({'vs': 'count'}))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
134 n = group.vs.count()
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
135 print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
136 print('-'*80)
1082
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
137 if len(videoSequences) > 0:
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
138 print('{} video sequences without a camera view:'.format(len(videoSequences)))
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
139 print([vs.idx for vs in videoSequences])
706034a4c6cd more info in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1074
diff changeset
140 print('-'*80)
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
141 print(data)
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
142
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
143 #################################
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
144 # Delete
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
145 #################################
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
146 if args.delete is not None:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
147 if args.delete == 'feature':
1026
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
148 response = input('Are you sure you want to delete the tracking results (SQLite files) of all these sites (y/n)?')
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
149 if response == 'y':
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
150 for vs in videoSequences:
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
151 p = parentPath.absolute()/vs.getDatabaseFilename()
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
152 p.unlink()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
153 elif args.delete in ['object', 'interaction']:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
154 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype'])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
155 for vs in videoSequences:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
156 storage.deleteFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.delete)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
157
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
158 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
159 # Process
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
160 #################################
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
161 if args.process in ['feature', 'object']: # tracking
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
162 if args.nProcesses == 1:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
163 for vs in videoSequences:
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
164 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
165 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
166 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
167 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
168 configFilename = args.configFilename
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
169 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
170 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
171 else: #caution: cameratype can be not none, but without parameters for undistortion
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
172 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
173 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
174 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
175 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
176 for vs in videoSequences:
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
177 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
178 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
179 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
180 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
181 configFilename = args.configFilename
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
182 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
183 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
184 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
185 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
186 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
187 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
188 pool.close()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
189 pool.join()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
190
1039
5621e4ad2428 removing prefix option to loadtrajectories from SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1036
diff changeset
191 elif args.process == 'prototype': # motion pattern learning
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
192 # learn by site by default -> group videos by camera view TODO
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
193 # by default, load all objects, learn and then assign (BUT not save the assignments)
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
194 for site in sites:
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
195 print('Learning motion patterns for site {} ({})'.format(site.idx, site.name))
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
196 objects = {}
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
197 object2VideoSequences = {}
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
198 for cv in site.cameraViews:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
199 for vs in cv.videoSequences:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
200 print('Loading '+vs.getDatabaseFilename())
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
201 objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
202 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
203 objectsWithFeatures = objects[vs.idx]
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
204 objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()]
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
205 prototypeType = 'feature'
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
206 else:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
207 prototypeType = args.trajectoryType
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
208 for obj in objects[vs.idx]:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
209 object2VideoSequences[obj] = vs
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
210 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
211 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
212 trainingObjects = [o for tmpobjects in objects.values() for o in tmpobjects]
1049
c9c03c97ed9f bug fix to store numpy integers in SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1048
diff changeset
213 if args.nMPObjects is not None and args.nMPObjects < len(trainingObjects):
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
214 m = int(np.floor(float(len(trainingObjects))/args.nMPObjects))
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
215 trainingObjects = trainingObjects[::m]
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
216 similarities = -np.ones((len(trainingObjects), len(trainingObjects)))
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
217 prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, trainingObjects, similarities, args.minSimilarity, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, [])
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
218 if args.outputPrototypeDatabaseFilename is None:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
219 outputPrototypeDatabaseFilename = args.databaseFilename
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
220 else:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
221 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
222 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
223 storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices])
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
224
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
225 elif args.process == 'interaction':
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
226 # safety analysis TODO make function in safety analysis script
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
227 if args.predictionMethod == 'cvd':
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
228 predictionParameters = prediction.CVDirectPredictionParameters()
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
229 elif args.predictionMethod == 'cve':
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
230 predictionParameters = prediction.CVExactPredictionParameters()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
231 for vs in videoSequences:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
232 print('Processing '+vs.getDatabaseFilename())
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
233 if args.configFilename is None:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
234 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
235 else:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
236 params = storage.ProcessParameters(args.configFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
237 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
238 interactions = events.createInteractions(objects)
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
239 if args.nProcesses == 1:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
240 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones)
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
241 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None) # params.crossingZones
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
242 else:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
243 #pool = Pool(processes = args.nProcesses)
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
244 nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses)))
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
245 jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None)) for i in range(args.nProcesses)] # params.crossingZones
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
246 processed = []
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
247 for job in jobs:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
248 processed += job.get()
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
249 #pool.close()
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
250 storage.saveIndicatorsToSqlite(str(parentPath/vs.getDatabaseFilename()), processed)
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
251
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
252 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
253 # Analyze
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
254 #################################
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
255 if args.analyze == 'object':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
256 # user speeds, accelerations
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
257 # aggregation per site
1068
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
258 if args.eventFilename is None:
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
259 print('Missing output filename (event-filename). Exiting')
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
260 sys.exit(0)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
261 data = [] # list of observation per site-user with time
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
262 headers = ['site', 'date', 'time', 'user_type']
1062
a2e20aba0740 bug correctiong
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1061
diff changeset
263 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
264 headers.extend(tmpheaders)
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
265 if args.nProcesses == 1:
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
266 for vs in videoSequences:
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
267 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles))
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
268 else:
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
269 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences]
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
270 for job in jobs:
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
271 data.extend(job.get())
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
272 pool.close()
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
273 data = pd.DataFrame(data, columns = headers)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
274 if args.output == 'figure':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
275 for name in headers[4:]:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
276 plt.ioff()
1068
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
277 plt.figure()
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
278 plt.boxplot([data.loc[data['site']==site.name, name] for site in sites], labels = [site.name for site in sites])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
279 plt.ylabel(name+' Speeds (km/h)')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
280 plt.savefig(name.lower()+'-speeds.png', dpi=dpi)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
281 plt.close()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
282 elif args.output == 'event':
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
283 data.to_csv(args.eventFilename, index = False)
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
284
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
285 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
286 indicatorIds = [2,5,7,10]
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
287 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
288 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
289 data = [] # list of observation per site-user with time
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
290 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types?
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
291 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
292 for i in indicatorIds[:3]:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
293 for h in tmpheaders:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
294 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h)
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
295 indicators = {}
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
296 interactions = {}
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
297 for vs in videoSequences:
1072
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
298 print('Extracting SMoS from '+vs.getDatabaseFilename())
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
299 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
300 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
301 for inter in interactions:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
302 if inter.length() > minDuration:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
303 d = vs.startTime.date()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
304 t = vs.startTime.time()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
305 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)]
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
306 pet = inter.getIndicator('Post Encroachment Time')
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
307 if pet is None:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
308 row.append(None)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
309 else:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
310 row.append(conversionFactors[10]*pet.getValues()[0])
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
311 for i in indicatorIds[:3]:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
312 indic = inter.getIndicator(events.Interaction.indicatorNames[i])
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
313 if indic is not None:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
314 #v = indic.getMostSevereValue()*
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
315 tmp = list(indic.values.values())
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
316 for method,func in aggFunctions.items():
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
317 agg = conversionFactors[i]*func(tmp)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
318 if method == 'centile':
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
319 row.extend(agg.tolist())
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
320 else:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
321 row.append(agg)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
322 else:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
323 row.extend([None]*len(aggFunctions))
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
324 data.append(row)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
325 data = pd.DataFrame(data, columns = headers)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
326 if args.output == 'figure':
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
327 for i in indicatorIds:
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
328 pass # tmp = [indicators[siteId][i] for siteId in indicators]
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
329 # plt.ioff()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
330 # plt.figure()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
331 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
332 # plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')')
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
333 # plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150)
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
334 # plt.close()
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
335 elif args.output == 'event':
c67f8c36ebc7 interaction extraction
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1071
diff changeset
336 data.to_csv(args.eventFilename, index = False)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
337
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
338 if args.analyze == 'event-speed': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
1084
1a7e0b2c858b remove debugging
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1083
diff changeset
339 data = pd.read_csv(args.eventFilename, parse_dates = [2])
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
340 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
341 # create time for end of each 15 min, then group by, using the agg method for each data column
1061
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1059
diff changeset
342 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
343 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
344 dataColumns = list(data.columns[4:])
1086
8734742c08c0 major refactoring of curvilinear trajectory projections
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1084
diff changeset
345 print(dataColumns)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
346 for h in dataColumns:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
347 for h2 in tmpheaders:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
348 headers.append(h+'-'+h2)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
349 if args.eventThresholds is not None:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
350 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
351 for t in args.eventThresholds:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
352 headers.append('n-{}-{}'.format(h, t))
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
353 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time())
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
354 outputData = []
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
355 for name, group in data.groupby(['site', 'date', 'intervalend15']):
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
356 row = []
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
357 row.extend(name)
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
358 groupStartTime = group.time.min()
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
359 groupEndTime = group.time.max()
1070
0154133e77df corrected bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1069
diff changeset
360 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
361 row.append(len(group))
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
362 for h in dataColumns:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
363 for method,func in aggFunctions.items():
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
364 aggregated = func(group[h])
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
365 if method == 'centile':
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
366 row.extend(aggregated)
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
367 else:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
368 row.append(aggregated)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
369 if args.eventThresholds is not None:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
370 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
371 for t in args.eventThresholds:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
372 row.append((group[h] > t).sum())
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
373 outputData.append(row)
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
374 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False)
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
375
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
376 elif args.analyze == 'event-interaction': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
1084
1a7e0b2c858b remove debugging
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1083
diff changeset
377 data = pd.read_csv(args.eventFilename, parse_dates = [2])
1083
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
378 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
379 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
380 dataColumns = list(data.columns[3:])
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
381 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
382 if not 'speed' in h.lower(): # proximity indicators are reversed, taking 85th centile of this column will yield the 15th centile (which we have to take the opposite again)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
383 data[h] = -data[h]
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
384 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
385 for h2 in tmpheaders:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
386 headers.append(h+'-'+h2)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
387 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
388 headers.append('n-{}-{}'.format(h, t))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
389 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time())
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
390 outputData = []
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
391 for name, group in data.groupby(['site', 'date', 'intervalend15']):
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
392 row = []
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
393 row.extend(name)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
394 groupStartTime = group.time.min()
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
395 groupEndTime = group.time.max()
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
396 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
397 row.append(len(group))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
398 for h in dataColumns:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
399 for method,func in aggFunctions.items():
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
400 tmp = group.loc[~group[h].isna(), h]
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
401 if len(tmp)>0:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
402 aggregated = func(tmp) # todo invert if the resulting stat is negative
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
403 if method == 'centile':
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
404 row.extend(np.abs(aggregated))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
405 else:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
406 row.append(np.abs(aggregated))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
407 else:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
408 row.extend([None]*len(aggFunctions))
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
409 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
410 if 'speed' in h.lower():
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
411 row.append((group[h] > t).sum())
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
412 else:
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
413 row.append((group[h] > -t).sum()) # take larger than than negative threshold for proximity indicators
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
414 outputData.append(row)
5b597b021aed added function to aggregate interactions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1082
diff changeset
415 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False)