Mercurial Hosting > traffic-intelligence

annotate scripts/process.py @ 1071:58994b08be42

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added multithreading for safety
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Wed, 18 Jul 2018 02:12:47 -0400
parents 0154133e77df
children c67f8c36ebc7 8ab92ee3cbef
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
998
933670761a57 updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 988
diff changeset
1 #! /usr/bin/env python3
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
2
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
3 import sys, argparse
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
4 from pathlib import Path
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
5 from multiprocessing.pool import Pool
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
6
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
7 #import matplotlib
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
8 #atplotlib.use('Agg')
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
9 import matplotlib.pyplot as plt
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
10 import numpy as np
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
11 import pandas as pd
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
12
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
13 from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml
1028
cc5cb04b04b0 major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1026
diff changeset
14 from trafficintelligence.metadata import *
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
15
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
17 # input
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*')
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
21
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
22 # main function
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
1036
0d7e5e290ea3 upload to pypi
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1028
diff changeset
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction'])
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
26 parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true')
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'event'])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
28
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
29 # common options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
30 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
31 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
1044
75a6ad604cc5 work on motion patterns
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1043
diff changeset
32 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature')
1007
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
33 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
192de96e5255 solved config filename bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1006
diff changeset
34 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
35 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int)
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
36
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
37 ### process options
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
38 # motion pattern learning and assignment
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
39 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes', default = 'prototypes.sqlite')
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
40 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
41 parser.add_argument('--nobjects-mp', dest = 'nMPObjects', help = 'number of objects/interactions to process', type = int)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
42 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
43 parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float)
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
44 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
45 parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float)
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
46 parser.add_argument('--min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
47 #parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
48 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
49 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
50 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
51 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
52 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
53 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
54
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
55 # safety analysis
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
56 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
57 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
58 # override other tracking config, erase sqlite?
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
59
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
60
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
61 # analysis options
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
63 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data')
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
69 dpi = 150
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
70 # unit of analysis: site - camera-view
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
71
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
72 # need way of selecting sites as similar as possible to sql alchemy syntax
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
73 # override tracking.cfg from db
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
74 # manage cfg files, overwrite them (or a subset of parameters)
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
75 # delete sqlite files
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
76 # info of metadata
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
77
983
7463c9bc846b work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff changeset
78 args = parser.parse_args()
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
79
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
80 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
81 # Data preparation
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
82 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
83 session = connectDatabase(args.metadataFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
84 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
85 videoSequences = []
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
86 sites = []
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
87 if args.videoIds is not None:
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
88 for videoId in args.videoIds:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
89 if '-' in videoId:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
90 videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)])
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
91 else:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
92 videoSequences.append(session.query(VideoSequence).get(int(videoId)))
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
93 videoSequences = [vs for vs in videoSequences if vs is not None]
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
94 sites = set([vs.cameraView.site for vs in videoSequences])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
95 elif args.siteIds is not None:
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
96 for siteId in args.siteIds:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
97 if '-' in siteId:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
98 sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)])
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
99 else:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
100 sites.append(session.query(Site).get(int(siteId)))
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
101 sites = [s for s in sites if s is not None]
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
102 for site in sites:
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
103 videoSequences.extend(getSiteVideoSequences(site))
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
104 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
105 print('No video/site to process')
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
106
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
107 if args.nProcesses > 1:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
108 pool = Pool(args.nProcesses)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
109
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
110 #################################
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
111 # Report progress in the processing
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
112 #################################
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
113 if args.progress:
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
114 print('Providing information on data progress')
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
115 headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
116 data = []
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
117 for site in sites:
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
118 unprocessedVideoSequences = []
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
119 for vs in getSiteVideoSequences(site):
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
120 if (parentPath/vs.getDatabaseFilename()).is_file(): # TODO check time of file?
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
121 tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename()))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
122 data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames])
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
123 else:
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
124 unprocessedVideoSequences.append(vs)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
125 data.append([site.name, vs.idx, False, False, False])
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
126 #if len(unprocessedVideoSequences):
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
127 # print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences)))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
128 data = pd.DataFrame(data, columns = headers)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
129 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
130 print('\t'+' '.join(headers[2:]))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
131 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
132 for name, group in data.groupby(['site']): #.agg({'vs': 'count'}))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
133 n = group.vs.count()
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
134 print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n))
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
135 print('-'*80)
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
136 print(data)
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
137
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
138 #################################
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
139 # Delete
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
140 #################################
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
141 if args.delete is not None:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
142 if args.delete == 'feature':
1026
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
143 response = input('Are you sure you want to delete the tracking results (SQLite files) of all these sites (y/n)?')
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
144 if response == 'y':
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
145 for vs in videoSequences:
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
146 p = parentPath.absolute()/vs.getDatabaseFilename()
73b124160911 more plumbing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1023
diff changeset
147 p.unlink()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
148 elif args.delete in ['object', 'interaction']:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
149 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype'])
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
150 for vs in videoSequences:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
151 storage.deleteFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.delete)
984
a69695d14e59 work on script for large datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 983
diff changeset
152
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
153 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
154 # Process
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
155 #################################
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
156 if args.process in ['feature', 'object']: # tracking
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
157 if args.nProcesses == 1:
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
158 for vs in videoSequences:
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
159 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
160 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
161 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
162 else:
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
163 configFilename = args.configFilename
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
164 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
165 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
166 else: #caution: cameratype can be not none, but without parameters for undistortion
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
167 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
168 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
169 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
170 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
171 for vs in videoSequences:
1065
d4d052a05337 added progress report functionality
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1064
diff changeset
172 if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object':
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
173 if args.configFilename is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
174 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
175 else:
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
176 configFilename = args.configFilename
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
177 if vs.cameraView.cameraType is None:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
178 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
179 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
180 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
181 else:
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
182 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
1004
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
183 pool.close()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
184 pool.join()
75601be6019f work on process
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1003
diff changeset
185
1039
5621e4ad2428 removing prefix option to loadtrajectories from SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1036
diff changeset
186 elif args.process == 'prototype': # motion pattern learning
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
187 # learn by site by default -> group videos by camera view TODO
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
188 # by default, load all objects, learn and then assign (BUT not save the assignments)
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
189 for site in sites:
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
190 print('Learning motion patterns for site {} ({})'.format(site.idx, site.name))
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
191 objects = {}
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
192 object2VideoSequences = {}
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
193 for cv in site.cameraViews:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
194 for vs in cv.videoSequences:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
195 print('Loading '+vs.getDatabaseFilename())
1046
f2ba9858e6c6 motion pattern learning seems to work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1045
diff changeset
196 objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
197 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
198 objectsWithFeatures = objects[vs.idx]
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
199 objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()]
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
200 prototypeType = 'feature'
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
201 else:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
202 prototypeType = args.trajectoryType
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
203 for obj in objects[vs.idx]:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
204 object2VideoSequences[obj] = vs
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
205 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
206 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
207 trainingObjects = [o for tmpobjects in objects.values() for o in tmpobjects]
1049
c9c03c97ed9f bug fix to store numpy integers in SQLite
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1048
diff changeset
208 if args.nMPObjects is not None and args.nMPObjects < len(trainingObjects):
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
209 m = int(np.floor(float(len(trainingObjects))/args.nMPObjects))
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
210 trainingObjects = trainingObjects[::m]
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
211 similarities = -np.ones((len(trainingObjects), len(trainingObjects)))
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
212 prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, trainingObjects, similarities, args.minSimilarity, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, [])
1045
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
213 if args.outputPrototypeDatabaseFilename is None:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
214 outputPrototypeDatabaseFilename = args.databaseFilename
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
215 else:
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
216 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
25db2383e7ae work in progress on process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1044
diff changeset
217 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
1048
27a822922cb0 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1047
diff changeset
218 storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices])
1043
b735895c8815 work in progress on process (learn motion patterns)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1039
diff changeset
219
1003
75af46516b2b work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 998
diff changeset
220 elif args.process == 'interaction':
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
221 # safety analysis TODO make function in safety analysis script
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
222 if args.predictionMethod == 'cvd':
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
223 predictionParameters = prediction.CVDirectPredictionParameters()
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
224 elif args.predictionMethod == 'cve':
987
f026ce2af637 found bug with direct ttc computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 986
diff changeset
225 predictionParameters = prediction.CVExactPredictionParameters()
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
226 for vs in videoSequences:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
227 print('Processing '+vs.getDatabaseFilename())
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
228 if args.configFilename is None:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
229 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
230 else:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
231 params = storage.ProcessParameters(args.configFilename)
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
232 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
233 interactions = events.createInteractions(objects)
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
234 if args.nProcesses == 1:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
235 #print(len(interactions), args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, params.crossingZones)
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
236 processed = events.computeIndicators(interactions, True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None) # params.crossingZones
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
237 else:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
238 #pool = Pool(processes = args.nProcesses)
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
239 nInteractionPerProcess = int(np.ceil(len(interactions)/float(args.nProcesses)))
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
240 jobs = [pool.apply_async(events.computeIndicators, args = (interactions[i*nInteractionPerProcess:(i+1)*nInteractionPerProcess], True, args.computePET, predictionParameters, params.collisionDistance, params.predictionTimeHorizon, False, False, None)) for i in range(args.nProcesses)] # params.crossingZones
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
241 processed = []
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
242 for job in jobs:
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
243 processed += job.get()
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
244 #pool.close()
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
245 storage.saveIndicatorsToSqlite(str(parentPath/vs.getDatabaseFilename()), processed)
1071
58994b08be42 added multithreading for safety
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1070
diff changeset
246
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
247 #################################
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
248 # Analyze
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
249 #################################
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
250 if args.analyze == 'object':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
251 # user speeds, accelerations
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
252 # aggregation per site
1068
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
253 if args.eventFilename is None:
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
254 print('Missing output filename (event-filename). Exiting')
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
255 sys.exit(0)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
256 data = [] # list of observation per site-user with time
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
257 headers = ['site', 'date', 'time', 'user_type']
1062
a2e20aba0740 bug correctiong
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1061
diff changeset
258 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
259 headers.extend(tmpheaders)
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
260 if args.nProcesses == 1:
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
261 for vs in videoSequences:
1067
092bd9c7deaf corrected bug with multiprocessing and centiles
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1066
diff changeset
262 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles))
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
263 else:
1067
092bd9c7deaf corrected bug with multiprocessing and centiles
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1066
diff changeset
264 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences]
1066
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
265 for job in jobs:
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
266 data.extend(job.get())
862b55a87e63 work on extracting information
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1065
diff changeset
267 pool.close()
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
268 data = pd.DataFrame(data, columns = headers)
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
269 if args.output == 'figure':
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
270 for name in headers[4:]:
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
271 plt.ioff()
1068
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1067
diff changeset
272 plt.figure()
1064
cbc026dacf0b changed interval string representation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1063
diff changeset
273 plt.boxplot([data.loc[data['site']==site.name, name] for site in sites], labels = [site.name for site in sites])
1023
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
274 plt.ylabel(name+' Speeds (km/h)')
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
275 plt.savefig(name.lower()+'-speeds.png', dpi=dpi)
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
276 plt.close()
a13f47c8931d work on processing large datasets (generate speed data)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1021
diff changeset
277 elif args.output == 'event':
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
278 data.to_csv(args.eventFilename, index = False)
1050
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
279
9d4a06f49cb8 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1049
diff changeset
280 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
281 indicatorIds = [2,5,7,10]
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
282 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
283 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
284 indicators = {}
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
285 interactions = {}
1009
0d29b75f74ea cleaning
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1007
diff changeset
286 for vs in videoSequences:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
287 if not vs.cameraView.siteIdx in interactions:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
288 interactions[vs.cameraView.siteIdx] = []
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
289 indicators[vs.cameraView.siteIdx] = {}
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
290 for i in indicatorIds:
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
291 indicators[vs.cameraView.siteIdx][i] = []
1021
16932cefabc1 work on paths in line with new configurations from tracker
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1009
diff changeset
292 interactions[vs.cameraView.siteIdx] += storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
293 print(vs.getDatabaseFilename(), len(interactions[vs.cameraView.siteIdx]))
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
294 for inter in interactions[vs.cameraView.siteIdx]:
985
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
295 for i in indicatorIds:
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
296 indic = inter.getIndicator(events.Interaction.indicatorNames[i])
668a85c963c3 work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 984
diff changeset
297 if indic is not None:
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
298 v = indic.getMostSevereValue()*conversionFactors[i]
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
299 if v < maxIndicatorValue[i]:
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
300 indicators[vs.cameraView.siteIdx][i].append(v)
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
301
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
302 for i in indicatorIds:
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
303 tmp = [indicators[siteId][i] for siteId in indicators]
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
304 plt.ioff()
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
305 plt.figure()
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
306 plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
988
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
307 plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')')
dc0be55e2bf5 new process functionalities
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 987
diff changeset
308 plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150)
986
3be8aaa47651 work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 985
diff changeset
309 plt.close()
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
310
1062
a2e20aba0740 bug correctiong
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1061
diff changeset
311 if args.analyze == 'event': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
312 data = pd.read_csv(args.eventFilename, parse_dates = [2])
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
313 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000)
1058
16575ca4537d work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1050
diff changeset
314 # create time for end of each 15 min, then group by, using the agg method for each data column
1061
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1059
diff changeset
315 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
316 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
317 dataColumns = list(data.columns[4:])
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
318 for h in dataColumns:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
319 for h2 in tmpheaders:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
320 headers.append(h+'-'+h2)
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
321 for h in dataColumns:
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
322 for t in args.eventThresholds:
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
323 headers.append('n-{}-{}'.format(h, t))
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
324 data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time())
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
325 outputData = []
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
326 for name, group in data.groupby(['site', 'date', 'intervalend15']):
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
327 row = []
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
328 row.extend(name)
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
329 groupStartTime = group.time.min()
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
330 groupEndTime = group.time.max()
1070
0154133e77df corrected bug
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1069
diff changeset
331 row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600)
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
332 row.append(len(group))
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
333 for h in dataColumns:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
334 for method,func in aggFunctions.items():
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
335 aggregated = func(group[h])
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
336 if method == 'centile':
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
337 row.extend(aggregated)
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
338 else:
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
339 row.append(aggregated)
1069
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
340 for h in dataColumns:
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
341 for t in args.eventThresholds:
9ee5c7636640 added severe event counting
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1068
diff changeset
342 row.append((group[h] > t).sum())
1059
a87b3072bd26 working version
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1058
diff changeset
343 outputData.append(row)
1063
3c37d8d20e97 minor addition to process.py
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 1062
diff changeset
344 pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False)