Mercurial Hosting > traffic-intelligence
comparison trafficintelligence/storage.py @ 1028:cc5cb04b04b0
major update using the trafficintelligence package name and install through pip
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Fri, 15 Jun 2018 11:19:10 -0400 |
parents | python/storage.py@73b124160911 |
children | aafbc0bab925 |
comparison
equal
deleted
inserted
replaced
1027:6129296848d3 | 1028:cc5cb04b04b0 |
---|---|
1 #! /usr/bin/env python | |
2 # -*- coding: utf-8 -*- | |
3 '''Various utilities to save and load data''' | |
4 | |
5 from trafficintelligence import utils, moving, events, indicators | |
6 from trafficintelligence.base import VideoFilenameAddable | |
7 | |
8 from pathlib import Path | |
9 import shutil | |
10 from copy import copy | |
11 import sqlite3, logging | |
12 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg | |
13 from pandas import read_csv, merge | |
14 | |
15 | |
16 commentChar = '#' | |
17 | |
18 delimiterChar = '%'; | |
19 | |
20 ngsimUserTypes = {'twowheels':1, | |
21 'car':2, | |
22 'truck':3} | |
23 | |
24 tableNames = {'feature':'positions', | |
25 'object': 'objects', | |
26 'objectfeatures': 'positions'} | |
27 | |
28 ######################### | |
29 # Sqlite | |
30 ######################### | |
31 | |
32 # utils | |
33 def printDBError(error): | |
34 print('DB Error: {}'.format(error)) | |
35 | |
36 def dropTables(connection, tableNames): | |
37 'deletes the table with names in tableNames' | |
38 try: | |
39 cursor = connection.cursor() | |
40 for tableName in tableNames: | |
41 cursor.execute('DROP TABLE IF EXISTS '+tableName) | |
42 except sqlite3.OperationalError as error: | |
43 printDBError(error) | |
44 | |
45 def deleteFromSqlite(filename, dataType): | |
46 'Deletes (drops) some tables in the filename depending on type of data' | |
47 if Path(filename).is_file(): | |
48 with sqlite3.connect(filename) as connection: | |
49 if dataType == 'object': | |
50 dropTables(connection, ['objects', 'objects_features']) | |
51 elif dataType == 'interaction': | |
52 dropTables(connection, ['interactions', 'indicators']) | |
53 elif dataType == 'bb': | |
54 dropTables(connection, ['bounding_boxes']) | |
55 elif dataType == 'pois': | |
56 dropTables(connection, ['gaussians2d', 'objects_pois']) | |
57 elif dataType == 'prototype': | |
58 dropTables(connection, ['prototypes', 'objects_prototypes']) | |
59 else: | |
60 print('Unknown data type {} to delete from database'.format(dataType)) | |
61 else: | |
62 print('{} does not exist'.format(filename)) | |
63 | |
64 def tableExists(connection, tableName): | |
65 'indicates if the table exists in the database' | |
66 try: | |
67 cursor = connection.cursor() | |
68 cursor.execute('SELECT COUNT(*) FROM SQLITE_MASTER WHERE type = \'table\' AND name = \''+tableName+'\'') | |
69 return cursor.fetchone()[0] == 1 | |
70 except sqlite3.OperationalError as error: | |
71 printDBError(error) | |
72 | |
73 def createTrajectoryTable(cursor, tableName): | |
74 if tableName.endswith('positions') or tableName.endswith('velocities'): | |
75 cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (trajectory_id INTEGER, frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") | |
76 else: | |
77 print('Unallowed name {} for trajectory table'.format(tableName)) | |
78 | |
79 def createObjectsTable(cursor): | |
80 cursor.execute("CREATE TABLE IF NOT EXISTS objects (object_id INTEGER, road_user_type INTEGER, n_objects INTEGER, PRIMARY KEY(object_id))") | |
81 | |
82 def createAssignmentTable(cursor, objectType1, objectType2, objectIdColumnName1, objectIdColumnName2): | |
83 cursor.execute("CREATE TABLE IF NOT EXISTS "+objectType1+"s_"+objectType2+"s ("+objectIdColumnName1+" INTEGER, "+objectIdColumnName2+" INTEGER, PRIMARY KEY("+objectIdColumnName1+","+objectIdColumnName2+"))") | |
84 | |
85 def createObjectsFeaturesTable(cursor): | |
86 cursor.execute("CREATE TABLE IF NOT EXISTS objects_features (object_id INTEGER, trajectory_id INTEGER, PRIMARY KEY(object_id, trajectory_id))") | |
87 | |
88 | |
89 def createCurvilinearTrajectoryTable(cursor): | |
90 cursor.execute("CREATE TABLE IF NOT EXISTS curvilinear_positions (trajectory_id INTEGER, frame_number INTEGER, s_coordinate REAL, y_coordinate REAL, lane TEXT, PRIMARY KEY(trajectory_id, frame_number))") | |
91 | |
92 def createFeatureCorrespondenceTable(cursor): | |
93 cursor.execute('CREATE TABLE IF NOT EXISTS feature_correspondences (trajectory_id INTEGER, source_dbname VARCHAR, db_trajectory_id INTEGER, PRIMARY KEY(trajectory_id))') | |
94 | |
95 def createInteractionTable(cursor): | |
96 cursor.execute('CREATE TABLE IF NOT EXISTS interactions (id INTEGER PRIMARY KEY, object_id1 INTEGER, object_id2 INTEGER, first_frame_number INTEGER, last_frame_number INTEGER, FOREIGN KEY(object_id1) REFERENCES objects(id), FOREIGN KEY(object_id2) REFERENCES objects(id))') | |
97 | |
98 def createIndicatorTable(cursor): | |
99 cursor.execute('CREATE TABLE IF NOT EXISTS indicators (interaction_id INTEGER, indicator_type INTEGER, frame_number INTEGER, value REAL, FOREIGN KEY(interaction_id) REFERENCES interactions(id), PRIMARY KEY(interaction_id, indicator_type, frame_number))') | |
100 | |
101 def insertTrajectoryQuery(tableName): | |
102 return "INSERT INTO "+tableName+" VALUES (?,?,?,?)" | |
103 | |
104 def insertObjectQuery(): | |
105 return "INSERT INTO objects VALUES (?,?,?)" | |
106 | |
107 def insertObjectFeatureQuery(): | |
108 return "INSERT INTO objects_features VALUES (?,?)" | |
109 | |
110 def createIndex(connection, tableName, columnName, unique = False): | |
111 '''Creates an index for the column in the table | |
112 I will make querying with a condition on this column faster''' | |
113 try: | |
114 cursor = connection.cursor() | |
115 s = "CREATE " | |
116 if unique: | |
117 s += "UNIQUE " | |
118 cursor.execute(s+"INDEX IF NOT EXISTS "+tableName+"_"+columnName+"_index ON "+tableName+"("+columnName+")") | |
119 connection.commit() | |
120 #connection.close() | |
121 except sqlite3.OperationalError as error: | |
122 printDBError(error) | |
123 | |
124 def getNumberRowsTable(connection, tableName, columnName = None): | |
125 '''Returns the number of rows for the table | |
126 If columnName is not None, means we want the number of distinct values for that column | |
127 (otherwise, we can just count(*))''' | |
128 try: | |
129 cursor = connection.cursor() | |
130 if columnName is None: | |
131 cursor.execute("SELECT COUNT(*) from "+tableName) | |
132 else: | |
133 cursor.execute("SELECT COUNT(DISTINCT "+columnName+") from "+tableName) | |
134 return cursor.fetchone()[0] | |
135 except sqlite3.OperationalError as error: | |
136 printDBError(error) | |
137 | |
138 def getMinMax(connection, tableName, columnName, minmax): | |
139 '''Returns max/min or both for given column in table | |
140 minmax must be string max, min or minmax''' | |
141 try: | |
142 cursor = connection.cursor() | |
143 if minmax == 'min' or minmax == 'max': | |
144 cursor.execute("SELECT "+minmax+"("+columnName+") from "+tableName) | |
145 elif minmax == 'minmax': | |
146 cursor.execute("SELECT MIN("+columnName+"), MAX("+columnName+") from "+tableName) | |
147 else: | |
148 print("Argument minmax unknown: {}".format(minmax)) | |
149 return cursor.fetchone()[0] | |
150 except sqlite3.OperationalError as error: | |
151 printDBError(error) | |
152 | |
153 def getObjectCriteria(objectNumbers): | |
154 if objectNumbers is None: | |
155 query = '' | |
156 elif type(objectNumbers) == int: | |
157 query = '<= {0}'.format(objectNumbers-1) | |
158 elif type(objectNumbers) == list: | |
159 query = 'in ('+', '.join([str(n) for n in objectNumbers])+')' | |
160 else: | |
161 print('objectNumbers {} are not a known type ({})'.format(objectNumbers, type(objectNumbers))) | |
162 query = '' | |
163 return query | |
164 | |
165 def loadTrajectoriesFromTable(connection, tableName, trajectoryType, objectNumbers = None, timeStep = None): | |
166 '''Loads trajectories (in the general sense) from the given table | |
167 can be positions or velocities | |
168 | |
169 returns a moving object''' | |
170 cursor = connection.cursor() | |
171 | |
172 try: | |
173 objectCriteria = getObjectCriteria(objectNumbers) | |
174 queryStatement = None | |
175 if trajectoryType == 'feature': | |
176 queryStatement = 'SELECT * from '+tableName | |
177 if objectNumbers is not None and timeStep is not None: | |
178 queryStatement += ' WHERE trajectory_id '+objectCriteria+' AND frame_number%{} = 0'.format(timeStep) | |
179 elif objectNumbers is not None: | |
180 queryStatement += ' WHERE trajectory_id '+objectCriteria | |
181 elif timeStep is not None: | |
182 queryStatement += ' WHERE frame_number%{} = 0'.format(timeStep) | |
183 queryStatement += ' ORDER BY trajectory_id, frame_number' | |
184 elif trajectoryType == 'object': | |
185 queryStatement = 'SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from '+tableName+' P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id' | |
186 if objectNumbers is not None: | |
187 queryStatement += ' AND OF.object_id '+objectCriteria | |
188 if timeStep is not None: | |
189 queryStatement += ' AND P.frame_number%{} = 0'.format(timeStep) | |
190 queryStatement += ' GROUP BY OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number' | |
191 elif trajectoryType in ['bbtop', 'bbbottom']: | |
192 if trajectoryType == 'bbtop': | |
193 corner = 'top_left' | |
194 elif trajectoryType == 'bbbottom': | |
195 corner = 'bottom_right' | |
196 queryStatement = 'SELECT object_id, frame_number, x_'+corner+', y_'+corner+' FROM '+tableName | |
197 if objectNumbers is not None and timeStep is not None: | |
198 queryStatement += ' WHERE object_id '+objectCriteria+' AND frame_number%{} = 0'.format(timeStep) | |
199 elif objectNumbers is not None: | |
200 queryStatement += ' WHERE object_id '+objectCriteria | |
201 elif timeStep is not None: | |
202 queryStatement += ' WHERE frame_number%{} = 0'.format(timeStep) | |
203 queryStatement += ' ORDER BY object_id, frame_number' | |
204 else: | |
205 print('Unknown trajectory type {}'.format(trajectoryType)) | |
206 if queryStatement is not None: | |
207 cursor.execute(queryStatement) | |
208 logging.debug(queryStatement) | |
209 except sqlite3.OperationalError as error: | |
210 printDBError(error) | |
211 return [] | |
212 | |
213 objId = -1 | |
214 obj = None | |
215 objects = [] | |
216 for row in cursor: | |
217 if row[0] != objId: | |
218 objId = row[0] | |
219 if obj is not None and (obj.length() == obj.positions.length() or (timeStep is not None and npceil(obj.length()/timeStep) == obj.positions.length())): | |
220 objects.append(obj) | |
221 elif obj is not None: | |
222 print('Object {} is missing {} positions'.format(obj.getNum(), int(obj.length())-obj.positions.length())) | |
223 obj = moving.MovingObject(row[0], timeInterval = moving.TimeInterval(row[1], row[1]), positions = moving.Trajectory([[row[2]],[row[3]]])) | |
224 else: | |
225 obj.timeInterval.last = row[1] | |
226 obj.positions.addPositionXY(row[2],row[3]) | |
227 | |
228 if obj is not None and (obj.length() == obj.positions.length() or (timeStep is not None and npceil(obj.length()/timeStep) == obj.positions.length())): | |
229 objects.append(obj) | |
230 elif obj is not None: | |
231 print('Object {} is missing {} positions'.format(obj.getNum(), int(obj.length())-obj.positions.length())) | |
232 | |
233 return objects | |
234 | |
235 def loadUserTypesFromTable(cursor, objectNumbers): | |
236 objectCriteria = getObjectCriteria(objectNumbers) | |
237 queryStatement = 'SELECT object_id, road_user_type FROM objects' | |
238 if objectNumbers is not None: | |
239 queryStatement += ' WHERE object_id '+objectCriteria | |
240 cursor.execute(queryStatement) | |
241 userTypes = {} | |
242 for row in cursor: | |
243 userTypes[row[0]] = row[1] | |
244 return userTypes | |
245 | |
246 def loadTrajectoriesFromSqlite(filename, trajectoryType, objectNumbers = None, withFeatures = False, timeStep = None, tablePrefix = None): | |
247 '''Loads the trajectories (in the general sense, | |
248 either features, objects (feature groups) or bounding box series) | |
249 The number loaded is either the first objectNumbers objects, | |
250 or the indices in objectNumbers from the database''' | |
251 objects = [] | |
252 with sqlite3.connect(filename) as connection: | |
253 if tablePrefix is None: | |
254 prefix = '' | |
255 else: | |
256 prefix = tablePrefix + '_' | |
257 objects = loadTrajectoriesFromTable(connection, prefix+'positions', trajectoryType, objectNumbers, timeStep) | |
258 objectVelocities = loadTrajectoriesFromTable(connection, prefix+'velocities', trajectoryType, objectNumbers, timeStep) | |
259 | |
260 if len(objectVelocities) > 0: | |
261 for o,v in zip(objects, objectVelocities): | |
262 if o.getNum() == v.getNum(): | |
263 o.velocities = v.positions | |
264 o.velocities.duplicateLastPosition() # avoid having velocity shorter by one position than positions | |
265 else: | |
266 print('Could not match positions {0} with velocities {1}'.format(o.getNum(), v.getNum())) | |
267 | |
268 if trajectoryType == 'object': | |
269 cursor = connection.cursor() | |
270 try: | |
271 # attribute feature numbers to objects | |
272 queryStatement = 'SELECT trajectory_id, object_id FROM objects_features' | |
273 if objectNumbers is not None: | |
274 queryStatement += ' WHERE object_id '+getObjectCriteria(objectNumbers) | |
275 queryStatement += ' ORDER BY object_id' # order is important to group all features per object | |
276 logging.debug(queryStatement) | |
277 cursor.execute(queryStatement) | |
278 | |
279 featureNumbers = {} | |
280 for row in cursor: | |
281 objId = row[1] | |
282 if objId not in featureNumbers: | |
283 featureNumbers[objId] = [row[0]] | |
284 else: | |
285 featureNumbers[objId].append(row[0]) | |
286 | |
287 for obj in objects: | |
288 obj.featureNumbers = featureNumbers[obj.getNum()] | |
289 | |
290 # load userType | |
291 userTypes = loadUserTypesFromTable(cursor, objectNumbers) | |
292 for obj in objects: | |
293 obj.userType = userTypes[obj.getNum()] | |
294 | |
295 if withFeatures: | |
296 nFeatures = 0 | |
297 for obj in objects: | |
298 nFeatures = max(nFeatures, max(obj.featureNumbers)) | |
299 features = loadTrajectoriesFromSqlite(filename, 'feature', nFeatures+1, timeStep = timeStep) | |
300 for obj in objects: | |
301 obj.setFeatures(features) | |
302 | |
303 except sqlite3.OperationalError as error: | |
304 printDBError(error) | |
305 return objects | |
306 | |
307 def loadObjectFeatureFrameNumbers(filename, objectNumbers = None): | |
308 'Loads the feature frame numbers for each object' | |
309 with sqlite3.connect(filename) as connection: | |
310 cursor = connection.cursor() | |
311 try: | |
312 queryStatement = 'SELECT OF.object_id, TL.trajectory_id, TL.length FROM (SELECT trajectory_id, max(frame_number)-min(frame_number) AS length FROM positions GROUP BY trajectory_id) TL, objects_features OF WHERE TL.trajectory_id = OF.trajectory_id' | |
313 if objectNumbers is not None: | |
314 queryStatement += ' AND object_id '+getObjectCriteria(objectNumbers) | |
315 queryStatement += ' ORDER BY OF.object_id, TL.length DESC' | |
316 logging.debug(queryStatement) | |
317 cursor.execute(queryStatement) | |
318 objectFeatureNumbers = {} | |
319 for row in cursor: | |
320 objId = row[0] | |
321 if objId in objectFeatureNumbers: | |
322 objectFeatureNumbers[objId].append(row[1]) | |
323 else: | |
324 objectFeatureNumbers[objId] = [row[1]] | |
325 return objectFeatureNumbers | |
326 except sqlite3.OperationalError as error: | |
327 printDBError(error) | |
328 return None | |
329 | |
330 def addCurvilinearTrajectoriesFromSqlite(filename, objects): | |
331 '''Adds curvilinear positions (s_coordinate, y_coordinate, lane) | |
332 from a database to an existing MovingObject dict (indexed by each objects's num)''' | |
333 with sqlite3.connect(filename) as connection: | |
334 cursor = connection.cursor() | |
335 | |
336 try: | |
337 cursor.execute('SELECT * from curvilinear_positions order by trajectory_id, frame_number') | |
338 except sqlite3.OperationalError as error: | |
339 printDBError(error) | |
340 return [] | |
341 | |
342 missingObjectNumbers = [] | |
343 objNum = None | |
344 for row in cursor: | |
345 if objNum != row[0]: | |
346 objNum = row[0] | |
347 if objNum in objects: | |
348 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() | |
349 else: | |
350 missingObjectNumbers.append(objNum) | |
351 if objNum in objects: | |
352 objects[objNum].curvilinearPositions.addPositionSYL(row[2],row[3],row[4]) | |
353 if len(missingObjectNumbers) > 0: | |
354 print('List of missing objects to attach corresponding curvilinear trajectories: {}'.format(missingObjectNumbers)) | |
355 | |
356 def saveTrajectoriesToTable(connection, objects, trajectoryType, tablePrefix = None): | |
357 'Saves trajectories in table tableName' | |
358 cursor = connection.cursor() | |
359 # Parse feature and/or object structure and commit to DB | |
360 if(trajectoryType == 'feature' or trajectoryType == 'object'): | |
361 # Extract features from objects | |
362 if trajectoryType == 'object': | |
363 features = [] | |
364 for obj in objects: | |
365 if obj.hasFeatures(): | |
366 features += obj.getFeatures() | |
367 if len(features) == 0: | |
368 print('Warning, objects have no features') # todo save centroid trajectories? | |
369 elif trajectoryType == 'feature': | |
370 features = objects | |
371 # Setup feature queries | |
372 if tablePrefix is None: | |
373 prefix = '' | |
374 else: | |
375 prefix = tablePrefix+'_' | |
376 createTrajectoryTable(cursor, prefix+"positions") | |
377 createTrajectoryTable(cursor, prefix+"velocities") | |
378 positionQuery = insertTrajectoryQuery(prefix+"positions") | |
379 velocityQuery = insertTrajectoryQuery(prefix+"velocities") | |
380 # Setup object queries | |
381 if trajectoryType == 'object': | |
382 createObjectsTable(cursor) | |
383 createObjectsFeaturesTable(cursor) | |
384 objectQuery = insertObjectQuery() | |
385 objectFeatureQuery = insertObjectFeatureQuery() | |
386 for feature in features: | |
387 num = feature.getNum() | |
388 frameNum = feature.getFirstInstant() | |
389 for p in feature.getPositions(): | |
390 cursor.execute(positionQuery, (num, frameNum, p.x, p.y)) | |
391 frameNum += 1 | |
392 velocities = feature.getVelocities() | |
393 if velocities is not None: | |
394 frameNum = feature.getFirstInstant() | |
395 for v in velocities[:-1]: | |
396 cursor.execute(velocityQuery, (num, frameNum, v.x, v.y)) | |
397 frameNum += 1 | |
398 if trajectoryType == 'object': | |
399 for obj in objects: | |
400 if obj.hasFeatures(): | |
401 for feature in obj.getFeatures(): | |
402 featureNum = feature.getNum() | |
403 cursor.execute(objectFeatureQuery, (obj.getNum(), featureNum)) | |
404 cursor.execute(objectQuery, (obj.getNum(), obj.getUserType(), 1)) | |
405 # Parse curvilinear position structure | |
406 elif(trajectoryType == 'curvilinear'): | |
407 createCurvilinearTrajectoryTable(cursor) | |
408 curvilinearQuery = "INSERT INTO curvilinear_positions VALUES (?,?,?,?,?)" | |
409 for obj in objects: | |
410 num = obj.getNum() | |
411 frameNum = obj.getFirstInstant() | |
412 for p in obj.getCurvilinearPositions(): | |
413 cursor.execute(curvilinearQuery, (num, frameNum, p[0], p[1], p[2])) | |
414 frameNum += 1 | |
415 else: | |
416 print('Unknown trajectory type {}'.format(trajectoryType)) | |
417 connection.commit() | |
418 | |
419 def saveTrajectoriesToSqlite(outputFilename, objects, trajectoryType): | |
420 '''Writes features, ie the trajectory positions (and velocities if exist) | |
421 with their instants to a specified sqlite file | |
422 Either feature positions (and velocities if they exist) | |
423 or curvilinear positions will be saved at a time''' | |
424 | |
425 with sqlite3.connect(outputFilename) as connection: | |
426 try: | |
427 saveTrajectoriesToTable(connection, objects, trajectoryType, None) | |
428 except sqlite3.OperationalError as error: | |
429 printDBError(error) | |
430 | |
431 def setRoadUserTypes(filename, objects): | |
432 '''Saves the user types of the objects in the sqlite database stored in filename | |
433 The objects should exist in the objects table''' | |
434 with sqlite3.connect(filename) as connection: | |
435 cursor = connection.cursor() | |
436 for obj in objects: | |
437 cursor.execute('update objects set road_user_type = {} WHERE object_id = {}'.format(obj.getUserType(), obj.getNum())) | |
438 connection.commit() | |
439 | |
440 def loadBBMovingObjectsFromSqlite(filename, objectType = 'bb', objectNumbers = None, timeStep = None): | |
441 '''Loads bounding box moving object from an SQLite | |
442 (format of SQLite output by the ground truth annotation tool | |
443 or Urban Tracker | |
444 | |
445 Load descriptions?''' | |
446 objects = [] | |
447 with sqlite3.connect(filename) as connection: | |
448 if objectType == 'bb': | |
449 topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep) | |
450 bottomCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbbottom', objectNumbers, timeStep) | |
451 userTypes = loadUserTypesFromTable(connection.cursor(), objectNumbers) # string format is same as object | |
452 | |
453 for t, b in zip(topCorners, bottomCorners): | |
454 num = t.getNum() | |
455 if t.getNum() == b.getNum(): | |
456 annotation = moving.BBMovingObject(num, t.getTimeInterval(), t, b, userTypes[num]) | |
457 objects.append(annotation) | |
458 else: | |
459 print ('Unknown type of bounding box {}'.format(objectType)) | |
460 return objects | |
461 | |
462 def saveInteraction(cursor, interaction): | |
463 roadUserNumbers = list(interaction.getRoadUserNumbers()) | |
464 cursor.execute('INSERT INTO interactions VALUES({}, {}, {}, {}, {})'.format(interaction.getNum(), roadUserNumbers[0], roadUserNumbers[1], interaction.getFirstInstant(), interaction.getLastInstant())) | |
465 | |
466 def saveInteractionsToSqlite(filename, interactions): | |
467 'Saves the interactions in the table' | |
468 with sqlite3.connect(filename) as connection: | |
469 cursor = connection.cursor() | |
470 try: | |
471 createInteractionTable(cursor) | |
472 for inter in interactions: | |
473 saveInteraction(cursor, inter) | |
474 except sqlite3.OperationalError as error: | |
475 printDBError(error) | |
476 connection.commit() | |
477 | |
478 def saveIndicator(cursor, interactionNum, indicator): | |
479 for instant in indicator.getTimeInterval(): | |
480 if indicator[instant]: | |
481 cursor.execute('INSERT INTO indicators VALUES({}, {}, {}, {})'.format(interactionNum, events.Interaction.indicatorNameToIndices[indicator.getName()], instant, indicator[instant])) | |
482 | |
483 def saveIndicatorsToSqlite(filename, interactions, indicatorNames = events.Interaction.indicatorNames): | |
484 'Saves the indicator values in the table' | |
485 with sqlite3.connect(filename) as connection: | |
486 cursor = connection.cursor() | |
487 try: | |
488 createInteractionTable(cursor) | |
489 createIndicatorTable(cursor) | |
490 for inter in interactions: | |
491 saveInteraction(cursor, inter) | |
492 for indicatorName in indicatorNames: | |
493 indicator = inter.getIndicator(indicatorName) | |
494 if indicator is not None: | |
495 saveIndicator(cursor, inter.getNum(), indicator) | |
496 except sqlite3.OperationalError as error: | |
497 printDBError(error) | |
498 connection.commit() | |
499 | |
500 def loadInteractionsFromSqlite(filename): | |
501 '''Loads interaction and their indicators | |
502 | |
503 TODO choose the interactions to load''' | |
504 interactions = [] | |
505 with sqlite3.connect(filename) as connection: | |
506 cursor = connection.cursor() | |
507 try: | |
508 cursor.execute('SELECT INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND WHERE INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type, IND.frame_number') | |
509 interactionNum = -1 | |
510 indicatorTypeNum = -1 | |
511 tmpIndicators = {} | |
512 for row in cursor: | |
513 if row[0] != interactionNum: | |
514 interactionNum = row[0] | |
515 interactions.append(events.Interaction(interactionNum, moving.TimeInterval(row[3],row[4]), row[1], row[2])) | |
516 interactions[-1].indicators = {} | |
517 if indicatorTypeNum != row[5] or row[0] != interactionNum: | |
518 indicatorTypeNum = row[5] | |
519 indicatorName = events.Interaction.indicatorNames[indicatorTypeNum] | |
520 indicatorValues = {row[6]:row[7]} | |
521 interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues, mostSevereIsMax = not indicatorName in events.Interaction.timeIndicators) | |
522 else: | |
523 indicatorValues[row[6]] = row[7] | |
524 interactions[-1].indicators[indicatorName].timeInterval.last = row[6] | |
525 except sqlite3.OperationalError as error: | |
526 printDBError(error) | |
527 return [] | |
528 return interactions | |
529 # load first and last object instants | |
530 # CREATE TEMP TABLE IF NOT EXISTS object_instants AS SELECT OF.object_id, min(frame_number) as first_instant, max(frame_number) as last_instant from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id group by OF.object_id order by OF.object_id | |
531 | |
532 def createBoundingBoxTable(filename, invHomography = None): | |
533 '''Create the table to store the object bounding boxes in image space | |
534 ''' | |
535 with sqlite3.connect(filename) as connection: | |
536 cursor = connection.cursor() | |
537 try: | |
538 cursor.execute('CREATE TABLE IF NOT EXISTS bounding_boxes (object_id INTEGER, frame_number INTEGER, x_top_left REAL, y_top_left REAL, x_bottom_right REAL, y_bottom_right REAL, PRIMARY KEY(object_id, frame_number))') | |
539 cursor.execute('INSERT INTO bounding_boxes SELECT object_id, frame_number, min(x), min(y), max(x), max(y) from ' | |
540 '(SELECT object_id, frame_number, (x*{}+y*{}+{})/w as x, (x*{}+y*{}+{})/w as y from ' | |
541 '(SELECT OF.object_id, P.frame_number, P.x_coordinate as x, P.y_coordinate as y, P.x_coordinate*{}+P.y_coordinate*{}+{} as w from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id)) '.format(invHomography[0,0], invHomography[0,1], invHomography[0,2], invHomography[1,0], invHomography[1,1], invHomography[1,2], invHomography[2,0], invHomography[2,1], invHomography[2,2])+ | |
542 'GROUP BY object_id, frame_number') | |
543 except sqlite3.OperationalError as error: | |
544 printDBError(error) | |
545 connection.commit() | |
546 | |
547 def loadBoundingBoxTableForDisplay(filename): | |
548 '''Loads bounding boxes from bounding_boxes table for display over trajectories''' | |
549 boundingBoxes = {} # list of bounding boxes for each instant | |
550 with sqlite3.connect(filename) as connection: | |
551 cursor = connection.cursor() | |
552 try: | |
553 cursor.execute('SELECT name FROM sqlite_master WHERE type=\'table\' AND name=\'bounding_boxes\'') | |
554 result = cursor.fetchall() | |
555 if len(result) > 0: | |
556 cursor.execute('SELECT * FROM bounding_boxes') | |
557 for row in cursor: | |
558 boundingBoxes.setdefault(row[1], []).append([moving.Point(row[2], row[3]), moving.Point(row[4], row[5])]) | |
559 except sqlite3.OperationalError as error: | |
560 printDBError(error) | |
561 return boundingBoxes | |
562 return boundingBoxes | |
563 | |
564 ######################### | |
565 # saving and loading for scene interpretation: POIs and Prototypes | |
566 ######################### | |
567 | |
568 def savePrototypesToSqlite(filename, prototypes): | |
569 '''save the prototypes (a prototype is defined by a filename, a number (id) and type''' | |
570 with sqlite3.connect(filename) as connection: | |
571 cursor = connection.cursor() | |
572 try: | |
573 cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_filename VARCHAR, prototype_id INTEGER, trajectory_type VARCHAR CHECK (trajectory_type IN (\"feature\", \"object\")), nmatchings INTEGER, PRIMARY KEY (prototype_filename, prototype_id, trajectory_type))') | |
574 for p in prototypes: | |
575 cursor.execute('INSERT INTO prototypes VALUES(?,?,?,?)', (p.getFilename(), p.getNum(), p.getTrajectoryType(), p.getNMatchings())) | |
576 except sqlite3.OperationalError as error: | |
577 printDBError(error) | |
578 connection.commit() | |
579 | |
580 def savePrototypeAssignmentsToSqlite(filename, objects, objectType, labels, prototypes): | |
581 with sqlite3.connect(filename) as connection: | |
582 cursor = connection.cursor() | |
583 try: | |
584 if objectType == 'feature': | |
585 tableName = 'features_prototypes' | |
586 objectIdColumnName = 'trajectory_id' | |
587 elif objectType == 'object': | |
588 tableName = 'objects_prototypes' | |
589 objectIdColumnName = 'object_id' | |
590 cursor.execute('CREATE TABLE IF NOT EXISTS '+tableName+' ('+objectIdColumnName+' INTEGER, prototype_filename VARCHAR, prototype_id INTEGER, trajectory_type VARCHAR CHECK (trajectory_type IN (\"feature\", \"object\")), PRIMARY KEY('+objectIdColumnName+', prototype_filename, prototype_id, trajectory_type))') | |
591 for obj, label in zip(objects, labels): | |
592 proto = prototypes[label] | |
593 cursor.execute('INSERT INTO objects_prototypes VALUES(?,?,?,?)', (obj.getNum(), proto.getFilename(), proto.getNum(), proto.getTrajectoryType())) | |
594 except sqlite3.OperationalError as error: | |
595 printDBError(error) | |
596 connection.commit() | |
597 | |
598 def loadPrototypesFromSqlite(filename, withTrajectories = True): | |
599 'Loads prototype ids and matchings (if stored)' | |
600 prototypes = [] | |
601 with sqlite3.connect(filename) as connection: | |
602 cursor = connection.cursor() | |
603 objects = [] | |
604 try: | |
605 cursor.execute('SELECT * FROM prototypes') | |
606 for row in cursor: | |
607 prototypes.append(moving.Prototype(row[0], row[1], row[2], row[3])) | |
608 if withTrajectories: | |
609 for p in prototypes: | |
610 p.setMovingObject(loadTrajectoriesFromSqlite(p.getFilename(), p.getTrajectoryType(), [p.getNum()])[0]) | |
611 # loadingInformation = {} # complicated slightly optimized | |
612 # for p in prototypes: | |
613 # dbfn = p.getFilename() | |
614 # trajType = p.getTrajectoryType() | |
615 # if (dbfn, trajType) in loadingInformation: | |
616 # loadingInformation[(dbfn, trajType)].append(p) | |
617 # else: | |
618 # loadingInformation[(dbfn, trajType)] = [p] | |
619 # for k, v in loadingInformation.iteritems(): | |
620 # objects += loadTrajectoriesFromSqlite(k[0], k[1], [p.getNum() for p in v]) | |
621 except sqlite3.OperationalError as error: | |
622 printDBError(error) | |
623 if len(set([p.getTrajectoryType() for p in prototypes])) > 1: | |
624 print('Different types of prototypes in database ({}).'.format(set([p.getTrajectoryType() for p in prototypes]))) | |
625 return prototypes | |
626 | |
627 def savePOIsToSqlite(filename, gmm, gmmType, gmmId): | |
628 '''Saves a Gaussian mixture model (of class sklearn.mixture.GaussianMixture) | |
629 gmmType is a type of GaussianMixture, learnt either from beginnings or ends of trajectories''' | |
630 with sqlite3.connect(filename) as connection: | |
631 cursor = connection.cursor() | |
632 if gmmType not in ['beginning', 'end']: | |
633 print('Unknown POI type {}. Exiting'.format(gmmType)) | |
634 import sys | |
635 sys.exit() | |
636 try: | |
637 cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))') | |
638 for i in range(gmm.n_components): | |
639 cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist()))) | |
640 connection.commit() | |
641 except sqlite3.OperationalError as error: | |
642 printDBError(error) | |
643 | |
644 def savePOIAssignmentsToSqlite(filename, objects): | |
645 'save the od fields of objects' | |
646 with sqlite3.connect(filename) as connection: | |
647 cursor = connection.cursor() | |
648 try: | |
649 cursor.execute('CREATE TABLE IF NOT EXISTS objects_pois (object_id INTEGER, origin_poi_id INTEGER, destination_poi_id INTEGER, PRIMARY KEY(object_id))') | |
650 for o in objects: | |
651 cursor.execute('INSERT INTO objects_pois VALUES(?,?,?)', (o.getNum(), o.od[0], o.od[1])) | |
652 connection.commit() | |
653 except sqlite3.OperationalError as error: | |
654 printDBError(error) | |
655 | |
656 def loadPOIsFromSqlite(filename): | |
657 'Loads all 2D Gaussians in the database' | |
658 from sklearn import mixture # todo if not avalaible, load data in duck-typed class with same fields | |
659 from ast import literal_eval | |
660 pois = [] | |
661 with sqlite3.connect(filename) as connection: | |
662 cursor = connection.cursor() | |
663 try: | |
664 cursor.execute('SELECT * from gaussians2d') | |
665 gmmId = None | |
666 gmm = [] | |
667 for row in cursor: | |
668 if gmmId is None or row[0] != gmmId: | |
669 if len(gmm) > 0: | |
670 tmp = mixture.GaussianMixture(len(gmm), covarianceType) | |
671 tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) | |
672 tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) | |
673 tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) | |
674 tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] | |
675 tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) | |
676 pois.append(tmp) | |
677 gaussian = {'type': row[2], | |
678 'mean': row[3:5], | |
679 'covar': array(literal_eval(row[5])), | |
680 'weight': row[7], | |
681 'precisions': array(literal_eval(row[8]))} | |
682 gmm = [gaussian] | |
683 covarianceType = row[6] | |
684 gmmId = row[0] | |
685 else: | |
686 gmm.append({'type': row[2], | |
687 'mean': row[3:5], | |
688 'covar': array(literal_eval(row[5])), | |
689 'weight': row[7], | |
690 'precisions': array(literal_eval(row[8]))}) | |
691 if len(gmm) > 0: | |
692 tmp = mixture.GaussianMixture(len(gmm), covarianceType) | |
693 tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) | |
694 tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) | |
695 tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) | |
696 tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] | |
697 tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) | |
698 pois.append(tmp) | |
699 except sqlite3.OperationalError as error: | |
700 printDBError(error) | |
701 return pois | |
702 | |
703 ######################### | |
704 # saving and loading for scene interpretation (Mohamed Gomaa Mohamed's PhD) | |
705 ######################### | |
706 | |
707 def writePrototypesToSqlite(prototypes,nMatching, outputFilename): | |
708 ''' prototype dataset is a dictionary with keys== routes, values== prototypes Ids ''' | |
709 connection = sqlite3.connect(outputFilename) | |
710 cursor = connection.cursor() | |
711 | |
712 cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))') | |
713 | |
714 for route in prototypes: | |
715 if prototypes[route]!=[]: | |
716 for i in prototypes[route]: | |
717 cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i])) | |
718 | |
719 connection.commit() | |
720 connection.close() | |
721 | |
722 def readPrototypesFromSqlite(filename): | |
723 ''' | |
724 This function loads the prototype file in the database | |
725 It returns a dictionary for prototypes for each route and nMatching | |
726 ''' | |
727 prototypes = {} | |
728 nMatching={} | |
729 | |
730 connection = sqlite3.connect(filename) | |
731 cursor = connection.cursor() | |
732 | |
733 try: | |
734 cursor.execute('SELECT * from prototypes order by prototype_id, routeIDstart,routeIDend, nMatching') | |
735 except sqlite3.OperationalError as error: | |
736 printDBError(error) | |
737 return [] | |
738 | |
739 for row in cursor: | |
740 route=(row[1],row[2]) | |
741 if route not in prototypes: | |
742 prototypes[route]=[] | |
743 prototypes[route].append(row[0]) | |
744 nMatching[row[0]]=row[3] | |
745 | |
746 connection.close() | |
747 return prototypes,nMatching | |
748 | |
749 def writeLabelsToSqlite(labels, outputFilename): | |
750 """ labels is a dictionary with keys: routes, values: prototypes Ids | |
751 """ | |
752 connection = sqlite3.connect(outputFilename) | |
753 cursor = connection.cursor() | |
754 | |
755 cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") | |
756 | |
757 for route in labels: | |
758 if labels[route]!=[]: | |
759 for i in labels[route]: | |
760 for j in labels[route][i]: | |
761 cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) | |
762 | |
763 connection.commit() | |
764 connection.close() | |
765 | |
766 def loadLabelsFromSqlite(filename): | |
767 labels = {} | |
768 | |
769 connection = sqlite3.connect(filename) | |
770 cursor = connection.cursor() | |
771 | |
772 try: | |
773 cursor.execute('SELECT * from labels order by object_id, routeIDstart,routeIDend, prototype_id') | |
774 except sqlite3.OperationalError as error: | |
775 printDBError(error) | |
776 return [] | |
777 | |
778 for row in cursor: | |
779 route=(row[1],row[2]) | |
780 p=row[3] | |
781 if route not in labels: | |
782 labels[route]={} | |
783 if p not in labels[route]: | |
784 labels[route][p]=[] | |
785 labels[route][p].append(row[0]) | |
786 | |
787 connection.close() | |
788 return labels | |
789 | |
790 def writeSpeedPrototypeToSqlite(prototypes,nmatching, outFilename): | |
791 """ to match the format of second layer prototypes""" | |
792 connection = sqlite3.connect(outFilename) | |
793 cursor = connection.cursor() | |
794 | |
795 cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") | |
796 | |
797 for route in prototypes: | |
798 if prototypes[route]!={}: | |
799 for i in prototypes[route]: | |
800 if prototypes[route][i]!= []: | |
801 for j in prototypes[route][i]: | |
802 cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) | |
803 | |
804 connection.commit() | |
805 connection.close() | |
806 | |
807 def loadSpeedPrototypeFromSqlite(filename): | |
808 """ | |
809 This function loads the prototypes table in the database of name <filename>. | |
810 """ | |
811 prototypes = {} | |
812 nMatching={} | |
813 connection = sqlite3.connect(filename) | |
814 cursor = connection.cursor() | |
815 | |
816 try: | |
817 cursor.execute('SELECT * from speedprototypes order by spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching') | |
818 except sqlite3.OperationalError as error: | |
819 printDBError(error) | |
820 return [] | |
821 | |
822 for row in cursor: | |
823 route=(row[2],row[3]) | |
824 if route not in prototypes: | |
825 prototypes[route]={} | |
826 if row[1] not in prototypes[route]: | |
827 prototypes[route][row[1]]=[] | |
828 prototypes[route][row[1]].append(row[0]) | |
829 nMatching[row[0]]=row[4] | |
830 | |
831 connection.close() | |
832 return prototypes,nMatching | |
833 | |
834 | |
835 def writeRoutesToSqlite(Routes, outputFilename): | |
836 """ This function writes the activity path define by start and end IDs""" | |
837 connection = sqlite3.connect(outputFilename) | |
838 cursor = connection.cursor() | |
839 | |
840 cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") | |
841 | |
842 for route in Routes: | |
843 if Routes[route]!=[]: | |
844 for i in Routes[route]: | |
845 cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) | |
846 | |
847 connection.commit() | |
848 connection.close() | |
849 | |
850 def loadRoutesFromSqlite(filename): | |
851 Routes = {} | |
852 | |
853 connection = sqlite3.connect(filename) | |
854 cursor = connection.cursor() | |
855 | |
856 try: | |
857 cursor.execute('SELECT * from routes order by object_id, routeIDstart,routeIDend') | |
858 except sqlite3.OperationalError as error: | |
859 printDBError(error) | |
860 return [] | |
861 | |
862 for row in cursor: | |
863 route=(row[1],row[2]) | |
864 if route not in Routes: | |
865 Routes[route]=[] | |
866 Routes[route].append(row[0]) | |
867 | |
868 connection.close() | |
869 return Routes | |
870 | |
871 def setRoutes(filename, objects): | |
872 connection = sqlite3.connect(filename) | |
873 cursor = connection.cursor() | |
874 for obj in objects: | |
875 cursor.execute('update objects set startRouteID = {} WHERE object_id = {}'.format(obj.startRouteID, obj.getNum())) | |
876 cursor.execute('update objects set endRouteID = {} WHERE object_id = {}'.format(obj.endRouteID, obj.getNum())) | |
877 connection.commit() | |
878 connection.close() | |
879 | |
880 ######################### | |
881 # txt files | |
882 ######################### | |
883 | |
884 def openCheck(filename, option = 'r', quitting = False): | |
885 '''Open file filename in read mode by default | |
886 and checks it is open''' | |
887 try: | |
888 return open(filename, option) | |
889 except IOError: | |
890 print('File {} could not be opened.'.format(filename)) | |
891 if quitting: | |
892 from sys import exit | |
893 exit() | |
894 return None | |
895 | |
896 def readline(f, commentCharacters = commentChar): | |
897 '''Modified readline function to skip comments | |
898 Can take a list of characters or a string (in will work in both)''' | |
899 s = f.readline() | |
900 while (len(s) > 0) and s[0] in commentCharacters: | |
901 s = f.readline() | |
902 return s.strip() | |
903 | |
904 def getLines(f, delimiterChar = delimiterChar, commentCharacters = commentChar): | |
905 '''Gets a complete entry (all the lines) in between delimiterChar.''' | |
906 dataStrings = [] | |
907 s = readline(f, commentCharacters) | |
908 while len(s) > 0 and s[0] != delimiterChar: | |
909 dataStrings += [s.strip()] | |
910 s = readline(f, commentCharacters) | |
911 return dataStrings | |
912 | |
913 def saveList(filename, l): | |
914 f = openCheck(filename, 'w') | |
915 for x in l: | |
916 f.write('{}\n'.format(x)) | |
917 f.close() | |
918 | |
919 def loadListStrings(filename, commentCharacters = commentChar): | |
920 f = openCheck(filename, 'r') | |
921 result = getLines(f, commentCharacters) | |
922 f.close() | |
923 return result | |
924 | |
925 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar): | |
926 values = [] | |
927 for l in loadListStrings(filename, commentCharacters): | |
928 if l.startswith(option): | |
929 values.append(l.split(delimiterChar)[1].strip()) | |
930 return values | |
931 | |
932 def addSectionHeader(propertiesFile, headerName = 'main'): | |
933 '''Add fake section header | |
934 | |
935 from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788 | |
936 use read_file in Python 3.2+ | |
937 ''' | |
938 yield '[{}]\n'.format(headerName) | |
939 for line in propertiesFile: | |
940 yield line | |
941 | |
942 def loadPemsTraffic(filename): | |
943 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse | |
944 into pandas dataframe''' | |
945 f = openCheck(filename) | |
946 l = f.readline().strip() | |
947 items = l.split(',') | |
948 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] | |
949 nLanes = (len(items)-len(headers))/3 | |
950 for i in range(nLanes): | |
951 headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)] | |
952 f.close() | |
953 return read_csv(filename, delimiter = ',', names = headers) | |
954 | |
955 def generatePDLaneColumn(data): | |
956 data['LANE'] = data['LANE\\LINK\\NO'].astype(str)+'_'+data['LANE\\INDEX'].astype(str) | |
957 | |
958 def convertTrajectoriesVissimToSqlite(filename): | |
959 '''Relies on a system call to sqlite3 | |
960 sqlite3 [file.sqlite] < import_fzp.sql''' | |
961 sqlScriptFilename = "import_fzp.sql" | |
962 # create sql file | |
963 out = openCheck(sqlScriptFilename, "w") | |
964 out.write(".separator \";\"\n"+ | |
965 "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+ | |
966 ".import "+filename+" curvilinear_positions\n"+ | |
967 "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") | |
968 out.close() | |
969 # system call | |
970 from subprocess import run | |
971 out = openCheck("err.log", "w") | |
972 run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) | |
973 out.close() | |
974 shutil.os.remove(sqlScriptFilename) | |
975 | |
976 def loadObjectNumbersInLinkFromVissimFile(filename, linkIds): | |
977 '''Finds the ids of the objects that go through any of the link in the list linkIds''' | |
978 with sqlite3.connect(filename) as connection: | |
979 cursor = connection.cursor() | |
980 queryStatement = 'SELECT DISTINCT trajectory_id FROM curvilinear_positions where link_id IN ('+','.join([str(id) for id in linkIds])+')' | |
981 try: | |
982 cursor.execute(queryStatement) | |
983 return [row[0] for row in cursor] | |
984 except sqlite3.OperationalError as error: | |
985 printDBError(error) | |
986 | |
987 def getNObjectsInLinkFromVissimFile(filename, linkIds): | |
988 '''Returns the number of objects that traveled through the link ids''' | |
989 with sqlite3.connect(filename) as connection: | |
990 cursor = connection.cursor() | |
991 queryStatement = 'SELECT link_id, COUNT(DISTINCT trajectory_id) FROM curvilinear_positions where link_id IN ('+','.join([str(id) for id in linkIds])+') GROUP BY link_id' | |
992 try: | |
993 cursor.execute(queryStatement) | |
994 return {row[0]:row[1] for row in cursor} | |
995 except sqlite3.OperationalError as error: | |
996 printDBError(error) | |
997 | |
998 def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, objectNumbers = None, warmUpLastInstant = None, usePandas = False, nDecimals = 2, lowMemory = True): | |
999 '''Reads data from VISSIM .fzp trajectory file | |
1000 simulationStepsPerTimeUnit is the number of simulation steps per unit of time used by VISSIM (second) | |
1001 for example, there seems to be 10 simulation steps per simulated second in VISSIM, | |
1002 so simulationStepsPerTimeUnit should be 10, | |
1003 so that all times correspond to the number of the simulation step (and can be stored as integers) | |
1004 | |
1005 Objects positions will be considered only after warmUpLastInstant | |
1006 (if the object has no such position, it won't be loaded) | |
1007 | |
1008 Assumed to be sorted over time | |
1009 Warning: if reading from SQLite a limited number of objects, objectNumbers will be the maximum object id''' | |
1010 objects = {} # dictionary of objects index by their id | |
1011 | |
1012 if usePandas: | |
1013 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory) | |
1014 generatePDLaneColumn(data) | |
1015 data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit | |
1016 if warmUpLastInstant is not None: | |
1017 data = data[data['TIME']>=warmUpLastInstant] | |
1018 grouped = data.loc[:,['NO','TIME']].groupby(['NO'], as_index = False) | |
1019 instants = grouped['TIME'].agg({'first': npmin, 'last': npmax}) | |
1020 for row_index, row in instants.iterrows(): | |
1021 objNum = int(row['NO']) | |
1022 tmp = data[data['NO'] == objNum] | |
1023 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last'])) | |
1024 # positions should be rounded to nDecimals decimals only | |
1025 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) | |
1026 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: | |
1027 return list(objects.values()) | |
1028 else: | |
1029 if filename.endswith(".fzp"): | |
1030 inputfile = openCheck(filename, quitting = True) | |
1031 line = readline(inputfile, '*$') | |
1032 while len(line) > 0:#for line in inputfile: | |
1033 data = line.strip().split(';') | |
1034 objNum = int(data[1]) | |
1035 instant = float(data[0])*simulationStepsPerTimeUnit | |
1036 s = float(data[4]) | |
1037 y = float(data[5]) | |
1038 lane = data[2]+'_'+data[3] | |
1039 if objNum not in objects: | |
1040 if warmUpLastInstant is None or instant >= warmUpLastInstant: | |
1041 if objectNumbers is None or len(objects) < objectNumbers: | |
1042 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) | |
1043 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() | |
1044 if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: | |
1045 objects[objNum].timeInterval.last = instant | |
1046 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) | |
1047 line = readline(inputfile, '*$') | |
1048 elif filename.endswith(".sqlite"): | |
1049 with sqlite3.connect(filename) as connection: | |
1050 cursor = connection.cursor() | |
1051 queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions' | |
1052 if objectNumbers is not None: | |
1053 queryStatement += ' WHERE trajectory_id '+getObjectCriteria(objectNumbers) | |
1054 queryStatement += ' ORDER BY trajectory_id, t' | |
1055 try: | |
1056 cursor.execute(queryStatement) | |
1057 for row in cursor: | |
1058 objNum = row[1] | |
1059 instant = row[0]*simulationStepsPerTimeUnit | |
1060 s = row[4] | |
1061 y = row[5] | |
1062 lane = '{}_{}'.format(row[2], row[3]) | |
1063 if objNum not in objects: | |
1064 if warmUpLastInstant is None or instant >= warmUpLastInstant: | |
1065 if objectNumbers is None or len(objects) < objectNumbers: | |
1066 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) | |
1067 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() | |
1068 if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: | |
1069 objects[objNum].timeInterval.last = instant | |
1070 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) | |
1071 except sqlite3.OperationalError as error: | |
1072 printDBError(error) | |
1073 else: | |
1074 print("File type of "+filename+" not supported (only .sqlite and .fzp files)") | |
1075 return list(objects.values()) | |
1076 | |
1077 def selectPDLanes(data, lanes = None): | |
1078 '''Selects the subset of data for the right lanes | |
1079 | |
1080 Lane format is a string 'x_y' where x is link index and y is lane index''' | |
1081 if lanes is not None: | |
1082 if 'LANE' not in data.columns: | |
1083 generatePDLaneColumn(data) | |
1084 indices = (data['LANE'] == lanes[0]) | |
1085 for l in lanes[1:]: | |
1086 indices = indices | (data['LANE'] == l) | |
1087 return data[indices] | |
1088 else: | |
1089 return data | |
1090 | |
1091 def countStoppedVehiclesVissim(filename, lanes = None, proportionStationaryTime = 0.7): | |
1092 '''Counts the number of vehicles stopped for a long time in a VISSIM trajectory file | |
1093 and the total number of vehicles | |
1094 | |
1095 Vehicles are considered finally stationary | |
1096 if more than proportionStationaryTime of their total time | |
1097 If lanes is not None, only the data for the selected lanes will be provided | |
1098 (format as string x_y where x is link index and y is lane index)''' | |
1099 if filename.endswith(".fzp"): | |
1100 columns = ['NO', '$VEHICLE:SIMSEC', 'POS'] | |
1101 if lanes is not None: | |
1102 columns += ['LANE\\LINK\\NO', 'LANE\\INDEX'] | |
1103 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory) | |
1104 data = selectPDLanes(data, lanes) | |
1105 data.sort(['$VEHICLE:SIMSEC'], inplace = True) | |
1106 | |
1107 nStationary = 0 | |
1108 nVehicles = 0 | |
1109 for name, group in data.groupby(['NO'], sort = False): | |
1110 nVehicles += 1 | |
1111 positions = array(group['POS']) | |
1112 diff = positions[1:]-positions[:-1] | |
1113 if npsum(diff == 0.) >= proportionStationaryTime*(len(positions)-1): | |
1114 nStationary += 1 | |
1115 elif filename.endswith(".sqlite"): | |
1116 # select trajectory_id, t, s_coordinate, speed from curvilinear_positions where trajectory_id between 1860 and 1870 and speed < 0.1 | |
1117 # pb of the meaning of proportionStationaryTime in arterial network? Why proportion of existence time? | |
1118 pass | |
1119 else: | |
1120 print("File type of "+filename+" not supported (only .sqlite and .fzp files)") | |
1121 | |
1122 return nStationary, nVehicles | |
1123 | |
1124 def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2, lowMemory = True): | |
1125 '''Counts the number of collisions per lane in a VISSIM trajectory file | |
1126 | |
1127 To distinguish between cars passing and collision, | |
1128 one checks when the sign of the position difference inverts | |
1129 (if the time are closer than collisionTimeDifference) | |
1130 If lanes is not None, only the data for the selected lanes will be provided | |
1131 (format as string x_y where x is link index and y is lane index)''' | |
1132 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory) | |
1133 data = selectPDLanes(data, lanes) | |
1134 data = data.convert_objects(convert_numeric=True) | |
1135 | |
1136 merged = merge(data, data, how='inner', left_on=['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC'], sort = False) | |
1137 merged = merged[merged['NO_x']>merged['NO_y']] | |
1138 | |
1139 nCollisions = 0 | |
1140 for name, group in merged.groupby(['LANE\\LINK\\NO', 'LANE\\INDEX', 'NO_x', 'NO_y']): | |
1141 diff = group['POS_x']-group['POS_y'] | |
1142 # diff = group['POS_x']-group['POS_y'] # to check the impact of convert_objects and the possibility of using type conversion in read_csv or function to convert strings if any | |
1143 if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0: | |
1144 xidx = diff[diff < 0].argmax() | |
1145 yidx = diff[diff > 0].argmin() | |
1146 if abs(group.loc[xidx, '$VEHICLE:SIMSEC'] - group.loc[yidx, '$VEHICLE:SIMSEC']) <= collisionTimeDifference: | |
1147 nCollisions += 1 | |
1148 | |
1149 # select TD1.link_id, TD1.lane_id from temp.diff_positions as TD1, temp.diff_positions as TD2 where TD1.link_id = TD2.link_id and TD1.lane_id = TD2.lane_id and TD1.id1 = TD2.id1 and TD1.id2 = TD2.id2 and TD1.t = TD2.t+0.1 and TD1.diff*TD2.diff < 0; # besoin de faire un group by?? | |
1150 # create temp table diff_positions as select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id; | |
1151 # SQL select link_id, lane_id, id1, id2, min(diff), max(diff) from (select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id) group by link_id, lane_id, id1, id2 having min(diff)*max(diff) < 0 | |
1152 return nCollisions | |
1153 | |
1154 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): | |
1155 '''Reads data from the trajectory data provided by NGSIM project | |
1156 and returns the list of Feature objects''' | |
1157 objects = [] | |
1158 | |
1159 inputfile = openCheck(filename, quitting = True) | |
1160 | |
1161 def createObject(numbers): | |
1162 firstFrameNum = int(numbers[1]) | |
1163 # do the geometry and usertype | |
1164 | |
1165 firstFrameNum = int(numbers[1]) | |
1166 lastFrameNum = firstFrameNum+int(numbers[2])-1 | |
1167 #time = moving.TimeInterval(firstFrameNum, firstFrameNum+int(numbers[2])-1) | |
1168 obj = moving.MovingObject(num = int(numbers[0]), | |
1169 timeInterval = moving.TimeInterval(firstFrameNum, lastFrameNum), | |
1170 positions = moving.Trajectory([[float(numbers[6])],[float(numbers[7])]]), | |
1171 userType = int(numbers[10])) | |
1172 obj.userType = int(numbers[10]) | |
1173 obj.laneNums = [int(numbers[13])] | |
1174 obj.precedingVehicles = [int(numbers[14])] # lead vehicle (before) | |
1175 obj.followingVehicles = [int(numbers[15])] # following vehicle (after) | |
1176 obj.spaceHeadways = [float(numbers[16])] # feet | |
1177 obj.timeHeadways = [float(numbers[17])] # seconds | |
1178 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate | |
1179 obj.speeds = [float(numbers[11])] | |
1180 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object | |
1181 return obj | |
1182 | |
1183 numbers = readline(inputfile).strip().split() | |
1184 if (len(numbers) > 0): | |
1185 obj = createObject(numbers) | |
1186 | |
1187 for line in inputfile: | |
1188 numbers = line.strip().split() | |
1189 if obj.getNum() != int(numbers[0]): | |
1190 # check and adapt the length to deal with issues in NGSIM data | |
1191 if (obj.length() != obj.positions.length()): | |
1192 print('length pb with object {} ({},{})'.format(obj.getNum(),obj.length(),obj.positions.length())) | |
1193 obj.last = obj.getFirstInstant()+obj.positions.length()-1 | |
1194 #obj.velocities = utils.computeVelocities(f.positions) # compare norm to speeds ? | |
1195 objects.append(obj) | |
1196 if (nObjects>0) and (len(objects)>=nObjects): | |
1197 break | |
1198 obj = createObject(numbers) | |
1199 else: | |
1200 obj.laneNums.append(int(numbers[13])) | |
1201 obj.positions.addPositionXY(float(numbers[6]), float(numbers[7])) | |
1202 obj.curvilinearPositions.addPositionSYL(float(numbers[5]), float(numbers[4]), obj.laneNums[-1]) | |
1203 obj.speeds.append(float(numbers[11])) | |
1204 obj.precedingVehicles.append(int(numbers[14])) | |
1205 obj.followingVehicles.append(int(numbers[15])) | |
1206 obj.spaceHeadways.append(float(numbers[16])) | |
1207 obj.timeHeadways.append(float(numbers[17])) | |
1208 | |
1209 if (obj.size[0] != float(numbers[8])): | |
1210 print('changed length obj {}'.format(obj.getNum())) | |
1211 if (obj.size[1] != float(numbers[9])): | |
1212 print('changed width obj {}'.format(obj.getNum())) | |
1213 | |
1214 inputfile.close() | |
1215 return objects | |
1216 | |
1217 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): | |
1218 '''Reads data from the trajectory data provided by NGSIM project | |
1219 and converts to our current format.''' | |
1220 if append: | |
1221 out = openCheck(outputfile,'a') | |
1222 else: | |
1223 out = openCheck(outputfile,'w') | |
1224 nObjectsPerType = [0,0,0] | |
1225 | |
1226 features = loadNgsimFile(inputfile, sequenceNum) | |
1227 for f in features: | |
1228 nObjectsPerType[f.userType-1] += 1 | |
1229 f.write(out) | |
1230 | |
1231 print(nObjectsPerType) | |
1232 | |
1233 out.close() | |
1234 | |
1235 def loadPinholeCameraModel(filename, tanalystFormat = True): | |
1236 '''Loads the data from a file containing the camera parameters | |
1237 (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) | |
1238 and returns a dictionary''' | |
1239 if tanalystFormat: | |
1240 f = openCheck(filename, quitting = True) | |
1241 content = getLines(f) | |
1242 cameraData = {} | |
1243 for l in content: | |
1244 tmp = l.split(':') | |
1245 cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.')) | |
1246 return cameraData | |
1247 else: | |
1248 print('Unknown camera model (not tanalyst format') | |
1249 return None | |
1250 | |
1251 def savePositionsToCsv(f, obj): | |
1252 timeInterval = obj.getTimeInterval() | |
1253 positions = obj.getPositions() | |
1254 curvilinearPositions = obj.getCurvilinearPositions() | |
1255 for i in range(int(obj.length())): | |
1256 p1 = positions[i] | |
1257 s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y) | |
1258 if curvilinearPositions is not None: | |
1259 p2 = curvilinearPositions[i] | |
1260 s += ',{},{}'.format(p2[0],p2[1]) | |
1261 f.write(s+'\n') | |
1262 | |
1263 def saveTrajectoriesToCsv(filename, objects): | |
1264 f = openCheck(filename, 'w') | |
1265 for i,obj in enumerate(objects): | |
1266 savePositionsToCsv(f, obj) | |
1267 f.close() | |
1268 | |
1269 | |
1270 ######################### | |
1271 # Utils to read .ini type text files for configuration, meta data... | |
1272 ######################### | |
1273 | |
1274 class ClassifierParameters(VideoFilenameAddable): | |
1275 'Class for the parameters of object classifiers' | |
1276 def loadConfigFile(self, filename): | |
1277 from configparser import ConfigParser | |
1278 | |
1279 config = ConfigParser() | |
1280 config.read_file(addSectionHeader(openCheck(filename))) | |
1281 | |
1282 parentPath = Path(filename).parent | |
1283 self.sectionHeader = config.sections()[0] | |
1284 | |
1285 self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename')) | |
1286 self.bikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'bv-svm-filename')) | |
1287 self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop') | |
1288 self.minNPixels = config.getint(self.sectionHeader, 'min-npixels-crop') | |
1289 x = config.getint(self.sectionHeader, 'hog-rescale-size') | |
1290 self.hogRescaleSize = (x, x) | |
1291 self.hogNOrientations = config.getint(self.sectionHeader, 'hog-norientations') | |
1292 x = config.getint(self.sectionHeader, 'hog-npixels-cell') | |
1293 self.hogNPixelsPerCell = (x, x) | |
1294 x = config.getint(self.sectionHeader, 'hog-ncells-block') | |
1295 self.hogNCellsPerBlock = (x, x) | |
1296 self.hogBlockNorm = config.get(self.sectionHeader, 'hog-block-norm') | |
1297 | |
1298 self.speedAggregationMethod = config.get(self.sectionHeader, 'speed-aggregation-method') | |
1299 self.nFramesIgnoreAtEnds = config.getint(self.sectionHeader, 'nframes-ignore-at-ends') | |
1300 self.speedAggregationCentile = config.getint(self.sectionHeader, 'speed-aggregation-centile') | |
1301 self.minSpeedEquiprobable = config.getfloat(self.sectionHeader, 'min-speed-equiprobable') | |
1302 self.maxPercentUnknown = config.getfloat(self.sectionHeader, 'max-prop-unknown-appearance') | |
1303 self.maxPedestrianSpeed = config.getfloat(self.sectionHeader, 'max-ped-speed') | |
1304 self.maxCyclistSpeed = config.getfloat(self.sectionHeader, 'max-cyc-speed') | |
1305 self.meanPedestrianSpeed = config.getfloat(self.sectionHeader, 'mean-ped-speed') | |
1306 self.stdPedestrianSpeed = config.getfloat(self.sectionHeader, 'std-ped-speed') | |
1307 self.locationCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-loc') | |
1308 self.scaleCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-scale') | |
1309 self.meanVehicleSpeed = config.getfloat(self.sectionHeader, 'mean-veh-speed') | |
1310 self.stdVehicleSpeed = config.getfloat(self.sectionHeader, 'std-veh-speed') | |
1311 | |
1312 def __init__(self, filename = None): | |
1313 if filename is not None and Path(filename).exists(): | |
1314 self.loadConfigFile(filename) | |
1315 else: | |
1316 print('Configuration filename {} could not be loaded.'.format(filename)) | |
1317 | |
1318 def convertToFrames(self, frameRate, speedRatio = 3.6): | |
1319 '''Converts parameters with a relationship to time in 'native' frame time | |
1320 speedRatio is the conversion from the speed unit in the config file | |
1321 to the distance per second | |
1322 | |
1323 ie param(config file) = speedRatio x fps x param(used in program) | |
1324 eg km/h = 3.6 (m/s to km/h) x frame/s x m/frame''' | |
1325 denominator = frameRate*speedRatio | |
1326 #denominator2 = denominator**2 | |
1327 self.minSpeedEquiprobable = self.minSpeedEquiprobable/denominator | |
1328 self.maxPedestrianSpeed = self.maxPedestrianSpeed/denominator | |
1329 self.maxCyclistSpeed = self.maxCyclistSpeed/denominator | |
1330 self.meanPedestrianSpeed = self.meanPedestrianSpeed/denominator | |
1331 self.stdPedestrianSpeed = self.stdPedestrianSpeed/denominator | |
1332 self.meanVehicleSpeed = self.meanVehicleSpeed/denominator | |
1333 self.stdVehicleSpeed = self.stdVehicleSpeed/denominator | |
1334 # special case for the lognormal distribution | |
1335 self.locationCyclistSpeed = self.locationCyclistSpeed-log(denominator) | |
1336 #self.scaleCyclistSpeed = self.scaleCyclistSpeed # no modification of scale | |
1337 | |
1338 | |
1339 class ProcessParameters(VideoFilenameAddable): | |
1340 '''Class for all parameters controlling data processing: input, | |
1341 method parameters, etc. for tracking and safety | |
1342 | |
1343 Note: framerate is already taken into account''' | |
1344 | |
1345 def loadConfigFile(self, filename): | |
1346 from configparser import ConfigParser | |
1347 | |
1348 config = ConfigParser(strict=False) | |
1349 config.read_file(addSectionHeader(openCheck(filename))) | |
1350 | |
1351 parentPath = Path(filename).parent | |
1352 self.sectionHeader = config.sections()[0] | |
1353 # Tracking/display parameters | |
1354 self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) | |
1355 self.databaseFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'database-filename')) | |
1356 self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename')) | |
1357 if Path(self.homographyFilename).exists(): | |
1358 self.homography = loadtxt(self.homographyFilename) | |
1359 else: | |
1360 self.homography = None | |
1361 self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename')) | |
1362 if Path(self.intrinsicCameraFilename).exists(): | |
1363 self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename) | |
1364 else: | |
1365 self.intrinsicCameraMatrix = None | |
1366 distortionCoefficients = getValuesFromINIFile(filename, 'distortion-coefficients', '=') | |
1367 self.distortionCoefficients = [float(x) for x in distortionCoefficients] | |
1368 self.undistortedImageMultiplication = config.getfloat(self.sectionHeader, 'undistorted-size-multiplication') | |
1369 self.undistort = config.getboolean(self.sectionHeader, 'undistort') | |
1370 self.firstFrameNum = config.getint(self.sectionHeader, 'frame1') | |
1371 self.videoFrameRate = config.getfloat(self.sectionHeader, 'video-fps') | |
1372 | |
1373 self.minFeatureTime = config.getfloat(self.sectionHeader, 'min-feature-time') | |
1374 | |
1375 self.classifierFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'classifier-filename')) | |
1376 | |
1377 # Safety parameters | |
1378 self.maxPredictedSpeed = config.getfloat(self.sectionHeader, 'max-predicted-speed')/3.6/self.videoFrameRate | |
1379 self.predictionTimeHorizon = config.getfloat(self.sectionHeader, 'prediction-time-horizon')*self.videoFrameRate | |
1380 self.collisionDistance = config.getfloat(self.sectionHeader, 'collision-distance') | |
1381 self.crossingZones = config.getboolean(self.sectionHeader, 'crossing-zones') | |
1382 self.predictionMethod = config.get(self.sectionHeader, 'prediction-method') | |
1383 self.nPredictedTrajectories = config.getint(self.sectionHeader, 'npredicted-trajectories') | |
1384 self.maxNormalAcceleration = config.getfloat(self.sectionHeader, 'max-normal-acceleration')/self.videoFrameRate**2 | |
1385 self.maxNormalSteering = config.getfloat(self.sectionHeader, 'max-normal-steering')/self.videoFrameRate | |
1386 self.minExtremeAcceleration = config.getfloat(self.sectionHeader, 'min-extreme-acceleration')/self.videoFrameRate**2 | |
1387 self.maxExtremeAcceleration = config.getfloat(self.sectionHeader, 'max-extreme-acceleration')/self.videoFrameRate**2 | |
1388 self.maxExtremeSteering = config.getfloat(self.sectionHeader, 'max-extreme-steering')/self.videoFrameRate | |
1389 self.useFeaturesForPrediction = config.getboolean(self.sectionHeader, 'use-features-prediction') | |
1390 self.constantSpeedPrototypePrediction = config.getboolean(self.sectionHeader, 'constant-speed') | |
1391 self.maxLcssDistance = config.getfloat(self.sectionHeader, 'max-lcss-distance') | |
1392 self.lcssMetric = config.get(self.sectionHeader, 'lcss-metric') | |
1393 self.minLcssSimilarity = config.getfloat(self.sectionHeader, 'min-lcss-similarity') | |
1394 | |
1395 def __init__(self, filename = None): | |
1396 if filename is not None and Path(filename).exists(): | |
1397 self.loadConfigFile(filename) | |
1398 else: | |
1399 print('Configuration filename {} could not be loaded.'.format(filename)) | |
1400 | |
1401 def processVideoArguments(args): | |
1402 '''Loads information from configuration file | |
1403 then checks what was passed on the command line | |
1404 for override (eg video filename and database filename''' | |
1405 parentPath = Path(args.configFilename).parent | |
1406 if args.configFilename is not None: # consider there is a configuration file | |
1407 params = ProcessParameters(args.configFilename) | |
1408 videoFilename = params.videoFilename | |
1409 databaseFilename = params.databaseFilename | |
1410 if params.homography is not None: | |
1411 invHomography = linalg.inv(params.homography) | |
1412 else: | |
1413 invHomography = None | |
1414 intrinsicCameraMatrix = params.intrinsicCameraMatrix | |
1415 distortionCoefficients = array(params.distortionCoefficients) | |
1416 undistortedImageMultiplication = params.undistortedImageMultiplication | |
1417 undistort = params.undistort | |
1418 firstFrameNum = params.firstFrameNum | |
1419 else: | |
1420 invHomography = None | |
1421 undistort = False | |
1422 intrinsicCameraMatrix = None | |
1423 distortionCoefficients = [] | |
1424 undistortedImageMultiplication = None | |
1425 undistort = False | |
1426 firstFrameNum = 0 | |
1427 | |
1428 # override video and database filenames if present on command line | |
1429 # if not absolute, make all filenames relative to the location of the configuration filename | |
1430 if args.videoFilename is not None: | |
1431 videoFilename = args.videoFilename | |
1432 else: | |
1433 videoFilename = params.videoFilename | |
1434 if args.databaseFilename is not None: | |
1435 databaseFilename = args.databaseFilename | |
1436 else: | |
1437 databaseFilename = params.databaseFilename | |
1438 | |
1439 return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum | |
1440 | |
1441 # deprecated | |
1442 class SceneParameters(object): | |
1443 def __init__(self, config, sectionName): | |
1444 from configparser import NoOptionError | |
1445 from ast import literal_eval | |
1446 try: | |
1447 self.sitename = config.get(sectionName, 'sitename') | |
1448 self.databaseFilename = config.get(sectionName, 'data-filename') | |
1449 self.homographyFilename = config.get(sectionName, 'homography-filename') | |
1450 self.calibrationFilename = config.get(sectionName, 'calibration-filename') | |
1451 self.videoFilename = config.get(sectionName, 'video-filename') | |
1452 self.frameRate = config.getfloat(sectionName, 'framerate') | |
1453 self.date = datetime.strptime(config.get(sectionName, 'date'), datetimeFormat) # 2011-06-22 11:00:39 | |
1454 self.translation = literal_eval(config.get(sectionName, 'translation')) # = [0.0, 0.0] | |
1455 self.rotation = config.getfloat(sectionName, 'rotation') | |
1456 self.duration = config.getint(sectionName, 'duration') | |
1457 except NoOptionError as e: | |
1458 print(e) | |
1459 print('Not a section for scene meta-data') | |
1460 | |
1461 @staticmethod | |
1462 def loadConfigFile(filename): | |
1463 from configparser import ConfigParser | |
1464 config = ConfigParser() | |
1465 config.readfp(openCheck(filename)) | |
1466 configDict = dict() | |
1467 for sectionName in config.sections(): | |
1468 configDict[sectionName] = SceneParameters(config, sectionName) | |
1469 return configDict | |
1470 | |
1471 | |
1472 if __name__ == "__main__": | |
1473 import doctest | |
1474 import unittest | |
1475 suite = doctest.DocFileSuite('tests/storage.txt') | |
1476 unittest.TextTestRunner().run(suite) | |
1477 # #doctest.testmod() | |
1478 # #doctest.testfile("example.txt") |