Mercurial Hosting > traffic-intelligence
changeset 746:e7ff0f60fef8
merged new developments (indicator and trajectory clustering)
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Thu, 10 Sep 2015 15:52:45 -0400 |
parents | c6d4ea05a2d0 (current diff) 3d0321abb564 (diff) |
children | d99866b0528a |
files | scripts/performance.py scripts/performance.sh |
diffstat | 13 files changed, 590 insertions(+), 415 deletions(-) [+] |
line wrap: on
line diff
--- a/python/events.py Mon Aug 10 01:06:59 2015 -0400 +++ b/python/events.py Thu Sep 10 15:52:45 2015 -0400 @@ -2,7 +2,7 @@ '''Libraries for events Interactions, pedestrian crossing...''' -import moving, prediction, indicators, utils, cvutils +import moving, prediction, indicators, utils, cvutils, ml from base import VideoFilenameAddable import numpy as np @@ -295,49 +295,8 @@ print('unknown type of point: '+pointType) return allPoints -def prototypeCluster(interactions, similarityMatrix, alignmentMatrix, indicatorName, minSimilarity): - '''Finds exemplar indicator time series for all interactions - Returns the prototype indices (in the interaction list) and the label of each indicator (interaction) - - if an indicator profile (time series) is different enough (<minSimilarity), - it will become a new prototype. - Non-prototype interactions will be assigned to an existing prototype''' - - # sort indicators based on length - indices = range(similarityMatrix.shape[0]) - def compare(i, j): - if len(interactions[i].getIndicator(indicatorName)) > len(interactions[j].getIndicator(indicatorName)): - return -1 - elif len(interactions[i].getIndicator(indicatorName)) == len(interactions[j].getIndicator(indicatorName)): - return 0 - else: - return 1 - indices.sort(compare) - # go through all indicators - prototypeIndices = [indices[0]] - for i in indices[1:]: - if similarityMatrix[i][prototypeIndices].max() < minSimilarity: - prototypeIndices.append(i) - - # assignment - labels = [-1]*similarityMatrix.shape[0] - indices = [i for i in range(similarityMatrix.shape[0]) if i not in prototypeIndices] - for i in prototypeIndices: - labels[i] = i - for i in indices: - prototypeIndex = similarityMatrix[i][prototypeIndices].argmax() - labels[i] = prototypeIndices[prototypeIndex] - - return prototypeIndices, labels - -def prototypeMultivariateCluster(interactions, similarityMatrics, indicatorNames, minSimilarities, minClusterSize): - '''Finds exmaple indicator time series (several indicators) for all interactions - - if any interaction indicator time series is different enough (<minSimilarity), - it will become a new prototype. - Non-prototype interactions will be assigned to an existing prototype if all indicators are similar enough''' - pass - +def prototypeCluster(interactions, similarities, indicatorName, minSimilarity, similarityFunc = None, minClusterSize = None, randomInitialization = False): + return ml.prototypeCluster([inter.getIndicator(indicatorName) for inter in interactions], similarities, minSimilarity, similarityFunc, minClusterSize, randomInitialization) class Crossing(moving.STObject): '''Class for the event of a street crossing
--- a/python/ml.py Mon Aug 10 01:06:59 2015 -0400 +++ b/python/ml.py Thu Sep 10 15:52:45 2015 -0400 @@ -111,13 +111,73 @@ code,distance = vq(features,centroids) # code starting from 0 (represent first cluster) to k-1 (last cluster) return code,sigma -def motionPatterLearning(objects, maxDistance): - ''' - Option to use only the (n?) longest features per object instead of all for speed up - TODO''' - pass +def prototypeCluster(instances, similarities, minSimilarity, similarityFunc = None, minClusterSize = None, randomInitialization = False): + '''Finds exemplar (prototype) instance that represent each cluster + Returns the prototype indices (in the instances list) and the cluster label of each instance + + the elements in the instances list must have a length (method __len__), or one can use the random initialization + the positions in the instances list corresponds to the similarities + if similarityFunc is provided, the similarities are calculated as needed (this is faster) if not in similarities (negative if not computed) + similarities must still be allocated with the right size + + if an instance is different enough (<minSimilarity), + it will become a new prototype. + Non-prototype instances will be assigned to an existing prototype + if minClusterSize is not None, the clusters will be refined by removing iteratively the smallest clusters + and reassigning all elements in the cluster until no cluster is smaller than minClusterSize''' -def prototypeCluster(): - ''' - TODO''' - pass + # sort instances based on length + indices = range(len(instances)) + if randomInitialization: + indices = np.random.permutation(indices) + else: + def compare(i, j): + if len(instances[i]) > len(instances[j]): + return -1 + elif len(instances[i]) == len(instances[j]): + return 0 + else: + return 1 + indices.sort(compare) + # go through all instances + prototypeIndices = [indices[0]] + for i in indices[1:]: + if similarityFunc is not None: + for j in prototypeIndices: + if similarities[i][j] < 0: + similarities[i][j] = similarityFunc(instances[i], instances[j]) + similarities[j][i] = similarities[i][j] + if similarities[i][prototypeIndices].max() < minSimilarity: + prototypeIndices.append(i) + + # assignment + indices = [i for i in range(similarities.shape[0]) if i not in prototypeIndices] + assign = True + while assign: + labels = [-1]*similarities.shape[0] + for i in prototypeIndices: + labels[i] = i + for i in indices: + if similarityFunc is not None: + for j in prototypeIndices: + if similarities[i][j] < 0: + similarities[i][j] = similarityFunc(instances[i], instances[j]) + similarities[j][i] = similarities[i][j] + prototypeIdx = similarities[i][prototypeIndices].argmax() + if similarities[i][prototypeIndices[prototypeIdx]] >= minSimilarity: + labels[i] = prototypeIndices[prototypeIdx] + else: + labels[i] = -1 # outlier + clusterSizes = {i: sum(np.array(labels) == i) for i in prototypeIndices} + smallestClusterIndex = min(clusterSizes, key = clusterSizes.get) + assign = (clusterSizes[smallestClusterIndex] < minClusterSize) + if assign: + prototypeIndices.remove(smallestClusterIndex) + indices.append(smallestClusterIndex) + + return prototypeIndices, labels + +def computeClusterSizes(labels, prototypeIndices, outlierIndex = -1): + clusterSizes = {i: sum(np.array(labels) == i) for i in prototypeIndices} + clusterSizes['outlier'] = sum(np.array(labels) == -1) + return clusterSizes
--- a/python/storage.py Mon Aug 10 01:06:59 2015 -0400 +++ b/python/storage.py Thu Sep 10 15:52:45 2015 -0400 @@ -34,6 +34,16 @@ except sqlite3.OperationalError as error: printDBError(error) +def tableExists(filename, tableName): + 'indicates if the table exists in the database' + try: + connection = sqlite3.connect(filename) + cursor = connection.cursor() + cursor.execute('SELECT COUNT(*) FROM SQLITE_MASTER WHERE type = \'table\' AND name = \''+tableName+'\'') + return cursor.fetchone()[0] == 1 + except sqlite3.OperationalError as error: + printDBError(error) + def createIndex(connection, tableName, columnName, unique = False): '''Creates an index for the column in the table I will make querying with a condition on this column faster''' @@ -109,208 +119,6 @@ connection.commit() connection.close() -def writeFeaturesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1): - '''write features trajectories maintain trajectory ID,velocities dataset ''' - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") - cursor.execute("CREATE TABLE IF NOT EXISTS \"velocities\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") - - if trajectoryType == 'feature': - if type(objectNumbers) == int and objectNumbers == -1: - for trajectory in objects: - trajectory_id = trajectory.num - frame_number = trajectory.timeInterval.first - for position,velocity in zip(trajectory.getPositions(),trajectory.getVelocities()): - cursor.execute("insert into positions (trajectory_id, frame_number, x_coordinate, y_coordinate) values (?,?,?,?)",(trajectory_id,frame_number,position.x,position.y)) - cursor.execute("insert into velocities (trajectory_id, frame_number, x_coordinate, y_coordinate) values (?,?,?,?)",(trajectory_id,frame_number,velocity.x,velocity.y)) - frame_number += 1 - - connection.commit() - connection.close() - -def writePrototypesToSqlite(prototypes,nMatching, outputFilename): - """ prototype dataset is a dictionary with keys== routes, values== prototypes Ids """ - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))") - - for route in prototypes.keys(): - if prototypes[route]!=[]: - for i in prototypes[route]: - cursor.execute("insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)",(i,route[0],route[1],nMatching[route][i])) - - connection.commit() - connection.close() - -def loadPrototypesFromSqlite(filename): - """ - This function loads the prototype file in the database - It returns a dictionary for prototypes for each route and nMatching - """ - prototypes = {} - nMatching={} - - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from prototypes order by prototype_id, routeIDstart,routeIDend, nMatching') - except sqlite3.OperationalError as error: - utils.printDBError(error) - return [] - - for row in cursor: - route=(row[1],row[2]) - if route not in prototypes.keys(): - prototypes[route]=[] - prototypes[route].append(row[0]) - nMatching[row[0]]=row[3] - - connection.close() - return prototypes,nMatching - -def writeLabelsToSqlite(labels, outputFilename): - """ labels is a dictionary with keys: routes, values: prototypes Ids - """ - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") - - for route in labels.keys(): - if labels[route]!=[]: - for i in labels[route]: - for j in labels[route][i]: - cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) - - connection.commit() - connection.close() - -def loadLabelsFromSqlite(filename): - labels = {} - - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from labels order by object_id, routeIDstart,routeIDend, prototype_id') - except sqlite3.OperationalError as error: - utils.printDBError(error) - return [] - - for row in cursor: - route=(row[1],row[2]) - p=row[3] - if route not in labels.keys(): - labels[route]={} - if p not in labels[route].keys(): - labels[route][p]=[] - labels[route][p].append(row[0]) - - connection.close() - return labels -def writeSpeedPrototypeToSqlite(prototypes,nmatching, outFilename): - """ to match the format of second layer prototypes""" - connection = sqlite3.connect(outFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS \"speedprototypes\"(spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") - - for route in prototypes.keys(): - if prototypes[route]!={}: - for i in prototypes[route]: - if prototypes[route][i]!= []: - for j in prototypes[route][i]: - cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) - - connection.commit() - connection.close() - -def loadSpeedPrototypeFromSqlite(filename): - """ - This function loads the prototypes table in the database of name <filename>. - """ - prototypes = {} - nMatching={} - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from speedprototypes order by spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching') - except sqlite3.OperationalError as error: - utils.printDBError(error) - return [] - - for row in cursor: - route=(row[2],row[3]) - if route not in prototypes.keys(): - prototypes[route]={} - if row[1] not in prototypes[route].keys(): - prototypes[route][row[1]]=[] - prototypes[route][row[1]].append(row[0]) - nMatching[row[0]]=row[4] - - connection.close() - return prototypes,nMatching - - -def writeRoutesToSqlite(Routes, outputFilename): - """ This function writes the activity path define by start and end IDs""" - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") - - for route in Routes.keys(): - if Routes[route]!=[]: - for i in Routes[route]: - cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) - - connection.commit() - connection.close() - -def loadRoutesFromSqlite(filename): - Routes = {} - - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from routes order by object_id, routeIDstart,routeIDend') - except sqlite3.OperationalError as error: - utils.printDBError(error) - return [] - - for row in cursor: - route=(row[1],row[2]) - if route not in Routes.keys(): - Routes[route]=[] - Routes[route].append(row[0]) - - connection.close() - return Routes - -def setRoutes(filename, objects): - connection = sqlite3.connect(filename) - cursor = connection.cursor() - for obj in objects: - cursor.execute('update objects set startRouteID = {} where object_id = {}'.format(obj.startRouteID, obj.getNum())) - cursor.execute('update objects set endRouteID = {} where object_id = {}'.format(obj.endRouteID, obj.getNum())) - connection.commit() - connection.close() - -def setRoadUserTypes(filename, objects): - '''Saves the user types of the objects in the sqlite database stored in filename - The objects should exist in the objects table''' - connection = sqlite3.connect(filename) - cursor = connection.cursor() - for obj in objects: - cursor.execute('update objects set road_user_type = {} where object_id = {}'.format(obj.getUserType(), obj.getNum())) - connection.commit() - connection.close() def loadPrototypeMatchIndexesFromSqlite(filename): """ @@ -477,6 +285,22 @@ connection.close() return objects +def savePrototypesToSqlite(filename, prototypes, trajectoryType = 'feature'): + 'Work in progress, do not use' + connection = sqlite3.connect(filename) + cursor = connection.cursor() + try: + cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (id INTEGER PRIMARY KEY, object_id INTEGER, trajectory_id INTEGER, nMatchings INTEGER, FOREIGN KEY(object_id) REFERENCES objects(id), FOREIGN KEY(trajectory_id) REFERENCES positions(trajectory_id))') + #for inter in interactions: + # saveInteraction(cursor, inter) + except sqlite3.OperationalError as error: + printDBError(error) + connection.commit() + connection.close() + +def loadPrototypesFromSqlite(filename): + pass + def loadGroundTruthFromSqlite(filename, gtType = 'bb', gtNumbers = None): 'Loads bounding box annotations (ground truth) from an SQLite ' connection = sqlite3.connect(filename) @@ -631,6 +455,214 @@ return boundingBoxes ######################### +# saving and loading for scene interpretation (Mohamed Gomaa Mohamed's PhD) +######################### + +def writeFeaturesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1): + '''write features trajectories maintain trajectory ID,velocities dataset ''' + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") + cursor.execute("CREATE TABLE IF NOT EXISTS \"velocities\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") + + if trajectoryType == 'feature': + if type(objectNumbers) == int and objectNumbers == -1: + for trajectory in objects: + trajectory_id = trajectory.num + frame_number = trajectory.timeInterval.first + for position,velocity in zip(trajectory.getPositions(),trajectory.getVelocities()): + cursor.execute("insert into positions (trajectory_id, frame_number, x_coordinate, y_coordinate) values (?,?,?,?)",(trajectory_id,frame_number,position.x,position.y)) + cursor.execute("insert into velocities (trajectory_id, frame_number, x_coordinate, y_coordinate) values (?,?,?,?)",(trajectory_id,frame_number,velocity.x,velocity.y)) + frame_number += 1 + + connection.commit() + connection.close() + +def writePrototypesToSqlite(prototypes,nMatching, outputFilename): + """ prototype dataset is a dictionary with keys== routes, values== prototypes Ids """ + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))") + + for route in prototypes.keys(): + if prototypes[route]!=[]: + for i in prototypes[route]: + cursor.execute("insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)",(i,route[0],route[1],nMatching[route][i])) + + connection.commit() + connection.close() + +def readPrototypesFromSqlite(filename): + """ + This function loads the prototype file in the database + It returns a dictionary for prototypes for each route and nMatching + """ + prototypes = {} + nMatching={} + + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from prototypes order by prototype_id, routeIDstart,routeIDend, nMatching') + except sqlite3.OperationalError as error: + utils.printDBError(error) + return [] + + for row in cursor: + route=(row[1],row[2]) + if route not in prototypes.keys(): + prototypes[route]=[] + prototypes[route].append(row[0]) + nMatching[row[0]]=row[3] + + connection.close() + return prototypes,nMatching + +def writeLabelsToSqlite(labels, outputFilename): + """ labels is a dictionary with keys: routes, values: prototypes Ids + """ + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") + + for route in labels.keys(): + if labels[route]!=[]: + for i in labels[route]: + for j in labels[route][i]: + cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) + + connection.commit() + connection.close() + +def loadLabelsFromSqlite(filename): + labels = {} + + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from labels order by object_id, routeIDstart,routeIDend, prototype_id') + except sqlite3.OperationalError as error: + utils.printDBError(error) + return [] + + for row in cursor: + route=(row[1],row[2]) + p=row[3] + if route not in labels.keys(): + labels[route]={} + if p not in labels[route].keys(): + labels[route][p]=[] + labels[route][p].append(row[0]) + + connection.close() + return labels + +def writeSpeedPrototypeToSqlite(prototypes,nmatching, outFilename): + """ to match the format of second layer prototypes""" + connection = sqlite3.connect(outFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS \"speedprototypes\"(spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") + + for route in prototypes.keys(): + if prototypes[route]!={}: + for i in prototypes[route]: + if prototypes[route][i]!= []: + for j in prototypes[route][i]: + cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) + + connection.commit() + connection.close() + +def loadSpeedPrototypeFromSqlite(filename): + """ + This function loads the prototypes table in the database of name <filename>. + """ + prototypes = {} + nMatching={} + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from speedprototypes order by spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching') + except sqlite3.OperationalError as error: + utils.printDBError(error) + return [] + + for row in cursor: + route=(row[2],row[3]) + if route not in prototypes.keys(): + prototypes[route]={} + if row[1] not in prototypes[route].keys(): + prototypes[route][row[1]]=[] + prototypes[route][row[1]].append(row[0]) + nMatching[row[0]]=row[4] + + connection.close() + return prototypes,nMatching + + +def writeRoutesToSqlite(Routes, outputFilename): + """ This function writes the activity path define by start and end IDs""" + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") + + for route in Routes.keys(): + if Routes[route]!=[]: + for i in Routes[route]: + cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) + + connection.commit() + connection.close() + +def loadRoutesFromSqlite(filename): + Routes = {} + + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from routes order by object_id, routeIDstart,routeIDend') + except sqlite3.OperationalError as error: + utils.printDBError(error) + return [] + + for row in cursor: + route=(row[1],row[2]) + if route not in Routes.keys(): + Routes[route]=[] + Routes[route].append(row[0]) + + connection.close() + return Routes + +def setRoutes(filename, objects): + connection = sqlite3.connect(filename) + cursor = connection.cursor() + for obj in objects: + cursor.execute('update objects set startRouteID = {} where object_id = {}'.format(obj.startRouteID, obj.getNum())) + cursor.execute('update objects set endRouteID = {} where object_id = {}'.format(obj.endRouteID, obj.getNum())) + connection.commit() + connection.close() + +def setRoadUserTypes(filename, objects): + '''Saves the user types of the objects in the sqlite database stored in filename + The objects should exist in the objects table''' + connection = sqlite3.connect(filename) + cursor = connection.cursor() + for obj in objects: + cursor.execute('update objects set road_user_type = {} where object_id = {}'.format(obj.getUserType(), obj.getNum())) + connection.commit() + connection.close() + +######################### # txt files ######################### @@ -797,7 +829,7 @@ return nStationary, nVehicles -def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2): +def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2, lowMemory = True): '''Counts the number of collisions per lane in a VISSIM trajectory file To distinguish between cars passing and collision, @@ -808,12 +840,14 @@ from pandas import read_csv, merge data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory) data = selectPDLanes(data, lanes) + data = data.convert_objects(convert_numeric=True) + merged = merge(data, data, how='inner', left_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], sort = False) merged = merged[merged['NO_x']>merged['NO_y']] nCollisions = 0 for name, group in merged.groupby(['LANE\LINK\NO', 'LANE\INDEX', 'NO_x', 'NO_y']): - diff = group['POS_x'].convert_objects(convert_numeric=True)-group['POS_y'].convert_objects(convert_numeric=True) + diff = group['POS_x']-group['POS_y'] # diff = group['POS_x']-group['POS_y'] # to check the impact of convert_objects and the possibility of using type conversion in read_csv or function to convert strings if any if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0: xidx = diff[diff < 0].argmax()
--- a/python/tests/moving.txt Mon Aug 10 01:06:59 2015 -0400 +++ b/python/tests/moving.txt Thu Sep 10 15:52:45 2015 -0400 @@ -38,7 +38,7 @@ 2 >>> TimeInterval(20,30).distance(TimeInterval(3,15)) 5 ->>> unionIntervals([TimeInterval(3,6), TimeInterval(8,10),TimeInterval(11,15)]) +>>> TimeInterval.unionIntervals([TimeInterval(3,6), TimeInterval(8,10),TimeInterval(11,15)]) [3, 15] >>> Point(3,4)-Point(1,7)
--- a/python/tests/utils.txt Mon Aug 10 01:06:59 2015 -0400 +++ b/python/tests/utils.txt Thu Sep 10 15:52:45 2015 -0400 @@ -1,6 +1,14 @@ >>> from utils import * >>> from moving import Point +>>> upperCaseFirstLetter('mmmm... donuts') +'Mmmm... Donuts' +>>> s = upperCaseFirstLetter('much ado about nothing') +>>> s == 'Much Ado About Nothing' +True +>>> upperCaseFirstLetter(s) == s +True + >>> computeChi2([],[]) 0.0 >>> computeChi2(range(1,10),range(1,10)) @@ -50,6 +58,13 @@ >>> mostCommon([range(2), range(4), range(2)]) [0, 1] +>>> res = sortByLength([range(3), range(4), range(1)]) +>>> [len(r) for r in res] +[1, 3, 4] +>>> res = sortByLength([range(3), range(4), range(1), range(5)], reverse = True) +>>> [len(r) for r in res] +[5, 4, 3, 1] + >>> lcss = LCSS(similarityFunc = lambda x,y: abs(x-y) <= 0.1) >>> lcss.compute(range(5), range(5)) 5
--- a/python/utils.py Mon Aug 10 01:06:59 2015 -0400 +++ b/python/utils.py Thu Sep 10 15:52:45 2015 -0400 @@ -7,12 +7,21 @@ from math import sqrt, ceil, floor from scipy.stats import kruskal, shapiro from scipy.spatial import distance -from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, median, isnan, ones, convolve, dtype, isnan, NaN, mean, ma +from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, median, isnan, ones, convolve, dtype, isnan, NaN, mean, ma, isinf datetimeFormat = "%Y-%m-%d %H:%M:%S" ######################### +# Strings +######################### + +def upperCaseFirstLetter(s): + words = s.split(' ') + lowerWords = [w[0].upper()+w[1:].lower() for w in words] + return ' '.join(lowerWords) + +######################### # Enumerations ######################### @@ -233,6 +242,19 @@ xsorted = sorted(D.keys()) return xsorted, [D[x] for x in xsorted] +def compareLengthForSort(i, j): + if len(i) < len(j): + return -1 + elif len(i) == len(j): + return 0 + else: + return 1 + +def sortByLength(instances, reverse = False): + '''Returns a new list with the instances sorted by length (method __len__) + reverse is passed to sorted''' + return sorted(instances, cmp = compareLengthForSort, reverse = reverse) + def ceilDecimals(v, nDecimals): '''Rounds the number at the nth decimal eg 1.23 at 0 decimal is 2, at 1 decimal is 1.3''' @@ -649,6 +671,8 @@ import sys sys.exit() else: + if similarityFunc is None and metric is not None and not isinf(delta): + print('Warning: you are using a cdist metric and a finite delta, which will make probably computation slower than using the equivalent similarityFunc (since all pairwise distances will be computed by cdist).') self.similarityFunc = similarityFunc self.metric = metric self.epsilon = epsilon @@ -865,7 +889,7 @@ def cleanFilename(s): 'cleans filenames obtained when contatenating figure characteristics' - return s.replace(' ','-').replace('.','').replace('/','-') + return s.replace(' ','-').replace('.','').replace('/','-').replace(',','') def listfiles(dirname, extension, remove = False): '''Returns the list of files with the extension in the directory dirname
--- a/samples/performances.txt Mon Aug 10 01:06:59 2015 -0400 +++ b/samples/performances.txt Thu Sep 10 15:52:45 2015 -0400 @@ -1,3 +1,9 @@ +python -m timeit 'from utils import LCSS; from numpy.random import random_sample; lcss = LCSS(metric = "cityblock", epsilon = 0.1); lcss.compute(random_sample((1000,2)), random_sample((2000, 2)))' +10 loops, best of 3: 1.6 sec per loop + +python -m timeit 'from utils import LCSS; from numpy.random import random_sample; lcss = LCSS(similarityFunc = lambda x,y: (abs(x[0]-y[0]) <= 0.1) and (abs(x[1]-y[1]) <= 0.1)); lcss.compute(random_sample((1000,2)), random_sample((2000, 2)))' +10 loops, best of 3: 3.48 sec per loop +######## Performance report using ../../Data/norway/2_video/20130606-0830.sqlite #### test query positions
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/learn-motion-patterns.py Thu Sep 10 15:52:45 2015 -0400 @@ -0,0 +1,65 @@ +#! /usr/bin/env python + +import sys, argparse + +#import matplotlib.pyplot as plt +import numpy as np + +import ml, utils, storage + +parser = argparse.ArgumentParser(description='The program learns prototypes for the motion patterns') #, epilog = '' +#parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') +parser.add_argument('-d', dest = 'databaseFilename', help = 'name of the Sqlite database file', required = True) +parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories to display', choices = ['objectfeatures', 'feature', 'object'], default = 'objectfeatures') +parser.add_argument('-n', dest = 'nTrajectories', help = 'number of the object or feature trajectories to load', type = int, default = None) +parser.add_argument('-e', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float, required = True) +parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance +parser.add_argument('-s', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float, required = True) +parser.add_argument('-c', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = None) +parser.add_argument('--display', dest = 'display', help = 'display trajectories', action = 'store_true') # default is manhattan distance + +args = parser.parse_args() + +# TODO parameters (random init?) and what to learn from: objects, features, longest features from objects +# TODO add possibility to cluter with velocities + +trajectoryType = args.trajectoryType +if args.trajectoryType == 'objectfeatures': + trajectoryType = 'object' + +#features = storage.loadTrajectoriesFromSqlite(databaseFilename, args.trajectoryType) +objects = storage.loadTrajectoriesFromSqlite(args.databaseFilename, trajectoryType, withFeatures = (args.trajectoryType == 'objectfeatures'), objectNumbers = args.nTrajectories) + +if args.trajectoryType == 'objectfeatures': + features = [] + for o in objects: + tmp = utils.sortByLength(o.getFeatures(), reverse = True) + features += tmp[:min(len(tmp), 3)] + objects = features + +trajectories = [o.getPositions().asArray().T for o in objects] + +lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon) +nTrajectories = len(trajectories) + +similarities = -np.ones((nTrajectories, nTrajectories)) +# for i in xrange(nTrajectories): +# for j in xrange(i): +# similarities[i,j] = lcss.computeNormalized(trajectories[i], trajectories[j]) +# similarities[j,i] = similarities[i,j] + +prototypeIndices, labels = ml.prototypeCluster(trajectories, similarities, args.minSimilarity, lambda x,y : lcss.computeNormalized(x, y), args.minClusterSize) # this line can be called again without reinitializing similarities + +if args.display: + from matplotlib.pyplot import figure + figure() + for i,o in enumerate(objects): + if i not in prototypeIndices: + if labels[i] < 0: + o.plot('kx') + else: + o.plot(utils.colors[labels[i]]) + for i in prototypeIndices: + objects[i].plot(utils.colors[i]+'o') + +# TODO store the prototypes (if features, easy, if objects, info must be stored about the type)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/performance-db.py Thu Sep 10 15:52:45 2015 -0400 @@ -0,0 +1,72 @@ +#! /usr/bin/env python + +import sys, shutil, os, sqlite3, timeit#, argparse + +import storage + +if len(sys.argv) >= 2: + dbFilename = sys.argv[1] +else: + dbFilename="./samples/laurier.sqlite" + +print("Performance report using {}".format(dbFilename)) +shutil.copy(dbFilename, dbFilename+'.test') +dbFilename=dbFilename+'.test' +connection = sqlite3.connect(dbFilename) + +nFeatures=storage.getNumberRowsTable(connection, "positions", "trajectory_id") +print dbFilename, nFeatures + +#### +# test query tmp trajectory instant table +#### +def selectTrajectoryIdInstant(connection, lastInstant): + cursor = connection.cursor() + for i in xrange(lastInstant): + cursor.execute("select trajectory_id from trajectory_instants where last_instant = {}".format(lastInstant)) + cursor.fetchall() + +def test2Setup(connection): + cursor = connection.cursor() + cursor.execute("CREATE TEMP TABLE IF NOT EXISTS trajectory_instants AS select trajectory_id, min(frame_number) as first_instant, max(frame_number) as last_instant, max(frame_number)-min(frame_number)+1 as length from positions group by trajectory_id") + return storage.getMinMax(connection, "trajectory_instants", "last_instant", "max") + +# lastInstant = test2Setup(connection) +# print timeit.timeit("selectTrajectoryIdInstant(connection, lastInstant)", setup="from __main__ import selectTrajectoryIdInstant, connection, lastInstant", number = 100) +# storage.createIndex(connection, "trajectory_instants", "last_instant") +# print timeit.timeit("selectTrajectoryIdInstant(connection, lastInstant)", setup="from __main__ import selectTrajectoryIdInstant, connection, lastInstant", number = 100) +# storage.createIndex(connection, "trajectory_instants", "trajectory_id", True) +# print timeit.timeit("selectTrajectoryIdInstant(connection, lastInstant)", setup="from __main__ import selectTrajectoryIdInstant, connection, lastInstant", number = 100) + + +#### +# test query positions/velocities in feature grouping +#### +def selectTrajectories(connection, nFeatures): + cursor = connection.cursor() + for i in xrange(nFeatures): + cursor.execute("select * from positions where trajectory_id = {} order by frame_number".format(i)) + cursor.fetchall() + +def dropIndices(connection): + cursor = connection.cursor() + cursor.execute("drop index if exists positions_trajectory_id_index") + #sqlite3 $dbFilename "drop index if exists frame_number_index" + #sqlite3 $dbFilename "drop index if exists tid_frame_index" + #sqlite3 $dbFilename "drop index if exists of_trajectory_id_index" + connection.commit() + +def test1Positions(): + print('''#### + test query positions + ####''') + print("no index") + dropIndices(connection) + print timeit.timeit("selectTrajectories(connection, nFeatures)", setup="from __main__ import selectTrajectories, connection, nFeatures", number = 100) + #### + print("with index on trajectory_id") + storage.createIndex(connection, "positions", "trajectory_id")#sqlite3 $dbFilename "create index trajectory_id_index on positions(trajectory_id)" + print timeit.timeit("selectTrajectories(connection, nFeatures)", setup="from __main__ import selectTrajectories, connection, nFeatures", number = 100) + +#### Cleanup +os.remove(dbFilename)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/performance-db.sh Thu Sep 10 15:52:45 2015 -0400 @@ -0,0 +1,83 @@ +#!/bin/bash +if [ $# -ge 1 ]; +then + dbFilename=$1 +else + dbFilename="./samples/laurier.sqlite" +fi +echo "Performance report using "$dbFilename +cp $dbFilename $dbFilename.test +dbFilename=$dbFilename.test +nFeatures=`sqlite3 $dbFilename "select count(distinct trajectory_id) from positions"` + +#### +# test query positions/velocities in feature grouping +#### +echo "####" +echo "test query positions" +echo "####" + +selectTrajectories() { + for (( i=1; i<=$nFeatures; i++ )) + do + sqlite3 $dbFilename "select * from positions where trajectory_id = ${i} order by frame_number" > /dev/null + done +} + +dropIndices() { + sqlite3 $dbFilename "drop index if exists trajectory_id_index" + sqlite3 $dbFilename "drop index if exists frame_number_index" + sqlite3 $dbFilename "drop index if exists tid_frame_index" + sqlite3 $dbFilename "drop index if exists of_trajectory_id_index" +} + +echo "no index" +dropIndices +time selectTrajectories +#### +echo "with index on trajectory_id" +sqlite3 $dbFilename "create index trajectory_id_index on positions(trajectory_id)" +time selectTrajectories +#### +echo "with index on trajectory_id and frame_number" +sqlite3 $dbFilename "create index frame_number_index on positions(frame_number)" +time selectTrajectories +#### +echo "with index on the 2 columns trajectory_id" +dropIndices +sqlite3 $dbFilename "create index tid_frame_index on positions(trajectory_id,frame_number)" +time selectTrajectories +#### +echo "query all feature positions, order by id and frame_number (index/no index)" +time (sqlite3 $dbFilename "select * from positions order by trajectory_id, frame_number" > /dev/null) +dropIndices +time (sqlite3 $dbFilename "select * from positions order by trajectory_id, frame_number" > /dev/null) +# no effect: probably because already ordered + +#### +# test query positions/velocities in feature grouping +#### +echo "####" +echo "test query object positions" +echo "####" + +selectObjectTrajectories() { + sqlite3 $dbFilename "SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from positions P, objects_features OF where P.trajectory_id = OF.trajectory_id group by OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number" > /dev/null +} +echo "no index" +dropIndices +time selectObjectTrajectories +#### +echo "index on trajectory_id" +sqlite3 $dbFilename "create index trajectory_id_index on positions(trajectory_id)" +time selectObjectTrajectories +#### +echo "index on trajectory_id in positions and objects_features" +sqlite3 $dbFilename "create index of_trajectory_id_index on objects_features(trajectory_id)" +time selectObjectTrajectories + + +### +echo "Cleaning up!" +#dropIndices +rm $dbFilename
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/performance-lcss.py Thu Sep 10 15:52:45 2015 -0400 @@ -0,0 +1,12 @@ +#! /usr/bin/env python + +import timeit + +vectorLength = 10 +number = 10 + +print('Default Python implementation with lambda') +print timeit.timeit('lcss.compute(random_sample(({},2)), random_sample(({}, 2)))'.format(vectorLength, vectorLength*2), setup = 'from utils import LCSS; from numpy.random import random_sample; lcss = LCSS(similarityFunc = lambda x,y: (abs(x[0]-y[0]) <= 0.1) and (abs(x[1]-y[1]) <= 0.1));', number = number) + +print('Using scipy distance.cdist') +print timeit.timeit('lcss.compute(random_sample(({},2)), random_sample(({}, 2)))'.format(vectorLength, vectorLength*2), setup = 'from utils import LCSS; from numpy.random import random_sample; lcss = LCSS(metric = "cityblock", epsilon = 0.1);', number = number)
--- a/scripts/performance.py Mon Aug 10 01:06:59 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,72 +0,0 @@ -#! /usr/bin/env python - -import sys, shutil, os, sqlite3, timeit#, argparse - -import storage - -if len(sys.argv) >= 2: - dbFilename = sys.argv[1] -else: - dbFilename="./samples/laurier.sqlite" - -print("Performance report using {}".format(dbFilename)) -shutil.copy(dbFilename, dbFilename+'.test') -dbFilename=dbFilename+'.test' -connection = sqlite3.connect(dbFilename) - -nFeatures=storage.getNumberRowsTable(connection, "positions", "trajectory_id") -print dbFilename, nFeatures - -#### -# test query tmp trajectory instant table -#### -def selectTrajectoryIdInstant(connection, lastInstant): - cursor = connection.cursor() - for i in xrange(lastInstant): - cursor.execute("select trajectory_id from trajectory_instants where last_instant = {}".format(lastInstant)) - cursor.fetchall() - -def test2Setup(connection): - cursor = connection.cursor() - cursor.execute("CREATE TEMP TABLE IF NOT EXISTS trajectory_instants AS select trajectory_id, min(frame_number) as first_instant, max(frame_number) as last_instant, max(frame_number)-min(frame_number)+1 as length from positions group by trajectory_id") - return storage.getMinMax(connection, "trajectory_instants", "last_instant", "max") - -# lastInstant = test2Setup(connection) -# print timeit.timeit("selectTrajectoryIdInstant(connection, lastInstant)", setup="from __main__ import selectTrajectoryIdInstant, connection, lastInstant", number = 100) -# storage.createIndex(connection, "trajectory_instants", "last_instant") -# print timeit.timeit("selectTrajectoryIdInstant(connection, lastInstant)", setup="from __main__ import selectTrajectoryIdInstant, connection, lastInstant", number = 100) -# storage.createIndex(connection, "trajectory_instants", "trajectory_id", True) -# print timeit.timeit("selectTrajectoryIdInstant(connection, lastInstant)", setup="from __main__ import selectTrajectoryIdInstant, connection, lastInstant", number = 100) - - -#### -# test query positions/velocities in feature grouping -#### -def selectTrajectories(connection, nFeatures): - cursor = connection.cursor() - for i in xrange(nFeatures): - cursor.execute("select * from positions where trajectory_id = {} order by frame_number".format(i)) - cursor.fetchall() - -def dropIndices(connection): - cursor = connection.cursor() - cursor.execute("drop index if exists positions_trajectory_id_index") - #sqlite3 $dbFilename "drop index if exists frame_number_index" - #sqlite3 $dbFilename "drop index if exists tid_frame_index" - #sqlite3 $dbFilename "drop index if exists of_trajectory_id_index" - connection.commit() - -def test1Positions(): - print('''#### - test query positions - ####''') - print("no index") - dropIndices(connection) - print timeit.timeit("selectTrajectories(connection, nFeatures)", setup="from __main__ import selectTrajectories, connection, nFeatures", number = 100) - #### - print("with index on trajectory_id") - storage.createIndex(connection, "positions", "trajectory_id")#sqlite3 $dbFilename "create index trajectory_id_index on positions(trajectory_id)" - print timeit.timeit("selectTrajectories(connection, nFeatures)", setup="from __main__ import selectTrajectories, connection, nFeatures", number = 100) - -#### Cleanup -os.remove(dbFilename)
--- a/scripts/performance.sh Mon Aug 10 01:06:59 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,83 +0,0 @@ -#!/bin/bash -if [ $# -ge 1 ]; -then - dbFilename=$1 -else - dbFilename="./samples/laurier.sqlite" -fi -echo "Performance report using "$dbFilename -cp $dbFilename $dbFilename.test -dbFilename=$dbFilename.test -nFeatures=`sqlite3 $dbFilename "select count(distinct trajectory_id) from positions"` - -#### -# test query positions/velocities in feature grouping -#### -echo "####" -echo "test query positions" -echo "####" - -selectTrajectories() { - for (( i=1; i<=$nFeatures; i++ )) - do - sqlite3 $dbFilename "select * from positions where trajectory_id = ${i} order by frame_number" > /dev/null - done -} - -dropIndices() { - sqlite3 $dbFilename "drop index if exists trajectory_id_index" - sqlite3 $dbFilename "drop index if exists frame_number_index" - sqlite3 $dbFilename "drop index if exists tid_frame_index" - sqlite3 $dbFilename "drop index if exists of_trajectory_id_index" -} - -echo "no index" -dropIndices -time selectTrajectories -#### -echo "with index on trajectory_id" -sqlite3 $dbFilename "create index trajectory_id_index on positions(trajectory_id)" -time selectTrajectories -#### -echo "with index on trajectory_id and frame_number" -sqlite3 $dbFilename "create index frame_number_index on positions(frame_number)" -time selectTrajectories -#### -echo "with index on the 2 columns trajectory_id" -dropIndices -sqlite3 $dbFilename "create index tid_frame_index on positions(trajectory_id,frame_number)" -time selectTrajectories -#### -echo "query all feature positions, order by id and frame_number (index/no index)" -time (sqlite3 $dbFilename "select * from positions order by trajectory_id, frame_number" > /dev/null) -dropIndices -time (sqlite3 $dbFilename "select * from positions order by trajectory_id, frame_number" > /dev/null) -# no effect: probably because already ordered - -#### -# test query positions/velocities in feature grouping -#### -echo "####" -echo "test query object positions" -echo "####" - -selectObjectTrajectories() { - sqlite3 $dbFilename "SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from positions P, objects_features OF where P.trajectory_id = OF.trajectory_id group by OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number" > /dev/null -} -echo "no index" -dropIndices -time selectObjectTrajectories -#### -echo "index on trajectory_id" -sqlite3 $dbFilename "create index trajectory_id_index on positions(trajectory_id)" -time selectObjectTrajectories -#### -echo "index on trajectory_id in positions and objects_features" -sqlite3 $dbFilename "create index of_trajectory_id_index on objects_features(trajectory_id)" -time selectObjectTrajectories - - -### -echo "Cleaning up!" -#dropIndices -rm $dbFilename