Mercurial Hosting > traffic-intelligence

comparison scripts/learn-motion-patterns.py @ 1054:d13f9bfbf3ff

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Retry
author Wendlasida
date Fri, 06 Jul 2018 18:42:58 -0400
parents f2ba9858e6c6
children
comparison
equal deleted inserted replaced
1053:60cc87e824c4 1054:d13f9bfbf3ff
3 import sys, argparse 3 import sys, argparse
4 4
5 import numpy as np 5 import numpy as np
6 import matplotlib.pyplot as plt 6 import matplotlib.pyplot as plt
7 7
8 from trafficintelligence import ml, utils, storage, moving 8 from trafficintelligence import ml, utils, storage, moving, processing
9 9
10 parser = argparse.ArgumentParser(description='''The program clusters trajectories, each cluster being represented by a trajectory. It can either work on the same dataset (database) or different ones, but only does learning or assignment at a time to avoid issues (the minimum cluster size argument is not used for now as it may change prototypes when assigning other trajectories)''') #, epilog = '' 10 parser = argparse.ArgumentParser(description='''The program clusters trajectories, each cluster being represented by a trajectory. It can either work on the same dataset (database) or different ones, but only does learning or assignment at a time to avoid issues''') #, epilog = ''
11 #parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') 11 #parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
12 parser.add_argument('-d', dest = 'databaseFilename', help = 'name of the Sqlite database file', required = True) 12 parser.add_argument('-d', dest = 'databaseFilename', help = 'name of the Sqlite database file', required = True)
13 parser.add_argument('-o', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes') 13 parser.add_argument('-o', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes')
14 parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with') 14 parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
15 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories to learn from', choices = ['objectfeature', 'feature', 'object'], default = 'objectfeatures') 15 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories to process', choices = ['feature', 'object'], default = 'feature')
16 parser.add_argument('--max-nobjectfeatures', dest = 'maxNObjectFeatures', help = 'maximum number of features per object to load', type = int, default = 1) 16 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
17 parser.add_argument('-n', dest = 'nTrajectories', help = 'number of the object or feature trajectories to load', type = int, default = None) 17 parser.add_argument('-n', dest = 'nObjects', help = 'number of the object or feature trajectories to load', type = int, default = None)
18 parser.add_argument('-e', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float, required = True) 18 parser.add_argument('-e', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float, required = True)
19 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance 19 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
20 parser.add_argument('-s', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float, required = True) 20 parser.add_argument('-s', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float, required = True)
21 parser.add_argument('-c', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0) 21 #parser.add_argument('-c', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
22 parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true') 22 parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
23 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true') 23 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
24 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true') 24 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
25 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int) 25 parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int)
26 parser.add_argument('--display', dest = 'display', help = 'display trajectories', action = 'store_true') 26 parser.add_argument('--display', dest = 'display', help = 'display trajectories', action = 'store_true')
27 parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
27 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true') 28 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
28 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true') 29 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
29 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true') 30 parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true')
30 31
31 args = parser.parse_args() 32 args = parser.parse_args()
37 # TODO? 4. when assigning, allow min cluster size only to avoid assigning to small clusters (but prototypes are not removed even if in small clusters, can be done after assignment with nmatchings) 38 # TODO? 4. when assigning, allow min cluster size only to avoid assigning to small clusters (but prototypes are not removed even if in small clusters, can be done after assignment with nmatchings)
38 39
39 # TODO add possibility to cluster with velocities 40 # TODO add possibility to cluster with velocities
40 # TODO add possibility to load all trajectories and use minclustersize 41 # TODO add possibility to load all trajectories and use minclustersize
41 42
42 # load trajectories to cluster or assign 43 if args.learn and args.assign:
43 if args.trajectoryType == 'objectfeature': 44 print('Cannot learn and assign simultaneously')
44 trajectoryType = 'feature' 45 sys.exit(0)
45 objectFeatureNumbers = storage.loadObjectFeatureFrameNumbers(args.databaseFilename, objectNumbers = args.nTrajectories) 46
46 featureNumbers = [] 47 objects = storage.loadTrajectoriesFromSqlite(args.databaseFilename, args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
47 for numbers in objectFeatureNumbers.values(): 48 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
48 featureNumbers += numbers[:min(len(numbers), args.maxNObjectFeatures)] 49 objectsWithFeatures = objects
49 objects = storage.loadTrajectoriesFromSqlite(args.databaseFilename, 'feature', objectNumbers = featureNumbers, timeStep = args.positionSubsamplingRate) 50 objects = [f for o in objectsWithFeatures for f in o.getFeatures()]
51 prototypeType = 'feature'
50 else: 52 else:
51 trajectoryType = args.trajectoryType 53 prototypeType = args.trajectoryType
52 objects = storage.loadTrajectoriesFromSqlite(args.databaseFilename, trajectoryType, objectNumbers = args.nTrajectories, timeStep = args.positionSubsamplingRate)
53
54 trajectories = [o.getPositions().asArray().T for o in objects]
55 54
56 # load initial prototypes, if any 55 # load initial prototypes, if any
57 if args.inputPrototypeDatabaseFilename is not None: 56 if args.inputPrototypeDatabaseFilename is not None:
58 initialPrototypes = storage.loadPrototypesFromSqlite(args.inputPrototypeDatabaseFilename, True) 57 initialPrototypes = storage.loadPrototypesFromSqlite(args.inputPrototypeDatabaseFilename, True)
59 trajectories = [p.getMovingObject().getPositions().asArray().T for p in initialPrototypes]+trajectories
60 if len(initialPrototypes) > 0:
61 initialPrototypeIndices = list(range(len(initialPrototypes)))
62 else:
63 initialPrototypeIndices = None
64 else: 58 else:
65 initialPrototypes = [] 59 initialPrototypes = []
66 initialPrototypeIndices = None
67 60
68 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon) 61 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
69 nTrajectories = len(trajectories) 62 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
63 nTrajectories = len(initialPrototypes)+len(objects)
64 if args.similaritiesFilename is not None:
65 similarities = np.loadtxt(args.similaritiesFilename)
66 if args.similaritiesFilename is None or similarities.shape[0] != nTrajectories or similarities.shape[1] != nTrajectories:
67 similarities = -np.ones((nTrajectories, nTrajectories))
70 68
71 similarities = -np.ones((nTrajectories, nTrajectories)) 69 prototypeIndices, labels = processing.learnAssignMotionPatterns(args.learn, args.assign, objects, similarities, args.minSimilarity, similarityFunc, 0, args.optimizeCentroid, args.randomInitialization, False, initialPrototypes)
72 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
73 # the next line can be called again without reinitializing similarities
74 if args.learn:
75 prototypeIndices = ml.prototypeCluster(trajectories, similarities, args.minSimilarity, similarityFunc, args.optimizeCentroid, args.randomInitialization, initialPrototypeIndices)
76 else:
77 prototypeIndices = initialPrototypeIndices
78 70
79 if args.assign: # TODO don't touch initial prototypes if not from same db as trajectories 71 if args.learn:# and not args.assign:
80 #if not args.learn and args.minClusterSize >= 1: # allow only
81 # print('Warning: you did not learn the prototypes and you are using minimum cluster size of {}, which may lead to removing prototypes and assigning them to others'.format(args.minClusterSize))
82 # if args.minClusterSize >= 1:
83 # if initialPrototypeIndices is None:
84 # prototypeIndices, labels = ml.assignToPrototypeClusters(trajectories, prototypeIndices, similarities, args.minSimilarity, similarityFunc, args.minClusterSize)
85 # else:
86 # print('Not assigning with non-zero minimum cluster size and initial prototypes (would remove initial prototypes based on other trajectories')
87 # else:
88 # prototypeIndices, labels = ml.assignToPrototypeClusters(trajectories, prototypeIndices, similarities, args.minSimilarity, similarityFunc)
89 assignedPrototypeIndices, labels = ml.assignToPrototypeClusters(trajectories, prototypeIndices, similarities, args.minSimilarity, similarityFunc)
90
91 if args.learn and not args.assign:
92 prototypes = [] 72 prototypes = []
93 for i in prototypeIndices: 73 for i in prototypeIndices:
94 if i<len(initialPrototypes): 74 if i<len(initialPrototypes):
95 prototypes.append(initialPrototypes[i]) 75 prototypes.append(initialPrototypes[i])
96 else: 76 else:
97 prototypes.append(moving.Prototype(args.databaseFilename, objects[i-len(initialPrototypes)].getNum(), trajectoryType)) 77 prototypes.append(moving.Prototype(args.databaseFilename, objects[i-len(initialPrototypes)].getNum(), prototypeType))
98 78
99 if args.outputPrototypeDatabaseFilename is None: 79 if args.outputPrototypeDatabaseFilename is None:
100 outputPrototypeDatabaseFilename = args.databaseFilename 80 outputPrototypeDatabaseFilename = args.databaseFilename
101 else: 81 else:
102 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename 82 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
108 for p in prototypes: 88 for p in prototypes:
109 p.getMovingObject().plot() 89 p.getMovingObject().plot()
110 plt.axis('equal') 90 plt.axis('equal')
111 plt.show() 91 plt.show()
112 92
113 if not args.learn and args.assign: # no modification to prototypes, can work with initialPrototypes 93 if args.assign: # not args.learn and no modification to prototypes, can work with initialPrototypes
114 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1) 94 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
115 for i in prototypeIndices: 95 for i in prototypeIndices:
116 nMatchings = clusterSizes[i]-1 96 nMatchings = clusterSizes[i]-1 # external prototypes
117 if initialPrototypes[i].nMatchings is None: 97 if initialPrototypes[i].nMatchings is None:
118 initialPrototypes[i].nMatchings = nMatchings 98 initialPrototypes[i].nMatchings = nMatchings
119 else: 99 else:
120 initialPrototypes[i].nMatchings += nMatchings 100 initialPrototypes[i].nMatchings += nMatchings
121 if args.outputPrototypeDatabaseFilename is None: 101 if args.outputPrototypeDatabaseFilename is None:
122 outputPrototypeDatabaseFilename = args.databaseFilename 102 outputPrototypeDatabaseFilename = args.databaseFilename
123 else: 103 else:
124 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename 104 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
125 storage.setPrototypeMatchingsInSqlite(outputPrototypeDatabaseFilename, initialPrototypes) 105 storage.setPrototypeMatchingsInSqlite(outputPrototypeDatabaseFilename, initialPrototypes)
126 if args.saveAssignments: 106 if args.saveAssignments:
127 if args.trajectoryType == 'objectfeature': # consider that the object is assigned through its longest features 107 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
108 # consider that the object is assigned through its longest features
109 # issues are inconsistencies in the number of matchings per prototype and display (will display features, not objects)
128 objectNumbers = [] 110 objectNumbers = []
129 objectLabels = [] 111 objectLabels = []
130 for objNum, objFeatureNumbers in objectFeatureNumbers.items(): 112 i = 0
113 for obj in objectsWithFeatures:
131 objLabels = [] 114 objLabels = []
132 for i, o in enumerate(objects): 115 for f in obj.getFeatures():
133 if o.getNum() in objFeatureNumbers: 116 if f == objects[i]:
134 objLabels.append(labels[i+len(initialPrototypes)]) 117 objLabels.append(labels[i+len(initialPrototypes)])
118 i += 1
119 else:
120 print('Issue with obj {} and feature {} (trajectory {})'.format(obj.getNum(), f.getNum(), i))
135 objectLabels.append(utils.mostCommon(objLabels)) 121 objectLabels.append(utils.mostCommon(objLabels))
136 objectNumbers.append(objNum) 122 objectNumbers.append(obj.getNum())
137 storage.savePrototypeAssignmentsToSqlite(args.databaseFilename, objectNumbers, 'object', objectLabels, initialPrototypes) 123 storage.savePrototypeAssignmentsToSqlite(args.databaseFilename, objectNumbers, 'object', objectLabels, initialPrototypes)
138 else: 124 else:
139 storage.savePrototypeAssignmentsToSqlite(args.databaseFilename, [obj.getNum() for obj in objects], trajectoryType, labels[len(initialPrototypes):], initialPrototypes) 125 storage.savePrototypeAssignmentsToSqlite(args.databaseFilename, [obj.getNum() for obj in objects], args.trajectoryType, labels[len(initialPrototypes):], initialPrototypes)
140 if args.display: 126 if args.display:
141 plt.figure() 127 plt.figure()
142 for i,o in enumerate(objects): 128 for i,o in enumerate(objects):
143 if labels[i+len(initialPrototypes)] < 0: 129 if labels[i+len(initialPrototypes)] < 0:
144 o.plot('kx-') 130 o.plot('kx-')
148 p.getMovingObject().plot(utils.colors[i]+'o') 134 p.getMovingObject().plot(utils.colors[i]+'o')
149 plt.axis('equal') 135 plt.axis('equal')
150 plt.show() 136 plt.show()
151 137
152 if (args.learn or args.assign) and args.saveSimilarities: 138 if (args.learn or args.assign) and args.saveSimilarities:
153 np.savetxt(utils.removeExtension(args.databaseFilename)+'-prototype-similarities.txt.gz', similarities, '%.4f') 139 if args.similaritiesFilename is not None:
140 np.savetxt(args.similaritiesFilename, similarities, '%.4f')
141 else:
142 np.savetxt(utils.removeExtension(args.databaseFilename)+'-prototype-similarities.txt.gz', similarities, '%.4f')