Mercurial Hosting > traffic-intelligence
changeset 1077:3939ae415be0
Merging
| author | Wendlasida |
|---|---|
| date | Fri, 20 Jul 2018 14:03:34 -0400 |
| parents | 108c5dc4e34a (current diff) 0154133e77df (diff) |
| children | 8cc3feb1c1c5 |
| files | ..hgignore.swp trafficintelligence/moving.py trafficintelligence/storage.py |
| diffstat | 11 files changed, 380 insertions(+), 241 deletions(-) [+] |
line wrap: on
line diff
--- a/c/Motion.cpp Fri Jul 20 13:50:43 2018 -0400 +++ b/c/Motion.cpp Fri Jul 20 14:03:34 2018 -0400 @@ -53,7 +53,7 @@ float disp = 0; for (unsigned int i=0; i<nDisplacements; i++) disp += displacementDistances[nPositions-2-i]; - result = disp <= minTotalFeatureDisplacement; + result = disp < minTotalFeatureDisplacement; } return result; }
--- a/scripts/compute-homography.py Fri Jul 20 13:50:43 2018 -0400 +++ b/scripts/compute-homography.py Fri Jul 20 14:03:34 2018 -0400 @@ -30,7 +30,7 @@ parser.add_argument('--display', dest = 'displayPoints', help = 'display original and projected points on both images', action = 'store_true') parser.add_argument('--intrinsic', dest = 'intrinsicCameraMatrixFilename', help = 'name of the intrinsic camera file') parser.add_argument('--distortion-coefficients', dest = 'distortionCoefficients', help = 'distortion coefficients', nargs = '*', type = float) -parser.add_argument('--undistorted-multiplication', dest = 'undistortedImageMultiplication', help = 'undistorted image multiplication', type = float) +parser.add_argument('--undistorted-multiplication', dest = 'undistortedImageMultiplication', help = 'undistorted image multiplication', type = float, default = 1.) parser.add_argument('--undistort', dest = 'undistort', help = 'undistort the video (because features have been extracted that way', action = 'store_true') parser.add_argument('--save', dest = 'saveImages', help = 'save the undistorted video frame (display option must be chosen)', action = 'store_true')
--- a/scripts/process.py Fri Jul 20 13:50:43 2018 -0400 +++ b/scripts/process.py Fri Jul 20 14:03:34 2018 -0400 @@ -8,7 +8,7 @@ #atplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np -from pandas import DataFrame +import pandas as pd from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml from trafficintelligence.metadata import * @@ -16,14 +16,15 @@ parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') # input parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) -parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int) +parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*') parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*') # main function parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction']) parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) -parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction']) +parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true') +parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'event']) # common options parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') @@ -60,11 +61,13 @@ # analysis options parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1) -parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = float, default = 15.) -parser.add_argument('--aggregation', dest = 'aggMethod', help = 'aggregation method per user/event and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) -parser.add_argument('--aggregation-centile', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) +parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15) +parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) +parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) +parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float) +parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data') dpi = 150 -# unit of analysis: site or video sequence? +# unit of analysis: site - camera-view # need way of selecting sites as similar as possible to sql alchemy syntax # override tracking.cfg from db @@ -82,16 +85,22 @@ videoSequences = [] sites = [] if args.videoIds is not None: - videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds] - siteIds = set([vs.cameraView.siteIdx for vs in videoSequences]) + for videoId in args.videoIds: + if '-' in videoId: + videoSequences.extend([session.query(VideoSequence).get(i) for i in moving.TimeInterval.parse(videoId)]) + else: + videoSequences.append(session.query(VideoSequence).get(int(videoId))) + videoSequences = [vs for vs in videoSequences if vs is not None] + sites = set([vs.cameraView.site for vs in videoSequences]) elif args.siteIds is not None: - siteIds = set(args.siteIds) - for siteId in siteIds: - tmpsites = getSite(session, siteId) - sites.extend(tmpsites) - for site in tmpsites: - for cv in site.cameraViews: - videoSequences.extend(cv.videoSequences) + for siteId in args.siteIds: + if '-' in siteId: + sites.extend([session.query(Site).get(i) for i in moving.TimeInterval.parse(siteId)]) + else: + sites.append(session.query(Site).get(int(siteId))) + sites = [s for s in sites if s is not None] + for site in sites: + videoSequences.extend(getSiteVideoSequences(site)) else: print('No video/site to process') @@ -99,6 +108,34 @@ pool = Pool(args.nProcesses) ################################# +# Report progress in the processing +################################# +if args.progress: + print('Providing information on data progress') + headers = ['site', 'vs', 'features', 'objects', 'interactions'] # todo add prototypes and object classification + data = [] + for site in sites: + unprocessedVideoSequences = [] + for vs in getSiteVideoSequences(site): + if (parentPath/vs.getDatabaseFilename()).is_file(): # TODO check time of file? + tableNames = storage.tableNames(str(parentPath.absolute()/vs.getDatabaseFilename())) + data.append([site.name, vs.idx, 'positions' in tableNames, 'objects' in tableNames, 'interactions' in tableNames]) + else: + unprocessedVideoSequences.append(vs) + data.append([site.name, vs.idx, False, False, False]) + #if len(unprocessedVideoSequences): + # print('Site {} ({}) has {} completely unprocessed video sequences'.format (site.name, site.idx, len(unprocessedVideoSequences))) + data = pd.DataFrame(data, columns = headers) + print('-'*80) + print('\t'+' '.join(headers[2:])) + print('-'*80) + for name, group in data.groupby(['site']): #.agg({'vs': 'count'})) + n = group.vs.count() + print('{}: {} % / {} % / {} % ({})'.format(name, 100*group.features.sum()/float(n), 100*group.objects.sum()/float(n), 100*group.interactions.sum()/float(n), n)) + print('-'*80) + print(data) + +################################# # Delete ################################# if args.delete is not None: @@ -119,20 +156,20 @@ if args.process in ['feature', 'object']: # tracking if args.nProcesses == 1: for vs in videoSequences: - if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object': + if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object': if args.configFilename is None: configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) else: configFilename = args.configFilename if vs.cameraView.cameraType is None: cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun) - else: + else: #caution: cameratype can be not none, but without parameters for undistortion cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun) else: print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename())) else: for vs in videoSequences: - if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object': + if not (parentPath/vs.getDatabaseFilename()).is_file() or args.process == 'object': if args.configFilename is None: configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) else: @@ -147,7 +184,7 @@ pool.join() elif args.process == 'prototype': # motion pattern learning - # learn by site by default -> group videos by site (or by camera view? TODO add cameraviews) + # learn by site by default -> group videos by camera view TODO # by default, load all objects, learn and then assign (BUT not save the assignments) for site in sites: print('Learning motion patterns for site {} ({})'.format(site.idx, site.name)) @@ -177,7 +214,6 @@ outputPrototypeDatabaseFilename = args.databaseFilename else: outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename - # TODO maintain mapping from object prototype to db filename + compute nmatchings before clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1) storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices]) @@ -213,46 +249,34 @@ if args.analyze == 'object': # user speeds, accelerations # aggregation per site + if args.eventFilename is None: + print('Missing output filename (event-filename). Exiting') + sys.exit(0) data = [] # list of observation per site-user with time - headers = ['sites', 'date', 'time', 'user_type'] - aggFunctions = {} - for method in args.aggMethod: - if method == 'centile': - aggFunctions[method] = utils.aggregationFunction(method, args.aggCentiles) - for c in args.aggCentiles: - headers.append('{}{}'.format(method,c)) - else: - aggFunctions[method] = utils.aggregationFunction(method) - headers.append(method) - for vs in videoSequences: - d = vs.startTime.date() - t1 = vs.startTime.time() - minUserDuration = args.minUserDuration*vs.cameraView.cameraType.frameRate - print('Extracting speed from '+vs.getDatabaseFilename()) - objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object', args.nObjects) - for o in objects: - if o.length() > minUserDuration: - row = [vs.cameraView.siteIdx, d, utils.framesToTime(o.getFirstInstant(), vs.cameraView.cameraType.frameRate, t1), o.getUserType()] - tmp = o.getSpeeds() - for method,func in aggFunctions.items(): - aggSpeeds = vs.cameraView.cameraType.frameRate*3.6*func(tmp) - if method == 'centile': - row += aggSpeeds.tolist() - else: - row.append(aggSpeeds) - data.append(row) - data = DataFrame(data, columns = headers) + headers = ['site', 'date', 'time', 'user_type'] + aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) + headers.extend(tmpheaders) + if args.nProcesses == 1: + for vs in videoSequences: + data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) + else: + jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences] + for job in jobs: + data.extend(job.get()) + pool.close() + data = pd.DataFrame(data, columns = headers) if args.output == 'figure': for name in headers[4:]: plt.ioff() plt.figure() - plt.boxplot([data.loc[data['sites']==siteId, name] for siteId in siteIds], labels = [session.query(Site).get(siteId).name for siteId in siteIds]) + plt.boxplot([data.loc[data['site']==site.name, name] for site in sites], labels = [site.name for site in sites]) plt.ylabel(name+' Speeds (km/h)') plt.savefig(name.lower()+'-speeds.png', dpi=dpi) plt.close() elif args.output == 'event': - data.to_csv('speeds.csv', index = False) -if args.analyze == 'interaction': + data.to_csv(args.eventFilename, index = False) + +if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data indicatorIds = [2,5,7,10] conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} @@ -282,3 +306,38 @@ plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')') plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150) plt.close() + +if args.analyze == 'event': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds + data = pd.read_csv(args.eventFilename, parse_dates = [2]) + #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000) + # create time for end of each 15 min, then group by, using the agg method for each data column + headers = ['site', 'date', 'intervalend15', 'duration', 'count'] + aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) + dataColumns = list(data.columns[4:]) + for h in dataColumns: + for h2 in tmpheaders: + headers.append(h+'-'+h2) + for h in dataColumns: + for t in args.eventThresholds: + headers.append('n-{}-{}'.format(h, t)) + data['intervalend15'] = data.time.apply(lambda t: (pd.Timestamp(year = t.year, month = t.month, day = t.day,hour = t.hour, minute = (t.minute // args.intervalDuration)*args.intervalDuration)+pd.Timedelta(minutes = 15)).time()) + outputData = [] + for name, group in data.groupby(['site', 'date', 'intervalend15']): + row = [] + row.extend(name) + groupStartTime = group.time.min() + groupEndTime = group.time.max() + row.append((groupEndTime.minute+1-groupStartTime.minute) % 60)#(name[2].minute*60+name[2].second-groupStartTime.minute*60+groupStartTime.second) % 3600) + row.append(len(group)) + for h in dataColumns: + for method,func in aggFunctions.items(): + aggregated = func(group[h]) + if method == 'centile': + row.extend(aggregated) + else: + row.append(aggregated) + for h in dataColumns: + for t in args.eventThresholds: + row.append((group[h] > t).sum()) + outputData.append(row) + pd.DataFrame(outputData, columns = headers).to_csv(utils.removeExtension(args.eventFilename)+'-aggregated.csv', index = False)
--- a/trafficintelligence/metadata.py Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/metadata.py Fri Jul 20 14:03:34 2018 -0400 @@ -366,6 +366,9 @@ 'Returns the site(s) matching the index' return session.query(CameraView).filter(CameraView.idx == int(viewId)).first() +def getSiteVideoSequences(site): + return [vs for cv in site.cameraViews for vs in cv.videoSequences] + def initializeSites(session, directoryName, nViewsPerSite = 1): '''Initializes default site objects and n camera views per site
--- a/trafficintelligence/moving.py Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/moving.py Fri Jul 20 14:03:34 2018 -0400 @@ -33,7 +33,7 @@ self.last=last def __str__(self): - return '[{0}, {1}]'.format(self.first, self.last) + return '{0}-{1}'.format(self.first, self.last) def __repr__(self): return self.__str__() @@ -69,6 +69,15 @@ self.last += offset @classmethod + def parse(cls, s): + if '-' in s: + tmp = s.split('-') + if len(tmp) == 2: + return cls(int(tmp[0]), int(tmp[1])) # TODO with floats? + print(s+' is not a valid representation of an interval') + return None + + @classmethod def union(cls, interval1, interval2): '''Smallest interval comprising self and interval2''' return cls(min(interval1.first, interval2.first), max(interval1.last, interval2.last))
--- a/trafficintelligence/processing.py Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/processing.py Fri Jul 20 14:03:34 2018 -0400 @@ -3,7 +3,7 @@ import numpy as np -from trafficintelligence import ml +from trafficintelligence import ml, storage, utils def extractSpeeds(objects, zone): speeds = {} @@ -18,6 +18,27 @@ objectsNotInZone.append(o) return speeds, objectsNotInZone +def extractVideoSequenceSpeeds(dbFilename, siteName, nObjects, startTime, frameRate, minUserDurationSeconds, aggMethods, aggCentiles): + data = [] + d = startTime.date() + t1 = startTime.time() + minUserDuration = minUserDurationSeconds*frameRate + print('Extracting speed from '+dbFilename) + aggFunctions, tmpheaders = utils.aggregationMethods(aggMethods, aggCentiles) + objects = storage.loadTrajectoriesFromSqlite(dbFilename, 'object', nObjects) + for o in objects: + if o.length() > minUserDuration: + row = [siteName, d, utils.framesToTime(o.getFirstInstant(), frameRate, t1), o.getUserType()] + tmp = o.getSpeeds() + for method,func in aggFunctions.items(): + aggSpeeds = frameRate*3.6*func(tmp) + if method == 'centile': + row.extend(aggSpeeds.tolist()) + else: + row.append(aggSpeeds) + data.append(row) + return data + def learnAssignMotionPatterns(learn, assign, objects, similarities, minSimilarity, similarityFunc, minClusterSize = 0, optimizeCentroid = False, randomInitialization = False, removePrototypesAfterAssignment = False, initialPrototypes = []): '''Learns motion patterns
--- a/trafficintelligence/storage.py Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/storage.py Fri Jul 20 14:03:34 2018 -0400 @@ -70,6 +70,18 @@ except sqlite3.OperationalError as error: printDBError(error) +def tableNames(filename): + 'Lists the names of the tables in the SQLite file' + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + try: + cursor = connection.cursor() + cursor.execute('SELECT name FROM sqlite_master WHERE type = \'table\'') + return [row[0] for row in cursor] + except sqlite3.OperationalError as error: + printDBError(error) + return [] + def createTrajectoryTable(cursor, tableName): if tableName.endswith('positions') or tableName.endswith('velocities'): cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (trajectory_id INTEGER, frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") @@ -259,60 +271,61 @@ The number loaded is either the first objectNumbers objects, or the indices in objectNumbers from the database''' objects = [] - with sqlite3.connect(filename) as connection: - objects = loadTrajectoriesFromTable(connection, 'positions', trajectoryType, objectNumbers, timeStep) - objectVelocities = loadTrajectoriesFromTable(connection, 'velocities', trajectoryType, objectNumbers, timeStep) + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + objects = loadTrajectoriesFromTable(connection, 'positions', trajectoryType, objectNumbers, timeStep) + objectVelocities = loadTrajectoriesFromTable(connection, 'velocities', trajectoryType, objectNumbers, timeStep) - if len(objectVelocities) > 0: - for o,v in zip(objects, objectVelocities): - if o.getNum() == v.getNum(): - o.velocities = v.positions - o.velocities.duplicateLastPosition() # avoid having velocity shorter by one position than positions - else: - print('Could not match positions {0} with velocities {1}'.format(o.getNum(), v.getNum())) + if len(objectVelocities) > 0: + for o,v in zip(objects, objectVelocities): + if o.getNum() == v.getNum(): + o.velocities = v.positions + o.velocities.duplicateLastPosition() # avoid having velocity shorter by one position than positions + else: + print('Could not match positions {0} with velocities {1}'.format(o.getNum(), v.getNum())) - if trajectoryType == 'object': - cursor = connection.cursor() - try: - # attribute feature numbers to objects - queryStatement = 'SELECT trajectory_id, object_id FROM objects_features' - if objectNumbers is not None: - queryStatement += ' WHERE object_id '+getObjectCriteria(objectNumbers) - queryStatement += ' ORDER BY object_id' # order is important to group all features per object - logging.debug(queryStatement) - cursor.execute(queryStatement) + if trajectoryType == 'object': + cursor = connection.cursor() + try: + # attribute feature numbers to objects + queryStatement = 'SELECT trajectory_id, object_id FROM objects_features' + if objectNumbers is not None: + queryStatement += ' WHERE object_id '+getObjectCriteria(objectNumbers) + queryStatement += ' ORDER BY object_id' # order is important to group all features per object + logging.debug(queryStatement) + cursor.execute(queryStatement) - featureNumbers = {} - for row in cursor: - objId = row[1] - if objId not in featureNumbers: - featureNumbers[objId] = [row[0]] - else: - featureNumbers[objId].append(row[0]) + featureNumbers = {} + for row in cursor: + objId = row[1] + if objId not in featureNumbers: + featureNumbers[objId] = [row[0]] + else: + featureNumbers[objId].append(row[0]) - for obj in objects: - obj.featureNumbers = featureNumbers[obj.getNum()] + for obj in objects: + obj.featureNumbers = featureNumbers[obj.getNum()] - # load userType - attributes = loadObjectAttributesFromTable(cursor, objectNumbers, True) - for obj in objects: - userType, nObjects = attributes[obj.getNum()] - obj.setUserType(userType) - obj.setNObjects(nObjects) - - # add features - if withFeatures: + # load userType + attributes = loadObjectAttributesFromTable(cursor, objectNumbers, True) for obj in objects: - obj.features = loadTrajectoriesFromSqlite(filename, 'feature', obj.featureNumbers, timeStep = timeStep) - elif nLongestFeaturesPerObject is not None: - for obj in objects: - queryStatement = 'SELECT trajectory_id, max(frame_number)-min(frame_number) AS length FROM positions WHERE trajectory_id '+getObjectCriteria(obj.featureNumbers)+' GROUP BY trajectory_id ORDER BY length DESC' - logging.debug(queryStatement) - cursor.execute(queryStatement) - obj.features = loadTrajectoriesFromSqlite(filename, 'feature', [row[0] for i,row in enumerate(cursor) if i<nLongestFeaturesPerObject], timeStep = timeStep) + userType, nObjects = attributes[obj.getNum()] + obj.setUserType(userType) + obj.setNObjects(nObjects) - except sqlite3.OperationalError as error: - printDBError(error) + # add features + if withFeatures: + for obj in objects: + obj.features = loadTrajectoriesFromSqlite(filename, 'feature', obj.featureNumbers, timeStep = timeStep) + elif nLongestFeaturesPerObject is not None: + for obj in objects: + queryStatement = 'SELECT trajectory_id, max(frame_number)-min(frame_number) AS length FROM positions WHERE trajectory_id '+getObjectCriteria(obj.featureNumbers)+' GROUP BY trajectory_id ORDER BY length DESC' + logging.debug(queryStatement) + cursor.execute(queryStatement) + obj.features = loadTrajectoriesFromSqlite(filename, 'feature', [row[0] for i,row in enumerate(cursor) if i<nLongestFeaturesPerObject], timeStep = timeStep) + + except sqlite3.OperationalError as error: + printDBError(error) return objects def loadObjectFeatureFrameNumbers(filename, objectNumbers = None): @@ -451,19 +464,20 @@ Load descriptions?''' objects = [] - with sqlite3.connect(filename) as connection: - if objectType == 'bb': - topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep) - bottomCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbbottom', objectNumbers, timeStep) - userTypes = loadObjectAttributesFromTable(connection.cursor(), objectNumbers) # string format is same as object + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + if objectType == 'bb': + topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep) + bottomCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbbottom', objectNumbers, timeStep) + userTypes = loadObjectAttributesFromTable(connection.cursor(), objectNumbers) # string format is same as object - for t, b in zip(topCorners, bottomCorners): - num = t.getNum() - if t.getNum() == b.getNum(): - annotation = moving.BBMovingObject(num, t.getTimeInterval(), t, b, userTypes[num]) - objects.append(annotation) - else: - print ('Unknown type of bounding box {}'.format(objectType)) + for t, b in zip(topCorners, bottomCorners): + num = t.getNum() + if t.getNum() == b.getNum(): + annotation = moving.BBMovingObject(num, t.getTimeInterval(), t, b, userTypes[num]) + objects.append(annotation) + else: + print ('Unknown type of bounding box {}'.format(objectType)) return objects def saveInteraction(cursor, interaction): @@ -509,29 +523,30 @@ TODO choose the interactions to load''' interactions = [] - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('SELECT INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND WHERE INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type, IND.frame_number') - interactionNum = -1 - indicatorTypeNum = -1 - tmpIndicators = {} - for row in cursor: - if row[0] != interactionNum: - interactionNum = row[0] - interactions.append(events.Interaction(interactionNum, moving.TimeInterval(row[3],row[4]), row[1], row[2])) - interactions[-1].indicators = {} - if indicatorTypeNum != row[5] or row[0] != interactionNum: - indicatorTypeNum = row[5] - indicatorName = events.Interaction.indicatorNames[indicatorTypeNum] - indicatorValues = {row[6]:row[7]} - interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues, mostSevereIsMax = not indicatorName in events.Interaction.timeIndicators) - else: - indicatorValues[row[6]] = row[7] - interactions[-1].indicators[indicatorName].timeInterval.last = row[6] - except sqlite3.OperationalError as error: - printDBError(error) - return [] + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('SELECT INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND WHERE INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type, IND.frame_number') + interactionNum = -1 + indicatorTypeNum = -1 + tmpIndicators = {} + for row in cursor: + if row[0] != interactionNum: + interactionNum = row[0] + interactions.append(events.Interaction(interactionNum, moving.TimeInterval(row[3],row[4]), row[1], row[2])) + interactions[-1].indicators = {} + if indicatorTypeNum != row[5] or row[0] != interactionNum: + indicatorTypeNum = row[5] + indicatorName = events.Interaction.indicatorNames[indicatorTypeNum] + indicatorValues = {row[6]:row[7]} + interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues, mostSevereIsMax = not indicatorName in events.Interaction.timeIndicators) + else: + indicatorValues[row[6]] = row[7] + interactions[-1].indicators[indicatorName].timeInterval.last = row[6] + except sqlite3.OperationalError as error: + printDBError(error) + return [] return interactions # load first and last object instants # CREATE TEMP TABLE IF NOT EXISTS object_instants AS SELECT OF.object_id, min(frame_number) as first_instant, max(frame_number) as last_instant from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id group by OF.object_id order by OF.object_id @@ -554,18 +569,18 @@ def loadBoundingBoxTableForDisplay(filename): '''Loads bounding boxes from bounding_boxes table for display over trajectories''' boundingBoxes = {} # list of bounding boxes for each instant - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('SELECT name FROM sqlite_master WHERE type=\'table\' AND name=\'bounding_boxes\'') - result = cursor.fetchall() - if len(result) > 0: - cursor.execute('SELECT * FROM bounding_boxes') - for row in cursor: - boundingBoxes.setdefault(row[1], []).append([moving.Point(row[2], row[3]), moving.Point(row[4], row[5])]) - except sqlite3.OperationalError as error: - printDBError(error) - return boundingBoxes + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('SELECT name FROM sqlite_master WHERE type=\'table\' AND name=\'bounding_boxes\'') + result = cursor.fetchall() + if len(result) > 0: + cursor.execute('SELECT * FROM bounding_boxes') + for row in cursor: + boundingBoxes.setdefault(row[1], []).append([moving.Point(row[2], row[3]), moving.Point(row[4], row[5])]) + except sqlite3.OperationalError as error: + printDBError(error) return boundingBoxes ######################### @@ -624,49 +639,52 @@ connection.commit() def loadPrototypeAssignmentsFromSqlite(filename, objectType): - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - tableName, objectIdColumnName = prototypeAssignmentNames(objectType) - cursor.execute('SELECT * FROM '+tableName) - prototypeAssignments = {} - for row in cursor: - p = moving.Prototype(row[1], row[2], row[3]) - if p in prototypeAssignments: - prototypeAssignments[p].append(row[0]) - else: - prototypeAssignments[p] = [row[0]] - return prototypeAssignments - except sqlite3.OperationalError as error: - printDBError(error) - + prototypeAssignments = {} + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + tableName, objectIdColumnName = prototypeAssignmentNames(objectType) + cursor.execute('SELECT * FROM '+tableName) + for row in cursor: + p = moving.Prototype(row[1], row[2], row[3]) + if p in prototypeAssignments: + prototypeAssignments[p].append(row[0]) + else: + prototypeAssignments[p] = [row[0]] + return prototypeAssignments + except sqlite3.OperationalError as error: + printDBError(error) + return prototypeAssignments + def loadPrototypesFromSqlite(filename, withTrajectories = True): 'Loads prototype ids and matchings (if stored)' prototypes = [] - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - objects = [] - try: - cursor.execute('SELECT * FROM prototypes') - for row in cursor: - prototypes.append(moving.Prototype(row[0], row[1], row[2], row[3])) - if withTrajectories: - for p in prototypes: - p.setMovingObject(loadTrajectoriesFromSqlite(p.getFilename(), p.getTrajectoryType(), [p.getNum()])[0]) - # loadingInformation = {} # complicated slightly optimized - # for p in prototypes: - # dbfn = p.getFilename() - # trajType = p.getTrajectoryType() - # if (dbfn, trajType) in loadingInformation: - # loadingInformation[(dbfn, trajType)].append(p) - # else: - # loadingInformation[(dbfn, trajType)] = [p] - # for k, v in loadingInformation.iteritems(): - # objects += loadTrajectoriesFromSqlite(k[0], k[1], [p.getNum() for p in v]) - except sqlite3.OperationalError as error: - printDBError(error) - if len(set([p.getTrajectoryType() for p in prototypes])) > 1: - print('Different types of prototypes in database ({}).'.format(set([p.getTrajectoryType() for p in prototypes]))) + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + objects = [] + try: + cursor.execute('SELECT * FROM prototypes') + for row in cursor: + prototypes.append(moving.Prototype(row[0], row[1], row[2], row[3])) + if withTrajectories: + for p in prototypes: + p.setMovingObject(loadTrajectoriesFromSqlite(p.getFilename(), p.getTrajectoryType(), [p.getNum()])[0]) + # loadingInformation = {} # complicated slightly optimized + # for p in prototypes: + # dbfn = p.getFilename() + # trajType = p.getTrajectoryType() + # if (dbfn, trajType) in loadingInformation: + # loadingInformation[(dbfn, trajType)].append(p) + # else: + # loadingInformation[(dbfn, trajType)] = [p] + # for k, v in loadingInformation.iteritems(): + # objects += loadTrajectoriesFromSqlite(k[0], k[1], [p.getNum() for p in v]) + except sqlite3.OperationalError as error: + printDBError(error) + if len(set([p.getTrajectoryType() for p in prototypes])) > 1: + print('Different types of prototypes in database ({}).'.format(set([p.getTrajectoryType() for p in prototypes]))) return prototypes def savePOIsToSqlite(filename, gmm, gmmType, gmmId): @@ -703,46 +721,47 @@ from sklearn import mixture # todo if not avalaible, load data in duck-typed class with same fields from ast import literal_eval pois = [] - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('SELECT * from gaussians2d') - gmmId = None - gmm = [] - for row in cursor: - if gmmId is None or row[0] != gmmId: - if len(gmm) > 0: - tmp = mixture.GaussianMixture(len(gmm), covarianceType) - tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) - tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) - tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) - tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] - tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) - pois.append(tmp) - gaussian = {'type': row[2], - 'mean': row[3:5], - 'covar': array(literal_eval(row[5])), - 'weight': row[7], - 'precisions': array(literal_eval(row[8]))} - gmm = [gaussian] - covarianceType = row[6] - gmmId = row[0] - else: - gmm.append({'type': row[2], - 'mean': row[3:5], - 'covar': array(literal_eval(row[5])), - 'weight': row[7], - 'precisions': array(literal_eval(row[8]))}) - if len(gmm) > 0: - tmp = mixture.GaussianMixture(len(gmm), covarianceType) - tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) - tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) - tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) - tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] - tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) - pois.append(tmp) - except sqlite3.OperationalError as error: - printDBError(error) + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('SELECT * from gaussians2d') + gmmId = None + gmm = [] + for row in cursor: + if gmmId is None or row[0] != gmmId: + if len(gmm) > 0: + tmp = mixture.GaussianMixture(len(gmm), covarianceType) + tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) + tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) + tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) + tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] + tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) + pois.append(tmp) + gaussian = {'type': row[2], + 'mean': row[3:5], + 'covar': array(literal_eval(row[5])), + 'weight': row[7], + 'precisions': array(literal_eval(row[8]))} + gmm = [gaussian] + covarianceType = row[6] + gmmId = row[0] + else: + gmm.append({'type': row[2], + 'mean': row[3:5], + 'covar': array(literal_eval(row[5])), + 'weight': row[7], + 'precisions': array(literal_eval(row[8]))}) + if len(gmm) > 0: + tmp = mixture.GaussianMixture(len(gmm), covarianceType) + tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) + tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) + tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) + tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] + tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) + pois.append(tmp) + except sqlite3.OperationalError as error: + printDBError(error) return pois ######################### @@ -1326,7 +1345,8 @@ self.stdVehicleSpeed = config.getfloat(self.sectionHeader, 'std-veh-speed') def __init__(self, filename = None): - if filename is not None and Path(filename).exists(): + self.configFilename = filename + if filename is not None and Path(filename).is_file(): self.loadConfigFile(filename) else: print('Configuration filename {} could not be loaded.'.format(filename)) @@ -1386,12 +1406,12 @@ self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) self.databaseFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'database-filename')) self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename')) - if Path(self.homographyFilename).exists(): + if Path(self.homographyFilename).is_file(): self.homography = loadtxt(self.homographyFilename) else: self.homography = None self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename')) - if Path(self.intrinsicCameraFilename).exists(): + if Path(self.intrinsicCameraFilename).is_file(): self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename) else: self.intrinsicCameraMatrix = None @@ -1444,11 +1464,11 @@ def __init__(self, filename = None): - if filename is not None and Path(filename).exists(): + self.configFilename = filename + if filename is not None and Path(filename).is_file(): self.loadConfigFile(filename) else: print('Configuration filename {} could not be loaded.'.format(filename)) - self.configFilename = filename def processVideoArguments(args): '''Loads information from configuration file
--- a/trafficintelligence/tests/moving.txt Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/tests/moving.txt Fri Jul 20 14:03:34 2018 -0400 @@ -7,7 +7,7 @@ >>> Interval(0,1).empty() False >>> Interval(0,1) -[0, 1] +0-1 >>> Interval(0,1).length() 1.0 >>> Interval(23.2,24.9).length() @@ -15,6 +15,17 @@ >>> Interval(10,8).length() 0.0 +>>> i = Interval.parse('3-5') +>>> i.first == 3 and i.last == 5 +True +>>> type(i) +<class 'trafficintelligence.moving.Interval'> +>>> i = TimeInterval.parse('3-5') +>>> type(i) +<class 'trafficintelligence.moving.TimeInterval'> +>>> list(i) +[3, 4, 5] + >>> TimeInterval(0,1).length() 2.0 >>> TimeInterval(10,8).length() @@ -45,7 +56,7 @@ >>> TimeInterval(20,30).distance(TimeInterval(3,15)) 5 >>> TimeInterval.unionIntervals([TimeInterval(3,6), TimeInterval(8,10),TimeInterval(11,15)]) -[3, 15] +3-15 >>> Point(0,3) == Point(0,3) True
--- a/trafficintelligence/tests/storage.txt Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/tests/storage.txt Fri Jul 20 14:03:34 2018 -0400 @@ -1,4 +1,5 @@ >>> from io import StringIO +>>> from os import remove >>> from trafficintelligence.storage import * >>> from trafficintelligence.utils import openCheck, readline >>> from trafficintelligence.moving import MovingObject, Point, TimeInterval, Trajectory, prepareSplines @@ -8,11 +9,7 @@ >>> nonexistentFilename = "nonexistent" >>> loadTrajectoriesFromSqlite(nonexistentFilename, 'feature') -DB Error: no such table: positions -DB Error: no such table: velocities [] ->>> from os import remove ->>> remove(nonexistentFilename) >>> o1 = MovingObject.generate(2, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) >>> o2 = MovingObject.generate(3, Point(1.,1.), Point(-0.5,-0.2), TimeInterval(0,9)) @@ -48,6 +45,8 @@ 4 >>> objects[1].positions.length() 4 +>>> remove('test.sqlite') + >>> align1 = Trajectory.fromPointList([Point(-1, 0), Point(20, 0)]) >>> align2 = Trajectory.fromPointList([Point(-9, -3), Point(6, 3)]) >>> align1.computeCumulativeDistances()
--- a/trafficintelligence/utils.py Fri Jul 20 13:50:43 2018 -0400 +++ b/trafficintelligence/utils.py Fri Jul 20 14:03:34 2018 -0400 @@ -342,6 +342,10 @@ def timeToFrames(t, frameRate): return frameRate*(t.hour*3600+t.minute*60+t.second) +def timeModulo(t, duration): + 'returns the time modulo the duration in min' + return time(t.hour, t.minute//duration, t.second) + def sortXY(X,Y): 'returns the sorted (x, Y(x)) sorted on X' D = {} @@ -591,6 +595,19 @@ print('Unknown aggregation method: {}'.format(funcStr)) return None +def aggregationMethods(methods, centiles = None): + aggFunctions = {} + headers = [] + for method in methods: + if method == 'centile': + aggFunctions[method] = aggregationFunction(method, centiles) + for c in centiles: + headers.append('{}{}'.format(method,c)) + else: + aggFunctions[method] = aggregationFunction(method) + headers.append(method) + return aggFunctions, headers + ######################### # regression analysis using statsmodels (and pandas) #########################