--- a/python/storage.py	Wed Jul 22 14:17:19 2015 -0400
+++ b/python/storage.py	Wed Jul 22 14:17:44 2015 -0400
@@ -6,6 +6,7 @@
 from base import VideoFilenameAddable
 
 import sqlite3, logging
+from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt
 
 
 commentChar = '#'
@@ -289,22 +290,16 @@
     connection.close()
     return matched_indexes
 
-def getTrajectoryIdQuery(objectNumbers, trajectoryType):
-    if trajectoryType == 'feature':
-        statementBeginning = 'where trajectory_id '
-    elif trajectoryType == 'object':
-        statementBeginning = 'and OF.object_id '
-    elif trajectoryType == 'bbtop' or 'bbbottom':
-        statementBeginning = 'where object_id '
-    else:
-        print('no trajectory type was chosen')
-
+def getObjectCriteria(objectNumbers):
     if objectNumbers is None:
         query = ''
     elif type(objectNumbers) == int:
-        query = statementBeginning+'between 0 and {0} '.format(objectNumbers)
+        query = 'between 0 and {0}'.format(objectNumbers-1)
     elif type(objectNumbers) == list:
-        query = statementBeginning+'in ('+', '.join([str(n) for n in objectNumbers])+') '
+        query = 'in ('+', '.join([str(n) for n in objectNumbers])+')'
+    else:
+        print('objectNumbers {} are not a known type ({})'.format(objectNumbers, type(objectNumbers)))
+        query = ''
     return query
 
 def loadTrajectoriesFromTable(connection, tableName, trajectoryType, objectNumbers = None):
@@ -315,13 +310,19 @@
     cursor = connection.cursor()
 
     try:
-        idQuery = getTrajectoryIdQuery(objectNumbers, trajectoryType)
+        objectCriteria = getObjectCriteria(objectNumbers)
         if trajectoryType == 'feature':
-            queryStatement = 'SELECT * from '+tableName+' '+idQuery+'ORDER BY trajectory_id, frame_number'
+            queryStatement = 'SELECT * from '+tableName
+            if objectNumbers is not None:
+                queryStatement += ' where trajectory_id '+objectCriteria
+            queryStatement += ' ORDER BY trajectory_id, frame_number'
             cursor.execute(queryStatement)
             logging.debug(queryStatement)
         elif trajectoryType == 'object':
-            queryStatement = 'SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from '+tableName+' P, objects_features OF where P.trajectory_id = OF.trajectory_id '+idQuery+'group by OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number'
+            queryStatement = 'SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from '+tableName+' P, objects_features OF where P.trajectory_id = OF.trajectory_id'
+            if objectNumbers is not None:
+                queryStatement += ' and OF.object_id '+objectCriteria
+            queryStatement += ' group by OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number'
             cursor.execute(queryStatement)
             logging.debug(queryStatement)
         elif trajectoryType in ['bbtop', 'bbbottom']:
@@ -329,7 +330,10 @@
                 corner = 'top_left'
             elif trajectoryType == 'bbbottom':
                 corner = 'bottom_right'
-            queryStatement = 'SELECT object_id, frame_number, x_'+corner+', y_'+corner+' FROM '+tableName+' '+idQuery+'ORDER BY object_id, frame_number'
+            queryStatement = 'SELECT object_id, frame_number, x_'+corner+', y_'+corner+' FROM '+tableName
+            if objectNumbers is not None:
+                queryStatement += ' where object_id '+objectCriteria
+            queryStatement += ' ORDER BY object_id, frame_number'
             cursor.execute(queryStatement)
             logging.debug(queryStatement)
         else:
@@ -361,17 +365,17 @@
     return objects
 
 def loadUserTypesFromTable(cursor, trajectoryType, objectNumbers):
-    objectIdQuery = getTrajectoryIdQuery(objectNumbers, trajectoryType)
-    if objectIdQuery == '':
-        cursor.execute('SELECT object_id, road_user_type from objects')
-    else:
-        cursor.execute('SELECT object_id, road_user_type from objects where '+objectIdQuery[7:])
+    objectCriteria = getObjectCriteria(objectNumbers)
+    queryStatement = 'SELECT object_id, road_user_type from objects'
+    if objectNumbers is not None:
+        queryStatement += ' where object_id '+objectCriteria
+    cursor.execute(queryStatement)
     userTypes = {}
     for row in cursor:
         userTypes[row[0]] = row[1]
     return userTypes
 
-def loadTrajectoriesFromSqlite(filename, trajectoryType, objectNumbers = None):
+def loadTrajectoriesFromSqlite(filename, trajectoryType, objectNumbers = None, withFeatures = False):
     '''Loads the first objectNumbers objects or the indices in objectNumbers from the database'''
     connection = sqlite3.connect(filename)
 
@@ -390,8 +394,11 @@
         cursor = connection.cursor()
         try:
             # attribute feature numbers to objects
-            objectIdQuery = getTrajectoryIdQuery(objectNumbers, trajectoryType)
-            queryStatement = 'SELECT P.trajectory_id, OF.object_id from positions P, objects_features OF where P.trajectory_id = OF.trajectory_id '+objectIdQuery+'group by P.trajectory_id order by OF.object_id' # order is important to group all features per object
+            objectCriteria = getObjectCriteria(objectNumbers)
+            queryStatement = 'SELECT P.trajectory_id, OF.object_id from positions P, objects_features OF where P.trajectory_id = OF.trajectory_id'
+            if objectNumbers is not None:
+                queryStatement += ' and OF.object_id '+objectCriteria
+            queryStatement += ' group by P.trajectory_id order by OF.object_id' # order is important to group all features per object
             cursor.execute(queryStatement) 
             logging.debug(queryStatement)
 
@@ -410,6 +417,14 @@
             userTypes = loadUserTypesFromTable(cursor, trajectoryType, objectNumbers)
             for obj in objects:
                 obj.userType = userTypes[obj.getNum()]
+
+            if withFeatures:
+                nFeatures = 0
+                for obj in objects:
+                    nFeatures = max(nFeatures, max(obj.featureNumbers))
+                features = loadTrajectoriesFromSqlite(filename, 'feature', nFeatures+1)
+                for obj in objects:
+                    obj.setFeatures(features)
              
         except sqlite3.OperationalError as error:
             printDBError(error)
@@ -512,7 +527,7 @@
     connection = sqlite3.connect(filename)
     cursor = connection.cursor()
     try:
-        cursor.execute('select INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND where INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type')
+        cursor.execute('select INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND where INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type, IND.frame_number')
         interactionNum = -1
         indicatorTypeNum = -1
         tmpIndicators = {}
@@ -521,13 +536,14 @@
                 interactionNum = row[0]
                 interactions.append(events.Interaction(interactionNum, moving.TimeInterval(row[3],row[4]), row[1], row[2]))
                 interactions[-1].indicators = {}
-            if indicatorTypeNum != row[5]:
+            if indicatorTypeNum != row[5] or row[0] != interactionNum:
+                indicatorTypeNum = row[5]
                 indicatorName = events.Interaction.indicatorNames[indicatorTypeNum]
                 indicatorValues = {row[6]:row[7]}
-                interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues)
-                indicatorTypeNum = row[5]
+                interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues, mostSevereIsMax = not indicatorName in events.Interaction.timeIndicators)
             else:
                 indicatorValues[row[6]] = row[7]
+                interactions[-1].indicators[indicatorName].timeInterval.last = row[6]
     except sqlite3.OperationalError as error:
         printDBError(error)
         return []
@@ -656,20 +672,21 @@
 
     if usePandas:
         from pandas import read_csv
-        from numpy import min, max, round
         data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1)
         generatePDLaneColumn(data)
         data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit
         if warmUpLastInstant is not None:
             data = data[data['TIME']>=warmUpLastInstant]
         grouped = data.loc[:,['NO','TIME']].groupby(['NO'], as_index = False)
-        instants = grouped['TIME'].agg({'first': min, 'last': max})
+        instants = grouped['TIME'].agg({'first': npmin, 'last': npmax})
         for row_index, row in instants.iterrows():
             objNum = int(row['NO'])
             tmp = data[data['NO'] == objNum]
             objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last']))
             # positions should be rounded to nDecimals decimals only
-            objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = round(tmp['POS'].tolist(), nDecimals), Y = round(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist())
+            objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist())
+            if nObjects > 0 and len(objects) >= nObjects:
+                break
         return objects.values()
     else:
         inputfile = openCheck(filename, quitting = True)
@@ -718,7 +735,6 @@
     If lanes is not None, only the data for the selected lanes will be provided
     (format as string x_y where x is link index and y is lane index)'''
     from pandas import read_csv
-    from numpy import array, sum as npsum
     columns = ['NO', '$VEHICLE:SIMSEC', 'POS']
     if lanes is not None:
         columns += ['LANE\LINK\NO', 'LANE\INDEX']
@@ -727,7 +743,6 @@
     data.sort(['$VEHICLE:SIMSEC'], inplace = True)
 
     nStationary = 0
-    from matplotlib.pyplot import plot, figure
     nVehicles = 0
     for name, group in data.groupby(['NO'], sort = False):
         nVehicles += 1
@@ -754,8 +769,9 @@
 
     nCollisions = 0
     for name, group in merged.groupby(['LANE\LINK\NO', 'LANE\INDEX', 'NO_x', 'NO_y']):
-        diff = group['POS_x']-group['POS_y']
-        if len(diff) >= 2 and min(diff) < 0 and max(diff) > 0:
+        diff = group['POS_x'].convert_objects(convert_numeric=True)-group['POS_y'].convert_objects(convert_numeric=True)
+        # diff = group['POS_x']-group['POS_y'] # to check the impact of convert_objects and the possibility of using type conversion in read_csv or function to convert strings if any
+        if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0:
             xidx = diff[diff < 0].argmax()
             yidx = diff[diff > 0].argmin()
             if abs(group.loc[xidx, '$VEHICLE:SIMSEC'] - group.loc[yidx, '$VEHICLE:SIMSEC']) <= collisionTimeDifference:
@@ -874,7 +890,6 @@
 
     def loadConfigFile(self, filename):
         from ConfigParser import ConfigParser
-        from numpy import loadtxt
         from os import path
 
         config = ConfigParser()
@@ -901,7 +916,21 @@
         self.videoFrameRate = config.getfloat(self.sectionHeader, 'video-fps')
 
         # Classification parameters
-        
+        self.speedAggregationMethod = config.get(self.sectionHeader, 'speed-aggregation-method')
+        self.nFramesIgnoreAtEnds = config.getint(self.sectionHeader, 'nframes-ignore-at-ends')
+        self.speedAggregationQuantile = config.getint(self.sectionHeader, 'speed-aggregation-quantile')
+        self.minSpeedEquiprobable = config.getfloat(self.sectionHeader, 'min-speed-equiprobable')
+        self.minNPixels = config.getint(self.sectionHeader, 'min-npixels-crop')
+        self.pedBikeCarSVMFilename = config.get(self.sectionHeader, 'pbv-svm-filename')
+        self.bikeCarSVMFilename = config.get(self.sectionHeader, 'bv-svm-filename')
+        self.maxPedestrianSpeed = config.getfloat(self.sectionHeader, 'max-ped-speed')
+        self.maxCyclistSpeed = config.getfloat(self.sectionHeader, 'max-cyc-speed')
+        self.meanPedestrianSpeed = config.getfloat(self.sectionHeader, 'mean-ped-speed')
+        self.stdPedestrianSpeed = config.getfloat(self.sectionHeader, 'std-ped-speed')
+        self.locationCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-loc')
+        self.scaleCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-scale')
+        self.meanVehicleSpeed = config.getfloat(self.sectionHeader, 'mean-veh-speed')
+        self.stdVehicleSpeed = config.getfloat(self.sectionHeader, 'std-veh-speed')
 
         # Safety parameters
         self.maxPredictedSpeed = config.getfloat(self.sectionHeader, 'max-predicted-speed')/3.6/self.videoFrameRate
@@ -921,6 +950,26 @@
         if filename is not None:
             self.loadConfigFile(filename)
 
+    def convertToFrames(self, speedRatio = 3.6):
+        '''Converts parameters with a relationship to time in 'native' frame time
+        speedRatio is the conversion from the speed unit in the config file
+        to the distance per second
+
+        ie param(config file) = speedRatio x fps x param(used in program)
+        eg km/h = 3.6 (m/s to km/h) x frame/s x m/frame'''
+        denominator = self.videoFrameRate*speedRatio
+        denominator2 = denominator**2
+        self.minSpeedEquiprobable = self.minSpeedEquiprobable/denominator
+        self.maxPedestrianSpeed = self.maxPedestrianSpeed/denominator
+        self.maxCyclistSpeed = self.maxCyclistSpeed/denominator
+        self.meanPedestrianSpeed = self.meanPedestrianSpeed/denominator
+        self.stdPedestrianSpeed = self.stdPedestrianSpeed/denominator
+        self.meanVehicleSpeed = self.meanVehicleSpeed/denominator
+        self.stdVehicleSpeed = self.stdVehicleSpeed/denominator
+        # special case for the lognormal distribution
+        self.locationCyclistSpeed = self.locationCyclistSpeed-log(denominator)
+        #self.scaleCyclistSpeed = self.scaleCyclistSpeed
+
 class SceneParameters(object):
     def __init__(self, config, sectionName):
         from ConfigParser import NoOptionError