Mercurial Hosting > traffic-intelligence

changeset 564:36605d843be5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
modified bug for reading vissim files, cleaned use of readline with multiple type of characters for comments (to ignore)
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Mon, 21 Jul 2014 16:17:22 -0400
parents 39de5c532559
children f86f5f25730a
files python/storage.py python/tests/storage.txt
diffstat 2 files changed, 55 insertions(+), 36 deletions(-) [+]
line wrap: on
line diff
--- a/python/storage.py	Sat Jul 19 23:44:39 2014 -0400
+++ b/python/storage.py	Mon Jul 21 16:17:22 2014 -0400
@@ -35,15 +35,15 @@
         printDBError(error)
 
 # IO to sqlite
-def writeTrajectoriesToSqlite(objects, outFilename, trajectoryType, objectNumbers = -1):
+def writeTrajectoriesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1):
     """
     This function writers trajectories to a specified sqlite file
     @param[in] objects -> a list of trajectories
     @param[in] trajectoryType -
-    @param[out] outFile -> the .sqlite file containting the written objects
+    @param[out] outputFilename -> the .sqlite file containting the written objects
     @param[in] objectNumber : number of objects loaded
     """
-    connection = sqlite3.connect(outFilename)
+    connection = sqlite3.connect(outputFilename)
     cursor = connection.cursor()
 
     schema = "CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))"
@@ -64,9 +64,9 @@
     connection.commit()
     connection.close()
 	
-def writeFeaturesToSqlite(objects, outFilename, trajectoryType, objectNumbers = -1):
+def writeFeaturesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1):
     '''write features trajectories maintain trajectory ID,velocities dataset  '''
-    connection = sqlite3.connect(outFilename)
+    connection = sqlite3.connect(outputFilename)
     cursor = connection.cursor()
 
     cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))")
@@ -85,9 +85,9 @@
     connection.commit()
     connection.close()
 	
-def writePrototypesToSqlite(prototypes,nMatching, outFilename):
+def writePrototypesToSqlite(prototypes,nMatching, outputFilename):
     """ prototype dataset is a dictionary with  keys== routes, values== prototypes Ids """
-    connection = sqlite3.connect(outFilename)
+    connection = sqlite3.connect(outputFilename)
     cursor = connection.cursor()
 
     cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))")
@@ -127,10 +127,10 @@
     connection.close()
     return prototypes,nMatching
 	
-def writeLabelsToSqlite(labels, outFilename):
+def writeLabelsToSqlite(labels, outputFilename):
     """ labels is a dictionary with  keys: routes, values: prototypes Ids
     """
-    connection = sqlite3.connect(outFilename)
+    connection = sqlite3.connect(outputFilename)
     cursor = connection.cursor()
 
     cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))")
@@ -168,9 +168,9 @@
     connection.close()
     return labels
 
-def writeRoutesToSqlite(Routes, outFilename):
+def writeRoutesToSqlite(Routes, outputFilename):
     """ This function writes the activity path define by start and end IDs"""
-    connection = sqlite3.connect(outFilename)
+    connection = sqlite3.connect(outputFilename)
     cursor = connection.cursor()
 
     cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))")
@@ -517,20 +517,21 @@
             exit()
         return None
 
-def readline(f, commentChar = commentChar):
-    '''Modified readline function to skip comments.'''
+def readline(f, commentCharacters = commentChar):
+    '''Modified readline function to skip comments
+    Can take a list of characters or a string (in will work in both)'''
     s = f.readline()
-    while (len(s) > 0) and s.startswith(commentChar):
+    while (len(s) > 0) and s[0] in commentCharacters:
         s = f.readline()
     return s.strip()
 
-def getLines(f, commentChar = commentChar):
+def getLines(f, commentCharacters = commentChar):
     '''Gets a complete entry (all the lines) in between delimiterChar.'''
     dataStrings = []
-    s = readline(f, commentChar)
+    s = readline(f, commentCharacters)
     while len(s) > 0:
         dataStrings += [s.strip()]
-        s = readline(f, commentChar)
+        s = readline(f, commentCharacters)
     return dataStrings
 
 def writeList(filename, l):
@@ -539,15 +540,15 @@
         f.write('{}\n'.format(x))
     f.close()
 
-def loadListStrings(filename, commentChar = commentChar):
+def loadListStrings(filename, commentCharacters = commentChar):
     f = openCheck(filename, 'r')
-    result = getLines(f, commentChar)
+    result = getLines(f, commentCharacters)
     f.close()
     return result
 
-def getValuesFromINIFile(filename, option, delimiterChar = '=', commentChar = commentChar):
+def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar):
     values = []
-    for l in loadListStrings(filename, commentChar):
+    for l in loadListStrings(filename, commentCharacters):
         if l.startswith(option):
             values.append(l.split(delimiterChar)[1].strip())
     return values
@@ -579,14 +580,12 @@
     objects = {} # dictionary of objects index by their id
     firstInstants = {}
 
-    infile = openCheck(filename, quitting = True)
+    inputfile = openCheck(filename, quitting = True)
 
     # data = pd.read_csv(filename, skiprows=15, delimiter=';')
     # skip header: 15 lines + 1
-    for i in range(16):
-        readline(infile)
-
-    for line in infile:
+    line = readline(inputfile, '*$')
+    while len(line) > 0:#for line in inputfile:
         data = line.strip().split(';')
         objNum = int(data[1])
         instant = int(float(data[0])*simulationStepsPerTimeUnit)
@@ -602,6 +601,7 @@
         if (warmUpLastInstant == None or firstInstants[objNum] >= warmUpLastInstant) and objNum in objects:
             objects[objNum].timeInterval.last = instant
             objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane)
+        line = readline(inputfile, '*$')
 
     return objects.values()
     
@@ -610,7 +610,7 @@
     and returns the list of Feature objects'''
     objects = []
 
-    infile = openCheck(filename, quitting = True)
+    inputfile = openCheck(filename, quitting = True)
 
     def createObject(numbers):
         firstFrameNum = int(numbers[1])
@@ -634,11 +634,11 @@
         obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object
         return obj
 
-    numbers = infile.readline().strip().split()
+    numbers = readline(inputfile).strip().split()
     if (len(numbers) > 0):
         obj = createObject(numbers)
 
-    for line in infile:
+    for line in inputfile:
         numbers = line.strip().split()
         if obj.getNum() != int(numbers[0]):
             # check and adapt the length to deal with issues in NGSIM data
@@ -665,19 +665,19 @@
             if (obj.size[1] != float(numbers[9])):
                 print 'changed width obj %d' % (obj.getNum())
     
-    infile.close()
+    inputfile.close()
     return objects
 
-def convertNgsimFile(inFile, outFile, append = False, nObjects = -1, sequenceNum = 0):
+def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0):
     '''Reads data from the trajectory data provided by NGSIM project
     and converts to our current format.'''
     if append:
-        out = openCheck(outFile,'a')
+        out = openCheck(outputfile,'a')
     else:
-        out = openCheck(outFile,'w')
+        out = openCheck(outputfile,'w')
     nObjectsPerType = [0,0,0]
 
-    features = loadNgsimFile(inFile, sequenceNum)
+    features = loadNgsimFile(inputfile, sequenceNum)
     for f in features:
         nObjectsPerType[f.userType-1] += 1
         f.write(out)
--- a/python/tests/storage.txt	Sat Jul 19 23:44:39 2014 -0400
+++ b/python/tests/storage.txt	Mon Jul 21 16:17:22 2014 -0400
@@ -1,12 +1,31 @@
 >>> from storage import *
+>>> from StringIO import StringIO
 
 >>> f = openCheck('non_existant_file.txt')
 File non_existant_file.txt could not be opened.
 
->>> loadPrototypeMatchIndexesFromSqlite("nonexistent")
+>>> nonexistentFilename = "nonexistent"
+>>> loadPrototypeMatchIndexesFromSqlite(nonexistentFilename)
 DB Error: no such table: prototypes
 []
->>> loadTrajectoriesFromSqlite("nonexistent", 'feature')
+>>> loadTrajectoriesFromSqlite(nonexistentFilename, 'feature')
 DB Error: no such table: positions
 DB Error: no such table: velocities
 []
+>>> from os import remove
+>>> remove(nonexistentFilename)
+
+>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf')
+>>> readline(strio)
+'sadlkfjsdlakjf'
+>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf')
+>>> readline(strio, ['#'])
+'sadlkfjsdlakjf'
+>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf')
+>>> readline(strio, ['%'])
+'# asdlfjasdlkj0'
+>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf')
+>>> readline(strio, '%*$')
+'# asdlfjasdlkj0'
+>>> readline(strio, '%#')
+'sadlkfjsdlakjf'