changeset 718:2cd245cb780d

added option to set low_memory = False for pandas.read_csv
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Thu, 30 Jul 2015 17:43:14 -0400
parents 9d6cd4e8dca3
children ea78c326b3a0
files python/storage.py
diffstat 1 files changed, 4 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/python/storage.py	Sat Jul 25 23:29:11 2015 -0400
+++ b/python/storage.py	Thu Jul 30 17:43:14 2015 -0400
@@ -701,7 +701,7 @@
 def generatePDLaneColumn(data):
     data['LANE'] = data['LANE\LINK\NO'].astype(str)+'_'+data['LANE\INDEX'].astype(str)
 
-def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, nObjects = -1, warmUpLastInstant = None, usePandas = False, nDecimals = 2):
+def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, nObjects = -1, warmUpLastInstant = None, usePandas = False, nDecimals = 2, lowMemory = True):
     '''Reads data from VISSIM .fzp trajectory file
     simulationStepsPerTimeUnit is the number of simulation steps per unit of time used by VISSIM
     for example, there seems to be 5 simulation steps per simulated second in VISSIM, 
@@ -716,7 +716,7 @@
 
     if usePandas:
         from pandas import read_csv
-        data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1)
+        data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory)
         generatePDLaneColumn(data)
         data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit
         if warmUpLastInstant is not None:
@@ -782,7 +782,7 @@
     columns = ['NO', '$VEHICLE:SIMSEC', 'POS']
     if lanes is not None:
         columns += ['LANE\LINK\NO', 'LANE\INDEX']
-    data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns)
+    data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory)
     data = selectPDLanes(data, lanes)
     data.sort(['$VEHICLE:SIMSEC'], inplace = True)
 
@@ -806,7 +806,7 @@
     If lanes is not None, only the data for the selected lanes will be provided
     (format as string x_y where x is link index and y is lane index)'''
     from pandas import read_csv, merge
-    data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'])
+    data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory)
     data = selectPDLanes(data, lanes)
     merged = merge(data, data, how='inner', left_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], sort = False)
     merged = merged[merged['NO_x']>merged['NO_y']]