Mercurial Hosting > traffic-intelligence
changeset 718:2cd245cb780d
added option to set low_memory = False for pandas.read_csv
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Thu, 30 Jul 2015 17:43:14 -0400 |
parents | 9d6cd4e8dca3 |
children | ea78c326b3a0 |
files | python/storage.py |
diffstat | 1 files changed, 4 insertions(+), 4 deletions(-) [+] |
line wrap: on
line diff
--- a/python/storage.py Sat Jul 25 23:29:11 2015 -0400 +++ b/python/storage.py Thu Jul 30 17:43:14 2015 -0400 @@ -701,7 +701,7 @@ def generatePDLaneColumn(data): data['LANE'] = data['LANE\LINK\NO'].astype(str)+'_'+data['LANE\INDEX'].astype(str) -def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, nObjects = -1, warmUpLastInstant = None, usePandas = False, nDecimals = 2): +def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, nObjects = -1, warmUpLastInstant = None, usePandas = False, nDecimals = 2, lowMemory = True): '''Reads data from VISSIM .fzp trajectory file simulationStepsPerTimeUnit is the number of simulation steps per unit of time used by VISSIM for example, there seems to be 5 simulation steps per simulated second in VISSIM, @@ -716,7 +716,7 @@ if usePandas: from pandas import read_csv - data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1) + data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory) generatePDLaneColumn(data) data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit if warmUpLastInstant is not None: @@ -782,7 +782,7 @@ columns = ['NO', '$VEHICLE:SIMSEC', 'POS'] if lanes is not None: columns += ['LANE\LINK\NO', 'LANE\INDEX'] - data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns) + data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory) data = selectPDLanes(data, lanes) data.sort(['$VEHICLE:SIMSEC'], inplace = True) @@ -806,7 +806,7 @@ If lanes is not None, only the data for the selected lanes will be provided (format as string x_y where x is link index and y is lane index)''' from pandas import read_csv, merge - data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS']) + data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory) data = selectPDLanes(data, lanes) merged = merge(data, data, how='inner', left_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], sort = False) merged = merged[merged['NO_x']>merged['NO_y']]