Mercurial Hosting > traffic-intelligence
comparison trafficintelligence/storage.py @ 1030:aafbc0bab925
moved method around to avoid cross-dependencies
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Tue, 19 Jun 2018 10:04:52 -0400 |
parents | cc5cb04b04b0 |
children | 8ffb3ae9f3d2 |
comparison
equal
deleted
inserted
replaced
1029:c6cf75a2ed08 | 1030:aafbc0bab925 |
---|---|
1 #! /usr/bin/env python | 1 #! /usr/bin/env python |
2 # -*- coding: utf-8 -*- | 2 # -*- coding: utf-8 -*- |
3 '''Various utilities to save and load data''' | 3 '''Various utilities to save and load data''' |
4 | |
5 from trafficintelligence import utils, moving, events, indicators | |
6 from trafficintelligence.base import VideoFilenameAddable | |
7 | 4 |
8 from pathlib import Path | 5 from pathlib import Path |
9 import shutil | 6 import shutil |
10 from copy import copy | 7 from copy import copy |
11 import sqlite3, logging | 8 import sqlite3, logging |
9 | |
12 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg | 10 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg |
13 from pandas import read_csv, merge | 11 from pandas import read_csv, merge |
14 | 12 |
15 | 13 from trafficintelligence import utils, moving, events, indicators |
16 commentChar = '#' | 14 from trafficintelligence.base import VideoFilenameAddable |
17 | 15 |
18 delimiterChar = '%'; | |
19 | 16 |
20 ngsimUserTypes = {'twowheels':1, | 17 ngsimUserTypes = {'twowheels':1, |
21 'car':2, | 18 'car':2, |
22 'truck':3} | 19 'truck':3} |
23 | 20 |
879 | 876 |
880 ######################### | 877 ######################### |
881 # txt files | 878 # txt files |
882 ######################### | 879 ######################### |
883 | 880 |
884 def openCheck(filename, option = 'r', quitting = False): | |
885 '''Open file filename in read mode by default | |
886 and checks it is open''' | |
887 try: | |
888 return open(filename, option) | |
889 except IOError: | |
890 print('File {} could not be opened.'.format(filename)) | |
891 if quitting: | |
892 from sys import exit | |
893 exit() | |
894 return None | |
895 | |
896 def readline(f, commentCharacters = commentChar): | |
897 '''Modified readline function to skip comments | |
898 Can take a list of characters or a string (in will work in both)''' | |
899 s = f.readline() | |
900 while (len(s) > 0) and s[0] in commentCharacters: | |
901 s = f.readline() | |
902 return s.strip() | |
903 | |
904 def getLines(f, delimiterChar = delimiterChar, commentCharacters = commentChar): | |
905 '''Gets a complete entry (all the lines) in between delimiterChar.''' | |
906 dataStrings = [] | |
907 s = readline(f, commentCharacters) | |
908 while len(s) > 0 and s[0] != delimiterChar: | |
909 dataStrings += [s.strip()] | |
910 s = readline(f, commentCharacters) | |
911 return dataStrings | |
912 | |
913 def saveList(filename, l): | 881 def saveList(filename, l): |
914 f = openCheck(filename, 'w') | 882 f = utils.openCheck(filename, 'w') |
915 for x in l: | 883 for x in l: |
916 f.write('{}\n'.format(x)) | 884 f.write('{}\n'.format(x)) |
917 f.close() | 885 f.close() |
918 | 886 |
919 def loadListStrings(filename, commentCharacters = commentChar): | 887 def loadListStrings(filename, commentCharacters = utils.commentChar): |
920 f = openCheck(filename, 'r') | 888 f = utils.openCheck(filename, 'r') |
921 result = getLines(f, commentCharacters) | 889 result = utils.getLines(f, commentCharacters) |
922 f.close() | 890 f.close() |
923 return result | 891 return result |
924 | 892 |
925 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar): | 893 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = utils.commentChar): |
926 values = [] | 894 values = [] |
927 for l in loadListStrings(filename, commentCharacters): | 895 for l in loadListStrings(filename, commentCharacters): |
928 if l.startswith(option): | 896 if l.startswith(option): |
929 values.append(l.split(delimiterChar)[1].strip()) | 897 values.append(l.split(delimiterChar)[1].strip()) |
930 return values | 898 return values |
940 yield line | 908 yield line |
941 | 909 |
942 def loadPemsTraffic(filename): | 910 def loadPemsTraffic(filename): |
943 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse | 911 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse |
944 into pandas dataframe''' | 912 into pandas dataframe''' |
945 f = openCheck(filename) | 913 f = utils.openCheck(filename) |
946 l = f.readline().strip() | 914 l = f.readline().strip() |
947 items = l.split(',') | 915 items = l.split(',') |
948 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] | 916 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] |
949 nLanes = (len(items)-len(headers))/3 | 917 nLanes = (len(items)-len(headers))/3 |
950 for i in range(nLanes): | 918 for i in range(nLanes): |
958 def convertTrajectoriesVissimToSqlite(filename): | 926 def convertTrajectoriesVissimToSqlite(filename): |
959 '''Relies on a system call to sqlite3 | 927 '''Relies on a system call to sqlite3 |
960 sqlite3 [file.sqlite] < import_fzp.sql''' | 928 sqlite3 [file.sqlite] < import_fzp.sql''' |
961 sqlScriptFilename = "import_fzp.sql" | 929 sqlScriptFilename = "import_fzp.sql" |
962 # create sql file | 930 # create sql file |
963 out = openCheck(sqlScriptFilename, "w") | 931 out = utils.openCheck(sqlScriptFilename, "w") |
964 out.write(".separator \";\"\n"+ | 932 out.write(".separator \";\"\n"+ |
965 "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+ | 933 "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+ |
966 ".import "+filename+" curvilinear_positions\n"+ | 934 ".import "+filename+" curvilinear_positions\n"+ |
967 "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") | 935 "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") |
968 out.close() | 936 out.close() |
969 # system call | 937 # system call |
970 from subprocess import run | 938 from subprocess import run |
971 out = openCheck("err.log", "w") | 939 out = utils.openCheck("err.log", "w") |
972 run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) | 940 run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) |
973 out.close() | 941 out.close() |
974 shutil.os.remove(sqlScriptFilename) | 942 shutil.os.remove(sqlScriptFilename) |
975 | 943 |
976 def loadObjectNumbersInLinkFromVissimFile(filename, linkIds): | 944 def loadObjectNumbersInLinkFromVissimFile(filename, linkIds): |
1025 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) | 993 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) |
1026 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: | 994 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: |
1027 return list(objects.values()) | 995 return list(objects.values()) |
1028 else: | 996 else: |
1029 if filename.endswith(".fzp"): | 997 if filename.endswith(".fzp"): |
1030 inputfile = openCheck(filename, quitting = True) | 998 inputfile = utils.openCheck(filename, quitting = True) |
1031 line = readline(inputfile, '*$') | 999 line = utils.readline(inputfile, '*$') |
1032 while len(line) > 0:#for line in inputfile: | 1000 while len(line) > 0:#for line in inputfile: |
1033 data = line.strip().split(';') | 1001 data = line.strip().split(';') |
1034 objNum = int(data[1]) | 1002 objNum = int(data[1]) |
1035 instant = float(data[0])*simulationStepsPerTimeUnit | 1003 instant = float(data[0])*simulationStepsPerTimeUnit |
1036 s = float(data[4]) | 1004 s = float(data[4]) |
1042 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) | 1010 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) |
1043 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() | 1011 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() |
1044 if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: | 1012 if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: |
1045 objects[objNum].timeInterval.last = instant | 1013 objects[objNum].timeInterval.last = instant |
1046 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) | 1014 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) |
1047 line = readline(inputfile, '*$') | 1015 line = utils.readline(inputfile, '*$') |
1048 elif filename.endswith(".sqlite"): | 1016 elif filename.endswith(".sqlite"): |
1049 with sqlite3.connect(filename) as connection: | 1017 with sqlite3.connect(filename) as connection: |
1050 cursor = connection.cursor() | 1018 cursor = connection.cursor() |
1051 queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions' | 1019 queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions' |
1052 if objectNumbers is not None: | 1020 if objectNumbers is not None: |
1154 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): | 1122 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): |
1155 '''Reads data from the trajectory data provided by NGSIM project | 1123 '''Reads data from the trajectory data provided by NGSIM project |
1156 and returns the list of Feature objects''' | 1124 and returns the list of Feature objects''' |
1157 objects = [] | 1125 objects = [] |
1158 | 1126 |
1159 inputfile = openCheck(filename, quitting = True) | 1127 inputfile = utils.openCheck(filename, quitting = True) |
1160 | 1128 |
1161 def createObject(numbers): | 1129 def createObject(numbers): |
1162 firstFrameNum = int(numbers[1]) | 1130 firstFrameNum = int(numbers[1]) |
1163 # do the geometry and usertype | 1131 # do the geometry and usertype |
1164 | 1132 |
1178 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate | 1146 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate |
1179 obj.speeds = [float(numbers[11])] | 1147 obj.speeds = [float(numbers[11])] |
1180 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object | 1148 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object |
1181 return obj | 1149 return obj |
1182 | 1150 |
1183 numbers = readline(inputfile).strip().split() | 1151 numbers = utils.readline(inputfile).strip().split() |
1184 if (len(numbers) > 0): | 1152 if (len(numbers) > 0): |
1185 obj = createObject(numbers) | 1153 obj = createObject(numbers) |
1186 | 1154 |
1187 for line in inputfile: | 1155 for line in inputfile: |
1188 numbers = line.strip().split() | 1156 numbers = line.strip().split() |
1216 | 1184 |
1217 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): | 1185 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): |
1218 '''Reads data from the trajectory data provided by NGSIM project | 1186 '''Reads data from the trajectory data provided by NGSIM project |
1219 and converts to our current format.''' | 1187 and converts to our current format.''' |
1220 if append: | 1188 if append: |
1221 out = openCheck(outputfile,'a') | 1189 out = utils.openCheck(outputfile,'a') |
1222 else: | 1190 else: |
1223 out = openCheck(outputfile,'w') | 1191 out = utils.openCheck(outputfile,'w') |
1224 nObjectsPerType = [0,0,0] | 1192 nObjectsPerType = [0,0,0] |
1225 | 1193 |
1226 features = loadNgsimFile(inputfile, sequenceNum) | 1194 features = loadNgsimFile(inputfile, sequenceNum) |
1227 for f in features: | 1195 for f in features: |
1228 nObjectsPerType[f.userType-1] += 1 | 1196 nObjectsPerType[f.userType-1] += 1 |
1235 def loadPinholeCameraModel(filename, tanalystFormat = True): | 1203 def loadPinholeCameraModel(filename, tanalystFormat = True): |
1236 '''Loads the data from a file containing the camera parameters | 1204 '''Loads the data from a file containing the camera parameters |
1237 (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) | 1205 (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) |
1238 and returns a dictionary''' | 1206 and returns a dictionary''' |
1239 if tanalystFormat: | 1207 if tanalystFormat: |
1240 f = openCheck(filename, quitting = True) | 1208 f = utils.openCheck(filename, quitting = True) |
1241 content = getLines(f) | 1209 content = utils.getLines(f) |
1242 cameraData = {} | 1210 cameraData = {} |
1243 for l in content: | 1211 for l in content: |
1244 tmp = l.split(':') | 1212 tmp = l.split(':') |
1245 cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.')) | 1213 cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.')) |
1246 return cameraData | 1214 return cameraData |
1259 p2 = curvilinearPositions[i] | 1227 p2 = curvilinearPositions[i] |
1260 s += ',{},{}'.format(p2[0],p2[1]) | 1228 s += ',{},{}'.format(p2[0],p2[1]) |
1261 f.write(s+'\n') | 1229 f.write(s+'\n') |
1262 | 1230 |
1263 def saveTrajectoriesToCsv(filename, objects): | 1231 def saveTrajectoriesToCsv(filename, objects): |
1264 f = openCheck(filename, 'w') | 1232 f = utils.openCheck(filename, 'w') |
1265 for i,obj in enumerate(objects): | 1233 for i,obj in enumerate(objects): |
1266 savePositionsToCsv(f, obj) | 1234 savePositionsToCsv(f, obj) |
1267 f.close() | 1235 f.close() |
1268 | 1236 |
1269 | 1237 |
1275 'Class for the parameters of object classifiers' | 1243 'Class for the parameters of object classifiers' |
1276 def loadConfigFile(self, filename): | 1244 def loadConfigFile(self, filename): |
1277 from configparser import ConfigParser | 1245 from configparser import ConfigParser |
1278 | 1246 |
1279 config = ConfigParser() | 1247 config = ConfigParser() |
1280 config.read_file(addSectionHeader(openCheck(filename))) | 1248 config.read_file(addSectionHeader(utils.openCheck(filename))) |
1281 | 1249 |
1282 parentPath = Path(filename).parent | 1250 parentPath = Path(filename).parent |
1283 self.sectionHeader = config.sections()[0] | 1251 self.sectionHeader = config.sections()[0] |
1284 | 1252 |
1285 self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename')) | 1253 self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename')) |
1344 | 1312 |
1345 def loadConfigFile(self, filename): | 1313 def loadConfigFile(self, filename): |
1346 from configparser import ConfigParser | 1314 from configparser import ConfigParser |
1347 | 1315 |
1348 config = ConfigParser(strict=False) | 1316 config = ConfigParser(strict=False) |
1349 config.read_file(addSectionHeader(openCheck(filename))) | 1317 config.read_file(addSectionHeader(utils.openCheck(filename))) |
1350 | 1318 |
1351 parentPath = Path(filename).parent | 1319 parentPath = Path(filename).parent |
1352 self.sectionHeader = config.sections()[0] | 1320 self.sectionHeader = config.sections()[0] |
1353 # Tracking/display parameters | 1321 # Tracking/display parameters |
1354 self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) | 1322 self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) |
1460 | 1428 |
1461 @staticmethod | 1429 @staticmethod |
1462 def loadConfigFile(filename): | 1430 def loadConfigFile(filename): |
1463 from configparser import ConfigParser | 1431 from configparser import ConfigParser |
1464 config = ConfigParser() | 1432 config = ConfigParser() |
1465 config.readfp(openCheck(filename)) | 1433 config.readfp(utils.openCheck(filename)) |
1466 configDict = dict() | 1434 configDict = dict() |
1467 for sectionName in config.sections(): | 1435 for sectionName in config.sections(): |
1468 configDict[sectionName] = SceneParameters(config, sectionName) | 1436 configDict[sectionName] = SceneParameters(config, sectionName) |
1469 return configDict | 1437 return configDict |
1470 | 1438 |