Mercurial Hosting > traffic-intelligence
comparison python/storage.py @ 998:933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Sun, 27 May 2018 23:22:48 -0400 |
parents | cc89267b5ff9 |
children | 75af46516b2b |
comparison
equal
deleted
inserted
replaced
997:4f3387a242a1 | 998:933670761a57 |
---|---|
632 print('Unknown POI type {}. Exiting'.format(gmmType)) | 632 print('Unknown POI type {}. Exiting'.format(gmmType)) |
633 import sys | 633 import sys |
634 sys.exit() | 634 sys.exit() |
635 try: | 635 try: |
636 cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))') | 636 cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))') |
637 for i in xrange(gmm.n_components): | 637 for i in range(gmm.n_components): |
638 cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist()))) | 638 cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist()))) |
639 connection.commit() | 639 connection.commit() |
640 except sqlite3.OperationalError as error: | 640 except sqlite3.OperationalError as error: |
641 printDBError(error) | 641 printDBError(error) |
642 | 642 |
708 connection = sqlite3.connect(outputFilename) | 708 connection = sqlite3.connect(outputFilename) |
709 cursor = connection.cursor() | 709 cursor = connection.cursor() |
710 | 710 |
711 cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))') | 711 cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))') |
712 | 712 |
713 for route in prototypes.keys(): | 713 for route in prototypes: |
714 if prototypes[route]!=[]: | 714 if prototypes[route]!=[]: |
715 for i in prototypes[route]: | 715 for i in prototypes[route]: |
716 cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i])) | 716 cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i])) |
717 | 717 |
718 connection.commit() | 718 connection.commit() |
735 printDBError(error) | 735 printDBError(error) |
736 return [] | 736 return [] |
737 | 737 |
738 for row in cursor: | 738 for row in cursor: |
739 route=(row[1],row[2]) | 739 route=(row[1],row[2]) |
740 if route not in prototypes.keys(): | 740 if route not in prototypes: |
741 prototypes[route]=[] | 741 prototypes[route]=[] |
742 prototypes[route].append(row[0]) | 742 prototypes[route].append(row[0]) |
743 nMatching[row[0]]=row[3] | 743 nMatching[row[0]]=row[3] |
744 | 744 |
745 connection.close() | 745 connection.close() |
751 connection = sqlite3.connect(outputFilename) | 751 connection = sqlite3.connect(outputFilename) |
752 cursor = connection.cursor() | 752 cursor = connection.cursor() |
753 | 753 |
754 cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") | 754 cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") |
755 | 755 |
756 for route in labels.keys(): | 756 for route in labels: |
757 if labels[route]!=[]: | 757 if labels[route]!=[]: |
758 for i in labels[route]: | 758 for i in labels[route]: |
759 for j in labels[route][i]: | 759 for j in labels[route][i]: |
760 cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) | 760 cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) |
761 | 761 |
775 return [] | 775 return [] |
776 | 776 |
777 for row in cursor: | 777 for row in cursor: |
778 route=(row[1],row[2]) | 778 route=(row[1],row[2]) |
779 p=row[3] | 779 p=row[3] |
780 if route not in labels.keys(): | 780 if route not in labels: |
781 labels[route]={} | 781 labels[route]={} |
782 if p not in labels[route].keys(): | 782 if p not in labels[route]: |
783 labels[route][p]=[] | 783 labels[route][p]=[] |
784 labels[route][p].append(row[0]) | 784 labels[route][p].append(row[0]) |
785 | 785 |
786 connection.close() | 786 connection.close() |
787 return labels | 787 return labels |
791 connection = sqlite3.connect(outFilename) | 791 connection = sqlite3.connect(outFilename) |
792 cursor = connection.cursor() | 792 cursor = connection.cursor() |
793 | 793 |
794 cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") | 794 cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") |
795 | 795 |
796 for route in prototypes.keys(): | 796 for route in prototypes: |
797 if prototypes[route]!={}: | 797 if prototypes[route]!={}: |
798 for i in prototypes[route]: | 798 for i in prototypes[route]: |
799 if prototypes[route][i]!= []: | 799 if prototypes[route][i]!= []: |
800 for j in prototypes[route][i]: | 800 for j in prototypes[route][i]: |
801 cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) | 801 cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) |
818 printDBError(error) | 818 printDBError(error) |
819 return [] | 819 return [] |
820 | 820 |
821 for row in cursor: | 821 for row in cursor: |
822 route=(row[2],row[3]) | 822 route=(row[2],row[3]) |
823 if route not in prototypes.keys(): | 823 if route not in prototypes: |
824 prototypes[route]={} | 824 prototypes[route]={} |
825 if row[1] not in prototypes[route].keys(): | 825 if row[1] not in prototypes[route]: |
826 prototypes[route][row[1]]=[] | 826 prototypes[route][row[1]]=[] |
827 prototypes[route][row[1]].append(row[0]) | 827 prototypes[route][row[1]].append(row[0]) |
828 nMatching[row[0]]=row[4] | 828 nMatching[row[0]]=row[4] |
829 | 829 |
830 connection.close() | 830 connection.close() |
836 connection = sqlite3.connect(outputFilename) | 836 connection = sqlite3.connect(outputFilename) |
837 cursor = connection.cursor() | 837 cursor = connection.cursor() |
838 | 838 |
839 cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") | 839 cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") |
840 | 840 |
841 for route in Routes.keys(): | 841 for route in Routes: |
842 if Routes[route]!=[]: | 842 if Routes[route]!=[]: |
843 for i in Routes[route]: | 843 for i in Routes[route]: |
844 cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) | 844 cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) |
845 | 845 |
846 connection.commit() | 846 connection.commit() |
858 printDBError(error) | 858 printDBError(error) |
859 return [] | 859 return [] |
860 | 860 |
861 for row in cursor: | 861 for row in cursor: |
862 route=(row[1],row[2]) | 862 route=(row[1],row[2]) |
863 if route not in Routes.keys(): | 863 if route not in Routes: |
864 Routes[route]=[] | 864 Routes[route]=[] |
865 Routes[route].append(row[0]) | 865 Routes[route].append(row[0]) |
866 | 866 |
867 connection.close() | 867 connection.close() |
868 return Routes | 868 return Routes |
926 for l in loadListStrings(filename, commentCharacters): | 926 for l in loadListStrings(filename, commentCharacters): |
927 if l.startswith(option): | 927 if l.startswith(option): |
928 values.append(l.split(delimiterChar)[1].strip()) | 928 values.append(l.split(delimiterChar)[1].strip()) |
929 return values | 929 return values |
930 | 930 |
931 class FakeSecHead(object): | 931 def addSectionHeader(propertiesFile, headerName = 'main'): |
932 '''Add fake section header [asection] | 932 '''Add fake section header |
933 | 933 |
934 from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788 | 934 from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788 |
935 use read_file in Python 3.2+ | 935 use read_file in Python 3.2+ |
936 ''' | 936 ''' |
937 def __init__(self, fp): | 937 yield '[{}]\n'.format(headerName) |
938 self.fp = fp | 938 for line in propertiesFile: |
939 self.sechead = '[main]\n' | 939 yield line |
940 | |
941 def readline(self): | |
942 if self.sechead: | |
943 try: return self.sechead | |
944 finally: self.sechead = None | |
945 else: return self.fp.readline() | |
946 | 940 |
947 def loadPemsTraffic(filename): | 941 def loadPemsTraffic(filename): |
948 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse | 942 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse |
949 into pandas dataframe''' | 943 into pandas dataframe''' |
950 f = openCheck(filename) | 944 f = openCheck(filename) |
951 l = f.readline().strip() | 945 l = f.readline().strip() |
952 items = l.split(',') | 946 items = l.split(',') |
953 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] | 947 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] |
954 nLanes = (len(items)-len(headers))/3 | 948 nLanes = (len(items)-len(headers))/3 |
955 for i in xrange(nLanes): | 949 for i in range(nLanes): |
956 headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)] | 950 headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)] |
957 f.close() | 951 f.close() |
958 return read_csv(filename, delimiter = ',', names = headers) | 952 return read_csv(filename, delimiter = ',', names = headers) |
959 | 953 |
960 def generatePDLaneColumn(data): | 954 def generatePDLaneColumn(data): |
1027 tmp = data[data['NO'] == objNum] | 1021 tmp = data[data['NO'] == objNum] |
1028 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last'])) | 1022 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last'])) |
1029 # positions should be rounded to nDecimals decimals only | 1023 # positions should be rounded to nDecimals decimals only |
1030 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) | 1024 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) |
1031 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: | 1025 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: |
1032 objects.values() | 1026 return list(objects.values()) |
1033 else: | 1027 else: |
1034 if filename.endswith(".fzp"): | 1028 if filename.endswith(".fzp"): |
1035 inputfile = openCheck(filename, quitting = True) | 1029 inputfile = openCheck(filename, quitting = True) |
1036 line = readline(inputfile, '*$') | 1030 line = readline(inputfile, '*$') |
1037 while len(line) > 0:#for line in inputfile: | 1031 while len(line) > 0:#for line in inputfile: |
1075 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) | 1069 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) |
1076 except sqlite3.OperationalError as error: | 1070 except sqlite3.OperationalError as error: |
1077 printDBError(error) | 1071 printDBError(error) |
1078 else: | 1072 else: |
1079 print("File type of "+filename+" not supported (only .sqlite and .fzp files)") | 1073 print("File type of "+filename+" not supported (only .sqlite and .fzp files)") |
1080 return objects.values() | 1074 return list(objects.values()) |
1081 | 1075 |
1082 def selectPDLanes(data, lanes = None): | 1076 def selectPDLanes(data, lanes = None): |
1083 '''Selects the subset of data for the right lanes | 1077 '''Selects the subset of data for the right lanes |
1084 | 1078 |
1085 Lane format is a string 'x_y' where x is link index and y is lane index''' | 1079 Lane format is a string 'x_y' where x is link index and y is lane index''' |
1255 | 1249 |
1256 def savePositionsToCsv(f, obj): | 1250 def savePositionsToCsv(f, obj): |
1257 timeInterval = obj.getTimeInterval() | 1251 timeInterval = obj.getTimeInterval() |
1258 positions = obj.getPositions() | 1252 positions = obj.getPositions() |
1259 curvilinearPositions = obj.getCurvilinearPositions() | 1253 curvilinearPositions = obj.getCurvilinearPositions() |
1260 for i in xrange(int(obj.length())): | 1254 for i in range(int(obj.length())): |
1261 p1 = positions[i] | 1255 p1 = positions[i] |
1262 s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y) | 1256 s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y) |
1263 if curvilinearPositions is not None: | 1257 if curvilinearPositions is not None: |
1264 p2 = curvilinearPositions[i] | 1258 p2 = curvilinearPositions[i] |
1265 s += ',{},{}'.format(p2[0],p2[1]) | 1259 s += ',{},{}'.format(p2[0],p2[1]) |
1277 ######################### | 1271 ######################### |
1278 | 1272 |
1279 class ClassifierParameters(VideoFilenameAddable): | 1273 class ClassifierParameters(VideoFilenameAddable): |
1280 'Class for the parameters of object classifiers' | 1274 'Class for the parameters of object classifiers' |
1281 def loadConfigFile(self, filename): | 1275 def loadConfigFile(self, filename): |
1282 from ConfigParser import ConfigParser | 1276 from configparser import ConfigParser |
1283 | 1277 |
1284 config = ConfigParser() | 1278 config = ConfigParser() |
1285 config.readfp(FakeSecHead(openCheck(filename))) | 1279 config.read_file(addSectionHeader(openCheck(filename))) |
1286 self.sectionHeader = config.sections()[0] | 1280 self.sectionHeader = config.sections()[0] |
1287 | 1281 |
1288 self.pedBikeCarSVMFilename = config.get(self.sectionHeader, 'pbv-svm-filename') | 1282 self.pedBikeCarSVMFilename = config.get(self.sectionHeader, 'pbv-svm-filename') |
1289 self.bikeCarSVMFilename = config.get(self.sectionHeader, 'bv-svm-filename') | 1283 self.bikeCarSVMFilename = config.get(self.sectionHeader, 'bv-svm-filename') |
1290 self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop') | 1284 self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop') |
1344 method parameters, etc. for tracking and safety | 1338 method parameters, etc. for tracking and safety |
1345 | 1339 |
1346 Note: framerate is already taken into account''' | 1340 Note: framerate is already taken into account''' |
1347 | 1341 |
1348 def loadConfigFile(self, filename): | 1342 def loadConfigFile(self, filename): |
1349 from ConfigParser import ConfigParser | 1343 from configparser import ConfigParser |
1350 | 1344 |
1351 config = ConfigParser() | 1345 config = ConfigParser(strict=False) |
1352 config.readfp(FakeSecHead(openCheck(filename))) | 1346 config.read_file(addSectionHeader(openCheck(filename))) |
1353 | 1347 |
1354 self.sectionHeader = config.sections()[0] | 1348 self.sectionHeader = config.sections()[0] |
1355 # Tracking/display parameters | 1349 # Tracking/display parameters |
1356 self.videoFilename = config.get(self.sectionHeader, 'video-filename') | 1350 self.videoFilename = config.get(self.sectionHeader, 'video-filename') |
1357 self.databaseFilename = config.get(self.sectionHeader, 'database-filename') | 1351 self.databaseFilename = config.get(self.sectionHeader, 'database-filename') |
1439 return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum | 1433 return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum |
1440 | 1434 |
1441 # deprecated | 1435 # deprecated |
1442 class SceneParameters(object): | 1436 class SceneParameters(object): |
1443 def __init__(self, config, sectionName): | 1437 def __init__(self, config, sectionName): |
1444 from ConfigParser import NoOptionError | 1438 from configparser import NoOptionError |
1445 from ast import literal_eval | 1439 from ast import literal_eval |
1446 try: | 1440 try: |
1447 self.sitename = config.get(sectionName, 'sitename') | 1441 self.sitename = config.get(sectionName, 'sitename') |
1448 self.databaseFilename = config.get(sectionName, 'data-filename') | 1442 self.databaseFilename = config.get(sectionName, 'data-filename') |
1449 self.homographyFilename = config.get(sectionName, 'homography-filename') | 1443 self.homographyFilename = config.get(sectionName, 'homography-filename') |
1458 print(e) | 1452 print(e) |
1459 print('Not a section for scene meta-data') | 1453 print('Not a section for scene meta-data') |
1460 | 1454 |
1461 @staticmethod | 1455 @staticmethod |
1462 def loadConfigFile(filename): | 1456 def loadConfigFile(filename): |
1463 from ConfigParser import ConfigParser | 1457 from configparser import ConfigParser |
1464 config = ConfigParser() | 1458 config = ConfigParser() |
1465 config.readfp(openCheck(filename)) | 1459 config.readfp(openCheck(filename)) |
1466 configDict = dict() | 1460 configDict = dict() |
1467 for sectionName in config.sections(): | 1461 for sectionName in config.sections(): |
1468 configDict[sectionName] = SceneParameters(config, sectionName) | 1462 configDict[sectionName] = SceneParameters(config, sectionName) |