Mercurial Hosting > traffic-intelligence

comparison python/storage.py @ 998:933670761a57

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Sun, 27 May 2018 23:22:48 -0400
parents cc89267b5ff9
children 75af46516b2b
comparison
equal deleted inserted replaced
997:4f3387a242a1 998:933670761a57
632 print('Unknown POI type {}. Exiting'.format(gmmType)) 632 print('Unknown POI type {}. Exiting'.format(gmmType))
633 import sys 633 import sys
634 sys.exit() 634 sys.exit()
635 try: 635 try:
636 cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))') 636 cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))')
637 for i in xrange(gmm.n_components): 637 for i in range(gmm.n_components):
638 cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist()))) 638 cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist())))
639 connection.commit() 639 connection.commit()
640 except sqlite3.OperationalError as error: 640 except sqlite3.OperationalError as error:
641 printDBError(error) 641 printDBError(error)
642 642
708 connection = sqlite3.connect(outputFilename) 708 connection = sqlite3.connect(outputFilename)
709 cursor = connection.cursor() 709 cursor = connection.cursor()
710 710
711 cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))') 711 cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))')
712 712
713 for route in prototypes.keys(): 713 for route in prototypes:
714 if prototypes[route]!=[]: 714 if prototypes[route]!=[]:
715 for i in prototypes[route]: 715 for i in prototypes[route]:
716 cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i])) 716 cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i]))
717 717
718 connection.commit() 718 connection.commit()
735 printDBError(error) 735 printDBError(error)
736 return [] 736 return []
737 737
738 for row in cursor: 738 for row in cursor:
739 route=(row[1],row[2]) 739 route=(row[1],row[2])
740 if route not in prototypes.keys(): 740 if route not in prototypes:
741 prototypes[route]=[] 741 prototypes[route]=[]
742 prototypes[route].append(row[0]) 742 prototypes[route].append(row[0])
743 nMatching[row[0]]=row[3] 743 nMatching[row[0]]=row[3]
744 744
745 connection.close() 745 connection.close()
751 connection = sqlite3.connect(outputFilename) 751 connection = sqlite3.connect(outputFilename)
752 cursor = connection.cursor() 752 cursor = connection.cursor()
753 753
754 cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") 754 cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))")
755 755
756 for route in labels.keys(): 756 for route in labels:
757 if labels[route]!=[]: 757 if labels[route]!=[]:
758 for i in labels[route]: 758 for i in labels[route]:
759 for j in labels[route][i]: 759 for j in labels[route][i]:
760 cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) 760 cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i))
761 761
775 return [] 775 return []
776 776
777 for row in cursor: 777 for row in cursor:
778 route=(row[1],row[2]) 778 route=(row[1],row[2])
779 p=row[3] 779 p=row[3]
780 if route not in labels.keys(): 780 if route not in labels:
781 labels[route]={} 781 labels[route]={}
782 if p not in labels[route].keys(): 782 if p not in labels[route]:
783 labels[route][p]=[] 783 labels[route][p]=[]
784 labels[route][p].append(row[0]) 784 labels[route][p].append(row[0])
785 785
786 connection.close() 786 connection.close()
787 return labels 787 return labels
791 connection = sqlite3.connect(outFilename) 791 connection = sqlite3.connect(outFilename)
792 cursor = connection.cursor() 792 cursor = connection.cursor()
793 793
794 cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") 794 cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))")
795 795
796 for route in prototypes.keys(): 796 for route in prototypes:
797 if prototypes[route]!={}: 797 if prototypes[route]!={}:
798 for i in prototypes[route]: 798 for i in prototypes[route]:
799 if prototypes[route][i]!= []: 799 if prototypes[route][i]!= []:
800 for j in prototypes[route][i]: 800 for j in prototypes[route][i]:
801 cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) 801 cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j]))
818 printDBError(error) 818 printDBError(error)
819 return [] 819 return []
820 820
821 for row in cursor: 821 for row in cursor:
822 route=(row[2],row[3]) 822 route=(row[2],row[3])
823 if route not in prototypes.keys(): 823 if route not in prototypes:
824 prototypes[route]={} 824 prototypes[route]={}
825 if row[1] not in prototypes[route].keys(): 825 if row[1] not in prototypes[route]:
826 prototypes[route][row[1]]=[] 826 prototypes[route][row[1]]=[]
827 prototypes[route][row[1]].append(row[0]) 827 prototypes[route][row[1]].append(row[0])
828 nMatching[row[0]]=row[4] 828 nMatching[row[0]]=row[4]
829 829
830 connection.close() 830 connection.close()
836 connection = sqlite3.connect(outputFilename) 836 connection = sqlite3.connect(outputFilename)
837 cursor = connection.cursor() 837 cursor = connection.cursor()
838 838
839 cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") 839 cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))")
840 840
841 for route in Routes.keys(): 841 for route in Routes:
842 if Routes[route]!=[]: 842 if Routes[route]!=[]:
843 for i in Routes[route]: 843 for i in Routes[route]:
844 cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) 844 cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1]))
845 845
846 connection.commit() 846 connection.commit()
858 printDBError(error) 858 printDBError(error)
859 return [] 859 return []
860 860
861 for row in cursor: 861 for row in cursor:
862 route=(row[1],row[2]) 862 route=(row[1],row[2])
863 if route not in Routes.keys(): 863 if route not in Routes:
864 Routes[route]=[] 864 Routes[route]=[]
865 Routes[route].append(row[0]) 865 Routes[route].append(row[0])
866 866
867 connection.close() 867 connection.close()
868 return Routes 868 return Routes
926 for l in loadListStrings(filename, commentCharacters): 926 for l in loadListStrings(filename, commentCharacters):
927 if l.startswith(option): 927 if l.startswith(option):
928 values.append(l.split(delimiterChar)[1].strip()) 928 values.append(l.split(delimiterChar)[1].strip())
929 return values 929 return values
930 930
931 class FakeSecHead(object): 931 def addSectionHeader(propertiesFile, headerName = 'main'):
932 '''Add fake section header [asection] 932 '''Add fake section header
933 933
934 from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788 934 from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788
935 use read_file in Python 3.2+ 935 use read_file in Python 3.2+
936 ''' 936 '''
937 def __init__(self, fp): 937 yield '[{}]\n'.format(headerName)
938 self.fp = fp 938 for line in propertiesFile:
939 self.sechead = '[main]\n' 939 yield line
940
941 def readline(self):
942 if self.sechead:
943 try: return self.sechead
944 finally: self.sechead = None
945 else: return self.fp.readline()
946 940
947 def loadPemsTraffic(filename): 941 def loadPemsTraffic(filename):
948 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse 942 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse
949 into pandas dataframe''' 943 into pandas dataframe'''
950 f = openCheck(filename) 944 f = openCheck(filename)
951 l = f.readline().strip() 945 l = f.readline().strip()
952 items = l.split(',') 946 items = l.split(',')
953 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] 947 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60']
954 nLanes = (len(items)-len(headers))/3 948 nLanes = (len(items)-len(headers))/3
955 for i in xrange(nLanes): 949 for i in range(nLanes):
956 headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)] 950 headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)]
957 f.close() 951 f.close()
958 return read_csv(filename, delimiter = ',', names = headers) 952 return read_csv(filename, delimiter = ',', names = headers)
959 953
960 def generatePDLaneColumn(data): 954 def generatePDLaneColumn(data):
1027 tmp = data[data['NO'] == objNum] 1021 tmp = data[data['NO'] == objNum]
1028 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last'])) 1022 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last']))
1029 # positions should be rounded to nDecimals decimals only 1023 # positions should be rounded to nDecimals decimals only
1030 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) 1024 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist())
1031 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: 1025 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers:
1032 objects.values() 1026 return list(objects.values())
1033 else: 1027 else:
1034 if filename.endswith(".fzp"): 1028 if filename.endswith(".fzp"):
1035 inputfile = openCheck(filename, quitting = True) 1029 inputfile = openCheck(filename, quitting = True)
1036 line = readline(inputfile, '*$') 1030 line = readline(inputfile, '*$')
1037 while len(line) > 0:#for line in inputfile: 1031 while len(line) > 0:#for line in inputfile:
1075 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) 1069 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane)
1076 except sqlite3.OperationalError as error: 1070 except sqlite3.OperationalError as error:
1077 printDBError(error) 1071 printDBError(error)
1078 else: 1072 else:
1079 print("File type of "+filename+" not supported (only .sqlite and .fzp files)") 1073 print("File type of "+filename+" not supported (only .sqlite and .fzp files)")
1080 return objects.values() 1074 return list(objects.values())
1081 1075
1082 def selectPDLanes(data, lanes = None): 1076 def selectPDLanes(data, lanes = None):
1083 '''Selects the subset of data for the right lanes 1077 '''Selects the subset of data for the right lanes
1084 1078
1085 Lane format is a string 'x_y' where x is link index and y is lane index''' 1079 Lane format is a string 'x_y' where x is link index and y is lane index'''
1255 1249
1256 def savePositionsToCsv(f, obj): 1250 def savePositionsToCsv(f, obj):
1257 timeInterval = obj.getTimeInterval() 1251 timeInterval = obj.getTimeInterval()
1258 positions = obj.getPositions() 1252 positions = obj.getPositions()
1259 curvilinearPositions = obj.getCurvilinearPositions() 1253 curvilinearPositions = obj.getCurvilinearPositions()
1260 for i in xrange(int(obj.length())): 1254 for i in range(int(obj.length())):
1261 p1 = positions[i] 1255 p1 = positions[i]
1262 s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y) 1256 s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y)
1263 if curvilinearPositions is not None: 1257 if curvilinearPositions is not None:
1264 p2 = curvilinearPositions[i] 1258 p2 = curvilinearPositions[i]
1265 s += ',{},{}'.format(p2[0],p2[1]) 1259 s += ',{},{}'.format(p2[0],p2[1])
1277 ######################### 1271 #########################
1278 1272
1279 class ClassifierParameters(VideoFilenameAddable): 1273 class ClassifierParameters(VideoFilenameAddable):
1280 'Class for the parameters of object classifiers' 1274 'Class for the parameters of object classifiers'
1281 def loadConfigFile(self, filename): 1275 def loadConfigFile(self, filename):
1282 from ConfigParser import ConfigParser 1276 from configparser import ConfigParser
1283 1277
1284 config = ConfigParser() 1278 config = ConfigParser()
1285 config.readfp(FakeSecHead(openCheck(filename))) 1279 config.read_file(addSectionHeader(openCheck(filename)))
1286 self.sectionHeader = config.sections()[0] 1280 self.sectionHeader = config.sections()[0]
1287 1281
1288 self.pedBikeCarSVMFilename = config.get(self.sectionHeader, 'pbv-svm-filename') 1282 self.pedBikeCarSVMFilename = config.get(self.sectionHeader, 'pbv-svm-filename')
1289 self.bikeCarSVMFilename = config.get(self.sectionHeader, 'bv-svm-filename') 1283 self.bikeCarSVMFilename = config.get(self.sectionHeader, 'bv-svm-filename')
1290 self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop') 1284 self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop')
1344 method parameters, etc. for tracking and safety 1338 method parameters, etc. for tracking and safety
1345 1339
1346 Note: framerate is already taken into account''' 1340 Note: framerate is already taken into account'''
1347 1341
1348 def loadConfigFile(self, filename): 1342 def loadConfigFile(self, filename):
1349 from ConfigParser import ConfigParser 1343 from configparser import ConfigParser
1350 1344
1351 config = ConfigParser() 1345 config = ConfigParser(strict=False)
1352 config.readfp(FakeSecHead(openCheck(filename))) 1346 config.read_file(addSectionHeader(openCheck(filename)))
1353 1347
1354 self.sectionHeader = config.sections()[0] 1348 self.sectionHeader = config.sections()[0]
1355 # Tracking/display parameters 1349 # Tracking/display parameters
1356 self.videoFilename = config.get(self.sectionHeader, 'video-filename') 1350 self.videoFilename = config.get(self.sectionHeader, 'video-filename')
1357 self.databaseFilename = config.get(self.sectionHeader, 'database-filename') 1351 self.databaseFilename = config.get(self.sectionHeader, 'database-filename')
1439 return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum 1433 return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum
1440 1434
1441 # deprecated 1435 # deprecated
1442 class SceneParameters(object): 1436 class SceneParameters(object):
1443 def __init__(self, config, sectionName): 1437 def __init__(self, config, sectionName):
1444 from ConfigParser import NoOptionError 1438 from configparser import NoOptionError
1445 from ast import literal_eval 1439 from ast import literal_eval
1446 try: 1440 try:
1447 self.sitename = config.get(sectionName, 'sitename') 1441 self.sitename = config.get(sectionName, 'sitename')
1448 self.databaseFilename = config.get(sectionName, 'data-filename') 1442 self.databaseFilename = config.get(sectionName, 'data-filename')
1449 self.homographyFilename = config.get(sectionName, 'homography-filename') 1443 self.homographyFilename = config.get(sectionName, 'homography-filename')
1458 print(e) 1452 print(e)
1459 print('Not a section for scene meta-data') 1453 print('Not a section for scene meta-data')
1460 1454
1461 @staticmethod 1455 @staticmethod
1462 def loadConfigFile(filename): 1456 def loadConfigFile(filename):
1463 from ConfigParser import ConfigParser 1457 from configparser import ConfigParser
1464 config = ConfigParser() 1458 config = ConfigParser()
1465 config.readfp(openCheck(filename)) 1459 config.readfp(openCheck(filename))
1466 configDict = dict() 1460 configDict = dict()
1467 for sectionName in config.sections(): 1461 for sectionName in config.sections():
1468 configDict[sectionName] = SceneParameters(config, sectionName) 1462 configDict[sectionName] = SceneParameters(config, sectionName)