Mercurial Hosting > traffic-intelligence
comparison python/storage.py @ 564:36605d843be5
modified bug for reading vissim files, cleaned use of readline with multiple type of characters for comments (to ignore)
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Mon, 21 Jul 2014 16:17:22 -0400 |
parents | ca6bded754ac |
children | 5bda87ac0a69 84690dfe5560 |
comparison
equal
deleted
inserted
replaced
563:39de5c532559 | 564:36605d843be5 |
---|---|
33 cursor.execute('DROP TABLE IF EXISTS '+tableName) | 33 cursor.execute('DROP TABLE IF EXISTS '+tableName) |
34 except sqlite3.OperationalError as error: | 34 except sqlite3.OperationalError as error: |
35 printDBError(error) | 35 printDBError(error) |
36 | 36 |
37 # IO to sqlite | 37 # IO to sqlite |
38 def writeTrajectoriesToSqlite(objects, outFilename, trajectoryType, objectNumbers = -1): | 38 def writeTrajectoriesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1): |
39 """ | 39 """ |
40 This function writers trajectories to a specified sqlite file | 40 This function writers trajectories to a specified sqlite file |
41 @param[in] objects -> a list of trajectories | 41 @param[in] objects -> a list of trajectories |
42 @param[in] trajectoryType - | 42 @param[in] trajectoryType - |
43 @param[out] outFile -> the .sqlite file containting the written objects | 43 @param[out] outputFilename -> the .sqlite file containting the written objects |
44 @param[in] objectNumber : number of objects loaded | 44 @param[in] objectNumber : number of objects loaded |
45 """ | 45 """ |
46 connection = sqlite3.connect(outFilename) | 46 connection = sqlite3.connect(outputFilename) |
47 cursor = connection.cursor() | 47 cursor = connection.cursor() |
48 | 48 |
49 schema = "CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))" | 49 schema = "CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))" |
50 cursor.execute(schema) | 50 cursor.execute(schema) |
51 | 51 |
62 cursor.execute(query,(trajectory_id,frame_number,position.x,position.y)) | 62 cursor.execute(query,(trajectory_id,frame_number,position.x,position.y)) |
63 | 63 |
64 connection.commit() | 64 connection.commit() |
65 connection.close() | 65 connection.close() |
66 | 66 |
67 def writeFeaturesToSqlite(objects, outFilename, trajectoryType, objectNumbers = -1): | 67 def writeFeaturesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1): |
68 '''write features trajectories maintain trajectory ID,velocities dataset ''' | 68 '''write features trajectories maintain trajectory ID,velocities dataset ''' |
69 connection = sqlite3.connect(outFilename) | 69 connection = sqlite3.connect(outputFilename) |
70 cursor = connection.cursor() | 70 cursor = connection.cursor() |
71 | 71 |
72 cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") | 72 cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") |
73 cursor.execute("CREATE TABLE IF NOT EXISTS \"velocities\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") | 73 cursor.execute("CREATE TABLE IF NOT EXISTS \"velocities\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") |
74 | 74 |
83 frame_number += 1 | 83 frame_number += 1 |
84 | 84 |
85 connection.commit() | 85 connection.commit() |
86 connection.close() | 86 connection.close() |
87 | 87 |
88 def writePrototypesToSqlite(prototypes,nMatching, outFilename): | 88 def writePrototypesToSqlite(prototypes,nMatching, outputFilename): |
89 """ prototype dataset is a dictionary with keys== routes, values== prototypes Ids """ | 89 """ prototype dataset is a dictionary with keys== routes, values== prototypes Ids """ |
90 connection = sqlite3.connect(outFilename) | 90 connection = sqlite3.connect(outputFilename) |
91 cursor = connection.cursor() | 91 cursor = connection.cursor() |
92 | 92 |
93 cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))") | 93 cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))") |
94 | 94 |
95 for route in prototypes.keys(): | 95 for route in prototypes.keys(): |
125 nMatching[row[0]]=row[3] | 125 nMatching[row[0]]=row[3] |
126 | 126 |
127 connection.close() | 127 connection.close() |
128 return prototypes,nMatching | 128 return prototypes,nMatching |
129 | 129 |
130 def writeLabelsToSqlite(labels, outFilename): | 130 def writeLabelsToSqlite(labels, outputFilename): |
131 """ labels is a dictionary with keys: routes, values: prototypes Ids | 131 """ labels is a dictionary with keys: routes, values: prototypes Ids |
132 """ | 132 """ |
133 connection = sqlite3.connect(outFilename) | 133 connection = sqlite3.connect(outputFilename) |
134 cursor = connection.cursor() | 134 cursor = connection.cursor() |
135 | 135 |
136 cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") | 136 cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") |
137 | 137 |
138 for route in labels.keys(): | 138 for route in labels.keys(): |
166 labels[route][p].append(row[0]) | 166 labels[route][p].append(row[0]) |
167 | 167 |
168 connection.close() | 168 connection.close() |
169 return labels | 169 return labels |
170 | 170 |
171 def writeRoutesToSqlite(Routes, outFilename): | 171 def writeRoutesToSqlite(Routes, outputFilename): |
172 """ This function writes the activity path define by start and end IDs""" | 172 """ This function writes the activity path define by start and end IDs""" |
173 connection = sqlite3.connect(outFilename) | 173 connection = sqlite3.connect(outputFilename) |
174 cursor = connection.cursor() | 174 cursor = connection.cursor() |
175 | 175 |
176 cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") | 176 cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") |
177 | 177 |
178 for route in Routes.keys(): | 178 for route in Routes.keys(): |
515 if quitting: | 515 if quitting: |
516 from sys import exit | 516 from sys import exit |
517 exit() | 517 exit() |
518 return None | 518 return None |
519 | 519 |
520 def readline(f, commentChar = commentChar): | 520 def readline(f, commentCharacters = commentChar): |
521 '''Modified readline function to skip comments.''' | 521 '''Modified readline function to skip comments |
522 Can take a list of characters or a string (in will work in both)''' | |
522 s = f.readline() | 523 s = f.readline() |
523 while (len(s) > 0) and s.startswith(commentChar): | 524 while (len(s) > 0) and s[0] in commentCharacters: |
524 s = f.readline() | 525 s = f.readline() |
525 return s.strip() | 526 return s.strip() |
526 | 527 |
527 def getLines(f, commentChar = commentChar): | 528 def getLines(f, commentCharacters = commentChar): |
528 '''Gets a complete entry (all the lines) in between delimiterChar.''' | 529 '''Gets a complete entry (all the lines) in between delimiterChar.''' |
529 dataStrings = [] | 530 dataStrings = [] |
530 s = readline(f, commentChar) | 531 s = readline(f, commentCharacters) |
531 while len(s) > 0: | 532 while len(s) > 0: |
532 dataStrings += [s.strip()] | 533 dataStrings += [s.strip()] |
533 s = readline(f, commentChar) | 534 s = readline(f, commentCharacters) |
534 return dataStrings | 535 return dataStrings |
535 | 536 |
536 def writeList(filename, l): | 537 def writeList(filename, l): |
537 f = openCheck(filename, 'w') | 538 f = openCheck(filename, 'w') |
538 for x in l: | 539 for x in l: |
539 f.write('{}\n'.format(x)) | 540 f.write('{}\n'.format(x)) |
540 f.close() | 541 f.close() |
541 | 542 |
542 def loadListStrings(filename, commentChar = commentChar): | 543 def loadListStrings(filename, commentCharacters = commentChar): |
543 f = openCheck(filename, 'r') | 544 f = openCheck(filename, 'r') |
544 result = getLines(f, commentChar) | 545 result = getLines(f, commentCharacters) |
545 f.close() | 546 f.close() |
546 return result | 547 return result |
547 | 548 |
548 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentChar = commentChar): | 549 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar): |
549 values = [] | 550 values = [] |
550 for l in loadListStrings(filename, commentChar): | 551 for l in loadListStrings(filename, commentCharacters): |
551 if l.startswith(option): | 552 if l.startswith(option): |
552 values.append(l.split(delimiterChar)[1].strip()) | 553 values.append(l.split(delimiterChar)[1].strip()) |
553 return values | 554 return values |
554 | 555 |
555 class FakeSecHead(object): | 556 class FakeSecHead(object): |
577 | 578 |
578 Assumed to be sorted over time''' | 579 Assumed to be sorted over time''' |
579 objects = {} # dictionary of objects index by their id | 580 objects = {} # dictionary of objects index by their id |
580 firstInstants = {} | 581 firstInstants = {} |
581 | 582 |
582 infile = openCheck(filename, quitting = True) | 583 inputfile = openCheck(filename, quitting = True) |
583 | 584 |
584 # data = pd.read_csv(filename, skiprows=15, delimiter=';') | 585 # data = pd.read_csv(filename, skiprows=15, delimiter=';') |
585 # skip header: 15 lines + 1 | 586 # skip header: 15 lines + 1 |
586 for i in range(16): | 587 line = readline(inputfile, '*$') |
587 readline(infile) | 588 while len(line) > 0:#for line in inputfile: |
588 | |
589 for line in infile: | |
590 data = line.strip().split(';') | 589 data = line.strip().split(';') |
591 objNum = int(data[1]) | 590 objNum = int(data[1]) |
592 instant = int(float(data[0])*simulationStepsPerTimeUnit) | 591 instant = int(float(data[0])*simulationStepsPerTimeUnit) |
593 s = float(data[4]) | 592 s = float(data[4]) |
594 y = float(data[5]) | 593 y = float(data[5]) |
600 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) | 599 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) |
601 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() | 600 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() |
602 if (warmUpLastInstant == None or firstInstants[objNum] >= warmUpLastInstant) and objNum in objects: | 601 if (warmUpLastInstant == None or firstInstants[objNum] >= warmUpLastInstant) and objNum in objects: |
603 objects[objNum].timeInterval.last = instant | 602 objects[objNum].timeInterval.last = instant |
604 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) | 603 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) |
604 line = readline(inputfile, '*$') | |
605 | 605 |
606 return objects.values() | 606 return objects.values() |
607 | 607 |
608 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): | 608 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): |
609 '''Reads data from the trajectory data provided by NGSIM project | 609 '''Reads data from the trajectory data provided by NGSIM project |
610 and returns the list of Feature objects''' | 610 and returns the list of Feature objects''' |
611 objects = [] | 611 objects = [] |
612 | 612 |
613 infile = openCheck(filename, quitting = True) | 613 inputfile = openCheck(filename, quitting = True) |
614 | 614 |
615 def createObject(numbers): | 615 def createObject(numbers): |
616 firstFrameNum = int(numbers[1]) | 616 firstFrameNum = int(numbers[1]) |
617 # do the geometry and usertype | 617 # do the geometry and usertype |
618 | 618 |
632 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate | 632 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate |
633 obj.speeds = [float(numbers[11])] | 633 obj.speeds = [float(numbers[11])] |
634 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object | 634 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object |
635 return obj | 635 return obj |
636 | 636 |
637 numbers = infile.readline().strip().split() | 637 numbers = readline(inputfile).strip().split() |
638 if (len(numbers) > 0): | 638 if (len(numbers) > 0): |
639 obj = createObject(numbers) | 639 obj = createObject(numbers) |
640 | 640 |
641 for line in infile: | 641 for line in inputfile: |
642 numbers = line.strip().split() | 642 numbers = line.strip().split() |
643 if obj.getNum() != int(numbers[0]): | 643 if obj.getNum() != int(numbers[0]): |
644 # check and adapt the length to deal with issues in NGSIM data | 644 # check and adapt the length to deal with issues in NGSIM data |
645 if (obj.length() != obj.positions.length()): | 645 if (obj.length() != obj.positions.length()): |
646 print 'length pb with object %s (%d,%d)' % (obj.getNum(),obj.length(),obj.positions.length()) | 646 print 'length pb with object %s (%d,%d)' % (obj.getNum(),obj.length(),obj.positions.length()) |
663 if (obj.size[0] != float(numbers[8])): | 663 if (obj.size[0] != float(numbers[8])): |
664 print 'changed length obj %d' % (obj.getNum()) | 664 print 'changed length obj %d' % (obj.getNum()) |
665 if (obj.size[1] != float(numbers[9])): | 665 if (obj.size[1] != float(numbers[9])): |
666 print 'changed width obj %d' % (obj.getNum()) | 666 print 'changed width obj %d' % (obj.getNum()) |
667 | 667 |
668 infile.close() | 668 inputfile.close() |
669 return objects | 669 return objects |
670 | 670 |
671 def convertNgsimFile(inFile, outFile, append = False, nObjects = -1, sequenceNum = 0): | 671 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): |
672 '''Reads data from the trajectory data provided by NGSIM project | 672 '''Reads data from the trajectory data provided by NGSIM project |
673 and converts to our current format.''' | 673 and converts to our current format.''' |
674 if append: | 674 if append: |
675 out = openCheck(outFile,'a') | 675 out = openCheck(outputfile,'a') |
676 else: | 676 else: |
677 out = openCheck(outFile,'w') | 677 out = openCheck(outputfile,'w') |
678 nObjectsPerType = [0,0,0] | 678 nObjectsPerType = [0,0,0] |
679 | 679 |
680 features = loadNgsimFile(inFile, sequenceNum) | 680 features = loadNgsimFile(inputfile, sequenceNum) |
681 for f in features: | 681 for f in features: |
682 nObjectsPerType[f.userType-1] += 1 | 682 nObjectsPerType[f.userType-1] += 1 |
683 f.write(out) | 683 f.write(out) |
684 | 684 |
685 print nObjectsPerType | 685 print nObjectsPerType |