Mercurial Hosting > traffic-intelligence
annotate scripts/merge-features.py @ 1156:f7fbe624fff7
added helper functions for categorical variables
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Fri, 25 Sep 2020 11:56:59 -0400 |
parents | cc5cb04b04b0 |
children |
rev | line source |
---|---|
998
933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
1 #! /usr/bin/env python3 |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
2 |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
3 import sys, argparse, os.path, sqlite3 |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
4 from datetime import datetime, timedelta |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
5 |
1028
cc5cb04b04b0
major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
6 from trafficintelligence import cvutils, utils, moving, storage |
cc5cb04b04b0
major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
7 from trafficintelligence.metadata import connectDatabase, Site, VideoSequence, CameraView, getSite |
cc5cb04b04b0
major update using the trafficintelligence package name and install through pip
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
998
diff
changeset
|
8 |
971
9897a13772fb
added utils to load video sequence in metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
842
diff
changeset
|
9 timeConverter = utils.TimeConverter() |
9897a13772fb
added utils to load video sequence in metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
842
diff
changeset
|
10 |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
11 parser = argparse.ArgumentParser(description='The program merges feature trajectories recorded from the same site synchronously between start and end time.') |
983
7463c9bc846b
work in progress on script to manage large dataset with multiple sites
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
971
diff
changeset
|
12 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
13 parser.add_argument('-n', dest = 'siteId', help = 'site id or site name', required = True) |
971
9897a13772fb
added utils to load video sequence in metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
842
diff
changeset
|
14 parser.add_argument('-f', dest = 'startTime', help = 'time to start merging features (format %%Y-%%m-%%d %%H:%%M:%%S, eg 2011-06-22 10:00:39)', type = timeConverter.convert) # if not provided, take common time interval |
9897a13772fb
added utils to load video sequence in metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
842
diff
changeset
|
15 parser.add_argument('-l', dest = 'endTime', help = 'time to stop merging features (format %%Y-%%m-%%d %%H:%%M:%%S, eg 2011-06-22 10:00:39)', type = timeConverter.convert) |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
16 parser.add_argument('-o', dest = 'outputDBFilename', help = 'name of the output SQLite file', required = True) |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
17 |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
18 args = parser.parse_args() |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
19 |
985
668a85c963c3
work on processing and managing large video datasets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
983
diff
changeset
|
20 session = connectDatabase(args.metadataFilename) |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
21 |
971
9897a13772fb
added utils to load video sequence in metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
842
diff
changeset
|
22 site = getSite(session, args.siteId) |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
23 if site is None: |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
24 print('Site {} was not found in {}. Exiting'.format(args.siteId, args.metadataFilename)) |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
25 sys.exit() |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
26 else: |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
27 site = site[0] |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
28 |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
29 startTime = datetime.strptime(args.startTime, utils.datetimeFormat) |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
30 endTime = datetime.strptime(args.endTime, utils.datetimeFormat) |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
31 processInterval = moving.TimeInterval(startTime, endTime) |
836
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
32 cameraViews = session.query(CameraView).filter(CameraView.site == site).filter(CameraView.virtual == False) |
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
33 videoSequences = session.query(VideoSequence).filter(VideoSequence.virtual == False).order_by(VideoSequence.startTime.asc()).all() #.order_by(VideoSequence.cameraViewIdx) .filter(VideoSequence.startTime <= startTime) |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
34 videoSequences = [vs for vs in videoSequences if vs.cameraView in cameraViews] |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
35 #timeIntervals = [v.intersection(startTime, endTime) for v in videoSequences] |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
36 #cameraViews = set([v.cameraView for v in videoSequences]) |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
37 |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
38 videoSequences = {cv: [v for v in videoSequences if v.cameraView == cv] for cv in cameraViews} |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
39 timeIntervals = {} |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
40 for cv in videoSequences: |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
41 timeIntervals[cv] = moving.TimeInterval.unionIntervals([v.getTimeInterval() for v in videoSequences[cv]]) |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
42 |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
43 # intersection of the time interval (union) for each camera view |
998
933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
44 commonTimeInterval = list(timeIntervals.values())[0] |
828
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
45 for inter in timeIntervals.values()[1:]: |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
46 commonTimeInterval = moving.TimeInterval.intersection(commonTimeInterval, inter) |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
47 commonTimeInterval = moving.TimeInterval.intersection(commonTimeInterval, processInterval) |
14e4ad7c7420
work on merging data for synchronized views
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
diff
changeset
|
48 |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
49 if commonTimeInterval.empty(): |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
50 print('Empty time interval. Exiting') |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
51 sys.exit() |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
52 |
829 | 53 if len(set([cv.cameraType.frameRate for cv in cameraViews])) > 1: |
54 print('Different framerates of the cameras ({}) are not handled yet. Exiting'.format([cv.cameraType.frameRate for cv in cameraViews])) | |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
55 else: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
56 frameRate = cv.cameraType.frameRate |
829 | 57 |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
58 try: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
59 outConnection = sqlite3.connect(args.outputDBFilename) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
60 outCursor = outConnection.cursor() |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
61 storage.createTrajectoryTable(outCursor, 'positions') |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
62 storage.createTrajectoryTable(outCursor, 'velocities') |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
63 storage.createFeatureCorrespondenceTable(outCursor) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
64 outConnection.commit() |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
65 except sqlite3.OperationalError as error: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
66 storage.printDBError(error) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
67 sys.exit() |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
68 |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
69 dirname = os.path.split(args.metadataFilename)[0] |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
70 if len(dirname) == 0: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
71 dirname = '.' |
829 | 72 |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
73 newTrajectoryId = -1 |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
74 # first frame num is commonTimeInterval |
998
933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
75 for cv, vs in videoSequences.items(): |
933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
76 print(cv.idx, cv.description) |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
77 for videoSequence in vs: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
78 try: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
79 vsConnection = sqlite3.connect(dirname+os.path.sep+videoSequence.getDatabaseFilename()) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
80 vsCursor = vsConnection.cursor() |
837
e01cabca4c55
minor modifications to merge-features
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
836
diff
changeset
|
81 firstFrameNum = utils.deltaFrames(videoSequence.startTime, commonTimeInterval.first, frameRate) |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
82 lastFrameNum = (commonTimeInterval.last-videoSequence.startTime).seconds*frameRate |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
83 # positions table |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
84 vsCursor.execute('SELECT * FROM positions WHERE frame_number BETWEEN {} AND {} ORDER BY trajectory_id'.format(firstFrameNum, lastFrameNum)) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
85 featureIdCorrespondences = {} |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
86 currentTrajectoryId = -1 |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
87 for row in vsCursor: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
88 if row[0] != currentTrajectoryId: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
89 currentTrajectoryId = row[0] |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
90 newTrajectoryId += 1 |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
91 featureIdCorrespondences[currentTrajectoryId] = newTrajectoryId |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
92 outCursor.execute(storage.insertTrajectoryQuery('positions'), (newTrajectoryId, row[1]-firstFrameNum, row[2], row[3])) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
93 # velocities table |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
94 for row in vsCursor: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
95 outCursor.execute(storage.insertTrajectoryQuery('velocities'), (featureIdCorrespondences[row[0]], row[1]-firstFrameNum, row[2], row[3])) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
96 # saving the id correspondences |
998
933670761a57
updated code to python 3 (tests pass and scripts run, but non-executed parts of code are probably still not correct)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
985
diff
changeset
|
97 for oldId, newId in featureIdCorrespondences.items(): |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
98 outCursor.execute("INSERT INTO feature_correspondences (trajectory_id, source_dbname, db_trajectory_id) VALUES ({},\"{}\",{})".format(newId, videoSequence.name, oldId)) |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
99 outConnection.commit() |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
100 except sqlite3.OperationalError as error: |
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
101 storage.printDBError(error) |
836
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
102 |
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
103 # save the information of the new virtual sequence and camera view in the metadata |
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
104 mergedCameraView = CameraView('merged', None, site, cv.cameraType, None, None, virtual = True) |
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
105 session.add(mergedCameraView) |
7058a40a4bbc
updated metadata and code to merge features from different cameras
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
830
diff
changeset
|
106 session.add(VideoSequence('merged', commonTimeInterval.first, commonTimeInterval.last-commonTimeInterval.first, mergedCameraView, virtual = True)) |
830
2a5856961933
first working version of feature merging (works with feature grouping)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
829
diff
changeset
|
107 session.commit() |