Mercurial Hosting > traffic-intelligence
changeset 1028:cc5cb04b04b0
major update using the trafficintelligence package name and install through pip
line wrap: on
line diff
--- a/Makefile Fri Jun 15 11:18:43 2018 -0400 +++ b/Makefile Fri Jun 15 11:19:10 2018 -0400 @@ -1,4 +1,5 @@ INSTALL_DIR = /usr/local/bin +PYTHON_LIB_DIR = cexe: @cd c && make feature-based-tracking @@ -10,16 +11,21 @@ @cd c && make clean @cd python && rm *.pyc -install: cexe +installpython: + @echo "=========================================" + @echo "Installing Python modules and scripts" + @tar cf /tmp/trafficintelligence.tar setup.py trafficintelligence + @gzip /tmp/trafficintelligence.tar + @pip3 install /tmp/trafficintelligence.tar.gz + @rm /tmp/trafficintelligence.tar.gz + @cp scripts/* $(INSTALL_DIR) + +install: cexe installpython @echo "=========================================" @echo "Installing for Linux" @echo "=========================================" - @echo "Copying feature-based tracking executable" + @echo "Installing feature-based tracking executable" @cp bin/feature-based-tracking /usr/local/bin - @echo "=========================================" - @echo "Copying Python scripts" - @cp scripts/* $(INSTALL_DIR) - uninstall: @echo "Uninstalling for Linux" rm $(INSTALL_DIR)/feature-based-tracking
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/python-requirements.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,9 @@ +matplotlib +numpy +scipy +scikit-image +scikit-learn +shapely +pandas +munkres +sqlalchemy
--- a/python/base.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -'''Module for few base classes to avoid issues of circular import''' - -class VideoFilenameAddable(object): - 'Base class with the capability to attach a video filename' - - def setVideoFilename(self, videoFilename): - self.videoFilename = videoFilename
--- a/python/cvutils.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,663 +0,0 @@ -#! /usr/bin/env python -'''Image/Video utilities''' - -import utils, moving - -try: - import cv2 - opencvAvailable = True -except ImportError: - print('OpenCV library could not be loaded (video replay functions will not be available)') # TODO change to logging module - opencvAvailable = False -try: - import skimage - skimageAvailable = True -except ImportError: - print('Scikit-image library could not be loaded (HoG-based classification methods will not be available)') - skimageAvailable = False - -from sys import stdout -from os import listdir -from subprocess import run -from math import floor, log10, ceil - -from numpy import dot, array, append, float32, loadtxt, savetxt, append, zeros, ones, identity, abs as npabs, logical_and, unravel_index, sum as npsum, isnan, mgrid, median, floor as npfloor, ceil as npceil -from numpy.linalg import inv -from matplotlib.mlab import find -from matplotlib.pyplot import imread, imsave - -videoFilenameExtensions = ['mov', 'avi', 'mp4', 'MOV', 'AVI', 'MP4'] -trackerExe = 'feature-based-tracking' -#importaggdraw # agg on top of PIL (antialiased drawing) - -cvRed = {'default': (0,0,255), - 'colorblind': (0,114,178)} -cvGreen = {'default': (0,255,0), - 'colorblind': (0,158,115)} -cvBlue = {'default': (255,0,0), - 'colorblind': (213,94,0)} -cvCyan = {'default': (255, 255, 0), - 'colorblind': (240,228,66)} -cvYellow = {'default': (0, 255, 255), - 'colorblind': (86,180,233)} -cvMagenta = {'default': (255, 0, 255), - 'colorblind': (204,121,167)} -cvWhite = {k: (255, 255, 255) for k in ['default', 'colorblind']} -cvBlack = {k: (0,0,0) for k in ['default', 'colorblind']} - -cvColors3 = {k: utils.PlottingPropertyValues([cvRed[k], cvGreen[k], cvBlue[k]]) for k in ['default', 'colorblind']} -cvColors = {k: utils.PlottingPropertyValues([cvRed[k], cvGreen[k], cvBlue[k], cvCyan[k], cvYellow[k], cvMagenta[k], cvWhite[k], cvBlack[k]]) for k in ['default', 'colorblind']} - -def quitKey(key): - return chr(key&255)== 'q' or chr(key&255) == 'Q' - -def saveKey(key): - return chr(key&255) == 's' - -def int2FOURCC(x): - fourcc = '' - for i in range(4): - fourcc += chr((x >> 8*i)&255) - return fourcc - -def rgb2gray(rgb): - return dot(rgb[...,:3], [0.299, 0.587, 0.144]) - -def matlab2PointCorrespondences(filename): - '''Loads and converts the point correspondences saved - by the matlab camera calibration tool''' - points = loadtxt(filename, delimiter=',') - savetxt(utils.removeExtension(filename)+'-point-correspondences.txt',append(points[:,:2].T, points[:,3:].T, axis=0)) - -def loadPointCorrespondences(filename): - '''Loads and returns the corresponding points in world (first 2 lines) and image spaces (last 2 lines)''' - points = loadtxt(filename, dtype=float32) - return (points[:2,:].T, points[2:,:].T) # (world points, image points) - -def cvMatToArray(cvmat): - '''Converts an OpenCV CvMat to numpy array.''' - print('Deprecated, use new interface') - a = zeros((cvmat.rows, cvmat.cols))#array([[0.0]*cvmat.width]*cvmat.height) - for i in range(cvmat.rows): - for j in range(cvmat.cols): - a[i,j] = cvmat[i,j] - return a - -def createWhiteImage(height, width, filename): - img = ones((height, width, 3), uint8)*255 - imsave(filename, img) - -if opencvAvailable: - def computeHomography(srcPoints, dstPoints, method=0, ransacReprojThreshold=3.0): - '''Returns the homography matrix mapping from srcPoints to dstPoints (dimension Nx2)''' - H, mask = cv2.findHomography(srcPoints, dstPoints, method, ransacReprojThreshold) - return H - - def cvPlot(img, positions, color, lastCoordinate = None, **kwargs): - if lastCoordinate is None: - last = positions.length()-1 - elif lastCoordinate >=0: - last = min(positions.length()-1, lastCoordinate) - for i in range(0, last): - cv2.line(img, positions[i].asint().astuple(), positions[i+1].asint().astuple(), color, **kwargs) - - def cvImshow(windowName, img, rescale = 1.0): - 'Rescales the image (in particular if too large)' - from cv2 import resize - if rescale != 1.: - size = (int(round(img.shape[1]*rescale)), int(round(img.shape[0]*rescale))) - resizedImg = resize(img, size) - cv2.imshow(windowName, resizedImg) - else: - cv2.imshow(windowName, img) - - def computeUndistortMaps(width, height, undistortedImageMultiplication, intrinsicCameraMatrix, distortionCoefficients): - newImgSize = (int(round(width*undistortedImageMultiplication)), int(round(height*undistortedImageMultiplication))) - newCameraMatrix = cv2.getDefaultNewCameraMatrix(intrinsicCameraMatrix, newImgSize, True) - return cv2.initUndistortRectifyMap(intrinsicCameraMatrix, array(distortionCoefficients), None, newCameraMatrix, newImgSize, cv2.CV_32FC1), newCameraMatrix - - def playVideo(filenames, windowNames = None, firstFrameNums = None, frameRate = -1, interactive = False, printFrames = True, text = None, rescale = 1., step = 1, colorBlind = False): - '''Plays the video(s)''' - if colorBlind: - colorType = 'colorblind' - else: - colorType = 'default' - if len(filenames) == 0: - print('Empty filename list') - return - if windowNames is None: - windowNames = ['frame{}'.format(i) for i in range(len(filenames))] - wait = 5 - if rescale == 1.: - for windowName in windowNames: - cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) - if frameRate > 0: - wait = int(round(1000./frameRate)) - if interactive: - wait = 0 - captures = [cv2.VideoCapture(fn) for fn in filenames] - if array([cap.isOpened() for cap in captures]).all(): - key = -1 - ret = True - nFramesShown = 0 - if firstFrameNums is not None: - for i in range(len(captures)): - captures[i].set(cv2.CAP_PROP_POS_FRAMES, firstFrameNums[i]) - while ret and not quitKey(key): - rets = [] - images = [] - for cap in captures: - ret, img = cap.read() - rets.append(ret) - images.append(img) - ret = array(rets).all() - if ret: - if printFrames: - print('frame shown {0}'.format(nFramesShown)) - for i in range(len(filenames)): - if text is not None: - cv2.putText(images[i], text, (10,50), cv2.FONT_HERSHEY_PLAIN, 1, cvRed[colorType]) - cvImshow(windowNames[i], images[i], rescale) # cv2.imshow('frame', img) - key = cv2.waitKey(wait) - if saveKey(key): - cv2.imwrite('image-{}.png'.format(frameNum), img) - nFramesShown += step - if step > 1: - for i in range(len(captures)): - captures[i].set(cv2.CAP_PROP_POS_FRAMES, firstFrameNums[i]+nFramesShown) - cv2.destroyAllWindows() - else: - print('Video captures for {} failed'.format(filenames)) - - def infoVideo(filename): - '''Provides all available info on video ''' - cvPropertyNames = {cv2.CAP_PROP_FORMAT: "format", - cv2.CAP_PROP_FOURCC: "codec (fourcc)", - cv2.CAP_PROP_FPS: "fps", - cv2.CAP_PROP_FRAME_COUNT: "number of frames", - cv2.CAP_PROP_FRAME_HEIGHT: "heigh", - cv2.CAP_PROP_FRAME_WIDTH: "width", - cv2.CAP_PROP_RECTIFICATION: "rectification", - cv2.CAP_PROP_SATURATION: "saturation"} - capture = cv2.VideoCapture(filename) - videoProperties = {} - if capture.isOpened(): - for cvprop in [#cv2.CAP_PROP_BRIGHTNESS - #cv2.CAP_PROP_CONTRAST - #cv2.CAP_PROP_CONVERT_RGB - #cv2.CAP_PROP_EXPOSURE - cv2.CAP_PROP_FORMAT, - cv2.CAP_PROP_FOURCC, - cv2.CAP_PROP_FPS, - cv2.CAP_PROP_FRAME_COUNT, - cv2.CAP_PROP_FRAME_HEIGHT, - cv2.CAP_PROP_FRAME_WIDTH, - #cv2.CAP_PROP_GAIN, - #cv2.CAP_PROP_HUE - #cv2.CAP_PROP_MODE - #cv2.CAP_PROP_POS_AVI_RATIO - #cv2.CAP_PROP_POS_FRAMES - #cv2.CAP_PROP_POS_MSEC - #cv2.CAP_PROP_RECTIFICATION, - #cv2.CAP_PROP_SATURATION - ]: - prop = capture.get(cvprop) - if cvprop == cv2.CAP_PROP_FOURCC and prop > 0: - prop = int2FOURCC(int(prop)) - videoProperties[cvPropertyNames[cvprop]] = prop - else: - print('Video capture for {} failed'.format(filename)) - return videoProperties - - def getImagesFromVideo(videoFilename, firstFrameNum = 0, lastFrameNum = 1, step = 1, saveImage = False, outputPrefix = 'image'): - '''Returns nFrames images from the video sequence''' - images = [] - capture = cv2.VideoCapture(videoFilename) - if capture.isOpened(): - rawCount = capture.get(cv2.CAP_PROP_FRAME_COUNT) - if rawCount < 0: - rawCount = lastFrameNum+1 - nDigits = int(floor(log10(rawCount)))+1 - ret = False - capture.set(cv2.CAP_PROP_POS_FRAMES, firstFrameNum) - frameNum = firstFrameNum - while frameNum<lastFrameNum and frameNum<rawCount: - ret, img = capture.read() - i = 0 - while not ret and i<10: - ret, img = capture.read() - i += 1 - if img is not None and img.size>0: - if saveImage: - frameNumStr = format(frameNum, '0{}d'.format(nDigits)) - cv2.imwrite(outputPrefix+frameNumStr+'.png', img) - else: - images.append(img) - frameNum +=step - if step > 1: - capture.set(cv2.CAP_PROP_POS_FRAMES, frameNum) - capture.release() - else: - print('Video capture for {} failed'.format(videoFilename)) - return images - - def getFPS(videoFilename): - capture = cv2.VideoCapture(videoFilename) - if capture.isOpened(): - fps = capture.get(cv2.CAP_PROP_FPS) - capture.release() - return fps - else: - print('Video capture for {} failed'.format(videoFilename)) - return None - - def imageBoxSize(obj, frameNum, width, height, px = 0.2, py = 0.2): - 'Computes the bounding box size of object at frameNum' - x = [] - y = [] - if obj.hasFeatures(): - for f in obj.getFeatures(): - if f.existsAtInstant(frameNum): - p = f.getPositionAtInstant(frameNum) - x.append(p.x) - y.append(p.y) - xmin = min(x) - xmax = max(x) - ymin = min(y) - ymax = max(y) - xMm = px * (xmax - xmin) - yMm = py * (ymax - ymin) - a = max(ymax - ymin + (2 * yMm), xmax - (xmin + 2 * xMm)) - yCropMin = int(max(0, .5 * (ymin + ymax - a))) - yCropMax = int(min(height - 1, .5 * (ymin + ymax + a))) - xCropMin = int(max(0, .5 * (xmin + xmax - a))) - xCropMax = int(min(width - 1, .5 * (xmin + xmax + a))) - return yCropMin, yCropMax, xCropMin, xCropMax - - def imageBox(img, obj, frameNum, width, height, px = 0.2, py = 0.2, minNPixels = 800): - 'Computes the bounding box of object at frameNum' - yCropMin, yCropMax, xCropMin, xCropMax = imageBoxSize(obj, frameNum, width, height, px, py) - if yCropMax != yCropMin and xCropMax != xCropMin and (yCropMax - yCropMin) * (xCropMax - xCropMin) > minNPixels: - return img[yCropMin : yCropMax, xCropMin : xCropMax] - else: - return None - - def tracking(configFilename, grouping, videoFilename = None, dbFilename = None, homographyFilename = None, maskFilename = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, dryRun = False): - '''Runs the tracker in a subprocess - if grouping is True, it is feature grouping - otherwise it is feature tracking''' - if grouping: - trackingMode = '--gf' - else: - trackingMode = '--tf' - cmd = [trackerExe, configFilename, trackingMode, '--quiet'] - - if videoFilename is not None: - cmd += ['--video-filename', videoFilename] - if dbFilename is not None: - cmd += ['--database-filename', dbFilename] - if homographyFilename is not None: - cmd += ['--homography-filename', homographyFilename] - if maskFilename is not None: - cmd += ['--mask-filename', maskFilename] - if undistort: - cmd += ['--undistort', 'true'] - if intrinsicCameraMatrix is not None: # we currently have to save a file - from time import time - intrinsicCameraFilename = '/tmp/intrinsic-{}.txt'.format(time()) - savetxt(intrinsicCameraFilename, intrinsicCameraMatrix) - cmd += ['--intrinsic-camera-filename', intrinsicCameraFilename] - if distortionCoefficients is not None: - cmd += ['--distortion-coefficients '+' '.join([str(x) for x in distortionCoefficients])] - if dryRun: - print(cmd) - else: - run(cmd) - - def displayTrajectories(videoFilename, objects, boundingBoxes = {}, homography = None, firstFrameNum = 0, lastFrameNumArg = None, printFrames = True, rescale = 1., nFramesStep = 1, saveAllImages = False, nZerosFilenameArg = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., annotations = [], gtMatches = {}, toMatches = {}, colorBlind = False): - '''Displays the objects overlaid frame by frame over the video ''' - if colorBlind: - colorType = 'colorblind' - else: - colorType = 'default' - - capture = cv2.VideoCapture(videoFilename) - width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)) - height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - windowName = 'frame' - if rescale == 1.: - cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) - - if undistort: # setup undistortion - [map1, map2], newCameraMatrix = computeUndistortMaps(width, height, undistortedImageMultiplication, intrinsicCameraMatrix, distortionCoefficients) - if capture.isOpened(): - key = -1 - ret = True - frameNum = firstFrameNum - capture.set(cv2.CAP_PROP_POS_FRAMES, firstFrameNum) - if lastFrameNumArg is None: - lastFrameNum = float("inf") - else: - lastFrameNum = lastFrameNumArg - if nZerosFilenameArg is None: - if lastFrameNumArg is None: - nZerosFilename = int(ceil(log10(objects[-1].getLastInstant()))) - else: - nZerosFilename = int(ceil(log10(lastFrameNum))) - else: - nZerosFilename = nZerosFilenameArg - while ret and not quitKey(key) and frameNum <= lastFrameNum: - ret, img = capture.read() - if ret: - if undistort: - img = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR) - if printFrames: - print('frame {0}'.format(frameNum)) - # plot objects - for obj in objects[:]: - if obj.existsAtInstant(frameNum): - if not hasattr(obj, 'projectedPositions'): - obj.projectedPositions = obj.getPositions().homographyProject(homography) - if undistort: - obj.projectedPositions = obj.projectedPositions.newCameraProject(newCameraMatrix) - cvPlot(img, obj.projectedPositions, cvColors[colorType][obj.getNum()], frameNum-obj.getFirstInstant()) - if frameNum not in boundingBoxes and obj.hasFeatures(): - yCropMin, yCropMax, xCropMin, xCropMax = imageBoxSize(obj, frameNum, homography, width, height) - cv2.rectangle(img, (xCropMin, yCropMin), (xCropMax, yCropMax), cvBlue[colorType], 1) - objDescription = '{} '.format(obj.num) - if moving.userTypeNames[obj.userType] != 'unknown': - objDescription += moving.userTypeNames[obj.userType][0].upper() - if len(annotations) > 0: # if we loaded annotations, but there is no match - if frameNum not in toMatches[obj.getNum()]: - objDescription += " FA" - cv2.putText(img, objDescription, obj.projectedPositions[frameNum-obj.getFirstInstant()].asint().astuple(), cv2.FONT_HERSHEY_PLAIN, 1, cvColors[colorType][obj.getNum()]) - if obj.getLastInstant() == frameNum: - objects.remove(obj) - # plot object bounding boxes - if frameNum in boundingBoxes: - for rect in boundingBoxes[frameNum]: - cv2.rectangle(img, rect[0].asint().astuple(), rect[1].asint().astuple(), cvColors[colorType][obj.getNum()]) - # plot ground truth - if len(annotations) > 0: - for gt in annotations: - if gt.existsAtInstant(frameNum): - if frameNum in gtMatches[gt.getNum()]: - color = cvColors[colorType][gtMatches[gt.getNum()][frameNum]] # same color as object - else: - color = cvRed[colorType] - cv2.putText(img, 'Miss', gt.topLeftPositions[frameNum-gt.getFirstInstant()].asint().astuple(), cv2.FONT_HERSHEY_PLAIN, 1, color) - cv2.rectangle(img, gt.topLeftPositions[frameNum-gt.getFirstInstant()].asint().astuple(), gt.bottomRightPositions[frameNum-gt.getFirstInstant()].asint().astuple(), color) - # saving images and going to next - if not saveAllImages: - cvImshow(windowName, img, rescale) - key = cv2.waitKey() - if saveAllImages or saveKey(key): - cv2.imwrite('image-{{:0{}}}.png'.format(nZerosFilename).format(frameNum), img) - frameNum += nFramesStep - if nFramesStep > 1: - capture.set(cv2.CAP_PROP_POS_FRAMES, frameNum) - cv2.destroyAllWindows() - else: - print('Cannot load file ' + videoFilename) - - def computeHomographyFromPDTV(camera): - '''Returns the homography matrix at ground level from PDTV camera - https://bitbucket.org/hakanardo/pdtv''' - # camera = pdtv.load(cameraFilename) - srcPoints = [[x,y] for x, y in zip([1.,2.,2.,1.],[1.,1.,2.,2.])] # need floats!! - dstPoints = [] - for srcPoint in srcPoints: - projected = camera.image_to_world(tuple(srcPoint)) - dstPoints.append([projected[0], projected[1]]) - H, mask = cv2.findHomography(array(srcPoints), array(dstPoints), method = 0) # No need for different methods for finding homography - return H - - def getIntrinsicCameraMatrix(cameraData): - return array([[cameraData['f']*cameraData['Sx']/cameraData['dx'], 0, cameraData['Cx']], - [0, cameraData['f']/cameraData['dy'], cameraData['Cy']], - [0, 0, 1.]]) - - def getDistortionCoefficients(cameraData): - return array([cameraData['k']]+4*[0]) - - def undistortedCoordinates(map1, map2, x, y, maxDistance = 1.): - '''Returns the coordinates of a point in undistorted image - map1 and map2 are the mapping functions from undistorted image - to distorted (original image) - map1(x,y) = originalx, originaly''' - distx = npabs(map1-x) - disty = npabs(map2-y) - indices = logical_and(distx<maxDistance, disty<maxDistance) - closeCoordinates = unravel_index(find(indices), distx.shape) # returns i,j, ie y,x - xWeights = 1-distx[indices] - yWeights = 1-disty[indices] - return dot(xWeights, closeCoordinates[1])/npsum(xWeights), dot(yWeights, closeCoordinates[0])/npsum(yWeights) - - def undistortTrajectoryFromCVMapping(map1, map2, t): - '''test 'perfect' inversion''' - undistortedTrajectory = moving.Trajectory() - for i,p in enumerate(t): - res = undistortedCoordinates(map1, map2, p.x,p.y) - if not isnan(res).any(): - undistortedTrajectory.addPositionXY(res[0], res[1]) - else: - print('{} {} {}'.format(i,p,res)) - return undistortedTrajectory - - def computeInverseMapping(originalImageSize, map1, map2): - 'Computes inverse mapping from maps provided by cv2.initUndistortRectifyMap' - invMap1 = -ones(originalImageSize) - invMap2 = -ones(originalImageSize) - for x in range(0,originalImageSize[1]): - for y in range(0,originalImageSize[0]): - res = undistortedCoordinates(x,y, map1, map2) - if not isnan(res).any(): - invMap1[y,x] = res[0] - invMap2[y,x] = res[1] - return invMap1, invMap2 - - def intrinsicCameraCalibration(path, checkerBoardSize=[6,7], secondPassSearch=False, display=False, fixK2 = True, fixK3 = True, zeroTangent = True): - ''' Camera calibration searches through all the images (jpg or png) located - in _path_ for matches to a checkerboard pattern of size checkboardSize. - These images should all be of the same camera with the same resolution. - - For best results, use an asymetric board and ensure that the image has - very high contrast, including the background. - - cherckerBoardSize is the number of internal corners (7x10 squares have 6x9 internal corners) - - The code below is based off of: - https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_calib3d/py_calibration/py_calibration.html - Modified by Paul St-Aubin - ''' - import glob, os - - # termination criteria - criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) - - # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0) - objp = zeros((checkerBoardSize[0]*checkerBoardSize[1],3), float32) - objp[:,:2] = mgrid[0:checkerBoardSize[1],0:checkerBoardSize[0]].T.reshape(-1,2) - - # Arrays to store object points and image points from all the images. - objpoints = [] # 3d point in real world space - imgpoints = [] # 2d points in image plane. - - ## Loop throuhg all images in _path_ - images = glob.glob(os.path.join(path,'*.[jJ][pP][gG]'))+glob.glob(os.path.join(path,'*.[jJ][pP][eE][gG]'))+glob.glob(os.path.join(path,'*.[pP][nN][gG]')) - for fname in images: - img = cv2.imread(fname) - gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) - - # Find the chess board corners - ret, corners = cv2.findChessboardCorners(gray, (checkerBoardSize[1],checkerBoardSize[0]), None) - - # If found, add object points, image points (after refining them) - if ret: - print('Found pattern in '+fname) - - if secondPassSearch: - corners = cv2.cornerSubPix(gray, corners, (11,11), (-1,-1), criteria) - - objpoints.append(objp) - imgpoints.append(corners) - - # Draw and display the corners - if display: - cv2.drawChessboardCorners(img, (checkerBoardSize[1],checkerBoardSize[0]), corners, ret) - if img is not None: - cv2.imshow('img',img) - cv2.waitKey(0) - else: - print('Pattern not found in '+fname) - ## Close up image loading and calibrate - cv2.destroyAllWindows() - if len(objpoints) == 0 or len(imgpoints) == 0: - return None - try: - flags = 0 - if fixK2: - flags += cv2.CALIB_FIX_K2 - if fixK3: - flags += cv2.CALIB_FIX_K3 - if zeroTangent: - flags += cv2.CALIB_ZERO_TANGENT_DIST - ret, camera_matrix, dist_coeffs, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None, flags = flags) - except NameError: - return None - savetxt('intrinsic-camera.txt', camera_matrix) - print('error: {}'.format(ret)) - return camera_matrix, dist_coeffs - - def undistortImage(img, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., interpolation=cv2.INTER_LINEAR): - '''Undistorts the image passed in argument''' - width = img.shape[1] - height = img.shape[0] - [map1, map2] = computeUndistortMaps(width, height, undistortedImageMultiplication, intrinsicCameraMatrix, distortionCoefficients) - return cv2.remap(img, map1, map2, interpolation=interpolation) - -def homographyProject(points, homography, output3D = False): - '''Returns the coordinates of the points (2xN array) projected through homography''' - if points.shape[0] != 2: - raise Exception('points of dimension {}'.format(points.shape)) - - if homography is not None and homography.size>0: - if output3D: - outputDim = 3 - else: - outputDim = 2 - augmentedPoints = append(points,[[1]*points.shape[1]], 0) # 3xN - prod = dot(homography, augmentedPoints) - return prod[:outputDim,:]/prod[2] - elif output3D: - return append(points,[[1]*points.shape[1]], 0) # 3xN - else: - return points - -def imageToWorldProject(points, intrinsicCameraMatrix = None, distortionCoefficients = None, homography = None): - '''Projects points (2xN array) from image (video) space to world space - 1. through undistorting if provided by intrinsic camera matrix and distortion coefficients - 2. through homograph projection (from ideal point (no camera) to world)''' - if points.shape[0] != 2: - raise Exception('points of dimension {}'.format(points.shape)) - - if intrinsicCameraMatrix is not None and distortionCoefficients is not None: - undistortedPoints = cv2.undistortPoints(points.T.reshape(1,points.shape[1], 2), intrinsicCameraMatrix, distortionCoefficients).reshape(-1,2) - return homographyProject(undistortedPoints.T, homography) - else: - return homographyProject(points, homography) - -def worldToImageProject(points, intrinsicCameraMatrix = None, distortionCoefficients = None, homography = None): - '''Projects points (2xN array) from image (video) space to world space - 1. through undistorting if provided by intrinsic camera matrix and distortion coefficients - 2. through homograph projection (from ideal point (no camera) to world)''' - if points.shape[0] != 2: - raise Exception('points of dimension {}'.format(points.shape)) - - if intrinsicCameraMatrix is not None and distortionCoefficients is not None: - projected3D = homographyProject(points, homography, True) - projected, jacobian = cv2.projectPoints(projected3D.T, (0.,0.,0.), (0.,0.,0.), intrinsicCameraMatrix, distortionCoefficients) # in: 3xN, out: 2x1xN - return projected.reshape(-1,2).T - else: - return homographyProject(points, homography) - -def newCameraProject(points, newCameraMatrix): - '''Projects points (2xN array) as if seen by camera - (or reverse by inverting the camera matrix)''' - if points.shape[0] != 2: - raise Exception('points of dimension {}'.format(points.shape)) - - if newCameraMatrix is not None: - augmentedPoints = append(points,[[1]*points.shape[1]], 0) # 3xN - projected = dot(newCameraMatrix, augmentedPoints) - return projected[:2,:] - else: - return points - -if opencvAvailable: - def computeTranslation(img1, img2, img1Points, maxTranslation2, minNMatches, windowSize = (5,5), level = 5, criteria = (cv2.TERM_CRITERIA_EPS, 0, 0.01)): - '''Computes the translation of img2 with respect to img1 - (loaded using OpenCV as numpy arrays) - img1Points are used to compute the translation - - TODO add diagnostic if data is all over the place, and it most likely is not a translation (eg zoom, other non linear distortion)''' - - nextPoints = array([]) - (img2Points, status, track_error) = cv2.calcOpticalFlowPyrLK(img1, img2, img1Points, nextPoints, winSize=windowSize, maxLevel=level, criteria=criteria) - # calcOpticalFlowPyrLK(prevImg, nextImg, prevPts[, nextPts[, status[, err[, winSize[, maxLevel[, criteria[, derivLambda[, flags]]]]]]]]) -> nextPts, status, err - delta = [] - for (k, (p1,p2)) in enumerate(zip(img1Points, img2Points)): - if status[k] == 1: - dp = p2-p1 - d = npsum(dp**2) - if d < maxTranslation2: - delta.append(dp) - if len(delta) >= minNMatches: - return median(delta, axis=0) - else: - print(dp) - return None - -if skimageAvailable: - from skimage.feature import hog - from skimage import color, transform - - def HOG(image, rescaleSize = (64, 64), orientations = 9, pixelsPerCell = (8,8), cellsPerBlock = (2,2), blockNorm = 'L1', visualize = False, transformSqrt = False): - bwImg = color.rgb2gray(image) - inputImg = transform.resize(bwImg, rescaleSize) - features = hog(inputImg, orientations, pixelsPerCell, cellsPerBlock, blockNorm, visualize, transformSqrt, True) - if visualize: - from matplotlib.pyplot import imshow, figure, subplot - hogViz = features[1] - features = features[0] - figure() - subplot(1,2,1) - imshow(inputImg) - subplot(1,2,2) - imshow(hogViz) - return float32(features) - - def createHOGTrainingSet(imageDirectory, classLabel, rescaleSize = (64,64), orientations = 9, pixelsPerCell = (8,8), blockNorm = 'L1', cellsPerBlock = (2, 2), visualize = False, transformSqrt = False): - inputData = [] - for filename in listdir(imageDirectory): - img = imread(imageDirectory+filename) - features = HOG(img, rescaleSize, orientations, pixelsPerCell, cellsPerBlock, blockNorm, visualize, transformSqrt) - inputData.append(features) - - nImages = len(inputData) - return array(inputData, dtype = float32), array([classLabel]*nImages) - - -######################### -# running tests -######################### - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/cvutils.txt') - #suite = doctest.DocTestSuite() - unittest.TextTestRunner().run(suite) - #doctest.testmod() - #doctest.testfile("example.txt")
--- a/python/events.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,341 +0,0 @@ -#! /usr/bin/env python -'''Libraries for events -Interactions, pedestrian crossing...''' - -import moving, prediction, indicators, utils, cvutils, ml -from base import VideoFilenameAddable - -import numpy as np - -import multiprocessing -import itertools, logging - - -def findRoute(prototypes,objects,i,j,noiseEntryNums,noiseExitNums,minSimilarity= 0.3, spatialThreshold=1.0, delta=180): - if i[0] not in noiseEntryNums: - prototypesRoutes= [ x for x in sorted(prototypes.keys()) if i[0]==x[0]] - elif i[1] not in noiseExitNums: - prototypesRoutes=[ x for x in sorted(prototypes.keys()) if i[1]==x[1]] - else: - prototypesRoutes=[x for x in sorted(prototypes.keys())] - routeSim={} - lcss = utils.LCSS(similarityFunc=lambda x,y: (distanceForLCSS(x,y) <= spatialThreshold),delta=delta) - for y in prototypesRoutes: - if y in prototypes: - prototypesIDs=prototypes[y] - similarity=[] - for x in prototypesIDs: - s=lcss.computeNormalized(objects[j].positions, objects[x].positions) - similarity.append(s) - routeSim[y]=max(similarity) - route=max(routeSim, key=routeSim.get) - if routeSim[route]>=minSimilarity: - return route - else: - return i - -def getRoute(obj,prototypes,objects,noiseEntryNums,noiseExitNums,useDestination=True): - route=(obj.startRouteID,obj.endRouteID) - if useDestination: - if route not in prototypes: - route= findRoute(prototypes,objects,route,obj.getNum(),noiseEntryNums,noiseExitNums) - return route - -class Interaction(moving.STObject, VideoFilenameAddable): - '''Class for an interaction between two road users - or a road user and an obstacle - - link to the moving objects - contains the indicators in a dictionary with the names as keys - ''' - - categories = {'Head On': 0, - 'rearend': 1, - 'side': 2, - 'parallel': 3} - - indicatorNames = ['Collision Course Dot Product', - 'Collision Course Angle', - 'Distance', - 'Minimum Distance', - 'Velocity Angle', - 'Speed Differential', - 'Collision Probability', - 'Time to Collision', # 7 - 'Probability of Successful Evasive Action', - 'predicted Post Encroachment Time', - 'Post Encroachment Time'] - - indicatorNameToIndices = utils.inverseEnumeration(indicatorNames) - - indicatorShortNames = ['CCDP', - 'CCA', - 'Dist', - 'MinDist', - 'VA', - 'SD', - 'PoC', - 'TTC', - 'P(SEA)', - 'pPET', - 'PET'] - - indicatorUnits = ['', - 'rad', - 'm', - 'm', - 'rad', - 'm/s', - '', - 's', - '', - 's', - 's'] - - timeIndicators = ['Time to Collision', 'predicted Post Encroachment Time'] - - def __init__(self, num = None, timeInterval = None, roaduserNum1 = None, roaduserNum2 = None, roadUser1 = None, roadUser2 = None, categoryNum = None): - moving.STObject.__init__(self, num, timeInterval) - if timeInterval is None and roadUser1 is not None and roadUser2 is not None: - self.timeInterval = roadUser1.commonTimeInterval(roadUser2) - self.roadUser1 = roadUser1 - self.roadUser2 = roadUser2 - if roaduserNum1 is not None and roaduserNum2 is not None: - self.roadUserNumbers = set([roaduserNum1, roaduserNum2]) - elif roadUser1 is not None and roadUser2 is not None: - self.roadUserNumbers = set([roadUser1.getNum(), roadUser2.getNum()]) - else: - self.roadUserNumbers = None - self.categoryNum = categoryNum - self.indicators = {} - self.interactionInterval = None - # list for collison points and crossing zones - self.collisionPoints = None - self.crossingZones = None - - def getRoadUserNumbers(self): - return self.roadUserNumbers - - def setRoadUsers(self, objects): - nums = sorted(list(self.getRoadUserNumbers())) - if nums[0]<len(objects) and objects[nums[0]].getNum() == nums[0]: - self.roadUser1 = objects[nums[0]] - if nums[1]<len(objects) and objects[nums[1]].getNum() == nums[1]: - self.roadUser2 = objects[nums[1]] - - if self.roadUser1 is None or self.roadUser2 is None: - self.roadUser1 = None - self.roadUser2 = None - i = 0 - while i < len(objects) and self.roadUser2 is None: - if objects[i].getNum() in nums: - if self.roadUser1 is None: - self.roadUser1 = objects[i] - else: - self.roadUser2 = objects[i] - i += 1 - - def getIndicator(self, indicatorName): - return self.indicators.get(indicatorName, None) - - def addIndicator(self, indicator): - if indicator is not None: - self.indicators[indicator.name] = indicator - - def getIndicatorValueAtInstant(self, indicatorName, instant): - indicator = self.getIndicator(indicatorName) - if indicator is not None: - return indicator[instant] - else: - return None - - def getIndicatorValuesAtInstant(self, instant): - '''Returns list of indicator values at instant - as dict (with keys from indicators dict)''' - values = {} - for k, indicator in self.indicators.items(): - values[k] = indicator[instant] - return values - - def plot(self, options = '', withOrigin = False, timeStep = 1, withFeatures = False, **kwargs): - self.roadUser1.plot(options, withOrigin, timeStep, withFeatures, **kwargs) - self.roadUser2.plot(options, withOrigin, timeStep, withFeatures, **kwargs) - - def plotOnWorldImage(self, nPixelsPerUnitDistance, options = '', withOrigin = False, timeStep = 1, **kwargs): - self.roadUser1.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, **kwargs) - self.roadUser2.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, **kwargs) - - def play(self, videoFilename, homography = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., allUserInstants = False): - if self.roadUser1 is not None and self.roadUser2 is not None: - if allUserInstants: - firstFrameNum = min(self.roadUser1.getFirstInstant(), self.roadUser2.getFirstInstant()) - lastFrameNum = max(self.roadUser1.getLastInstant(), self.roadUser2.getLastInstant()) - else: - firstFrameNum = self.getFirstInstant() - lastFrameNum = self.getLastInstant() - cvutils.displayTrajectories(videoFilename, [self.roadUser1, self.roadUser2], homography = homography, firstFrameNum = firstFrameNum, lastFrameNumArg = lastFrameNum, undistort = undistort, intrinsicCameraMatrix = intrinsicCameraMatrix, distortionCoefficients = distortionCoefficients, undistortedImageMultiplication = undistortedImageMultiplication) - else: - print('Please set the interaction road user attributes roadUser1 and roadUser1 through the method setRoadUsers') - - def computeIndicators(self): - '''Computes the collision course cosine only if the cosine is positive''' - collisionCourseDotProducts = {}#[0]*int(self.timeInterval.length()) - collisionCourseAngles = {} - velocityAngles = {} - distances = {}#[0]*int(self.timeInterval.length()) - speedDifferentials = {} - interactionInstants = [] - for instant in self.timeInterval: - deltap = self.roadUser1.getPositionAtInstant(instant)-self.roadUser2.getPositionAtInstant(instant) - v1 = self.roadUser1.getVelocityAtInstant(instant) - v2 = self.roadUser2.getVelocityAtInstant(instant) - deltav = v2-v1 - velocityAngles[instant] = np.arccos(moving.Point.dot(v1, v2)/(v1.norm2()*v2.norm2())) - collisionCourseDotProducts[instant] = moving.Point.dot(deltap, deltav) - distances[instant] = deltap.norm2() - speedDifferentials[instant] = deltav.norm2() - if collisionCourseDotProducts[instant] > 0: - interactionInstants.append(instant) - if distances[instant] != 0 and speedDifferentials[instant] != 0: - collisionCourseAngles[instant] = np.arccos(collisionCourseDotProducts[instant]/(distances[instant]*speedDifferentials[instant])) - - if len(interactionInstants) >= 2: - self.interactionInterval = moving.TimeInterval(interactionInstants[0], interactionInstants[-1]) - else: - self.interactionInterval = moving.TimeInterval() - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[0], collisionCourseDotProducts)) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[1], collisionCourseAngles)) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[2], distances, mostSevereIsMax = False)) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[4], velocityAngles)) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[5], speedDifferentials)) - - # if we have features, compute other indicators - if self.roadUser1.hasFeatures() and self.roadUser2.hasFeatures(): - minDistances={} - for instant in self.timeInterval: - minDistances[instant] = moving.MovingObject.minDistance(self.roadUser1, self.roadUser2, instant) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[3], minDistances, mostSevereIsMax = False)) - - def computeCrossingsCollisions(self, predictionParameters, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, timeInterval = None): - '''Computes all crossing and collision points at each common instant for two road users. ''' - TTCs = {} - collisionProbabilities = {} - if timeInterval is not None: - commonTimeInterval = timeInterval - else: - commonTimeInterval = self.timeInterval - self.collisionPoints, crossingZones = predictionParameters.computeCrossingsCollisions(self.roadUser1, self.roadUser2, collisionDistanceThreshold, timeHorizon, computeCZ, debug, commonTimeInterval) - for i, cps in self.collisionPoints.items(): - TTCs[i] = prediction.SafetyPoint.computeExpectedIndicator(cps) - collisionProbabilities[i] = sum([p.probability for p in cps]) - if len(TTCs) > 0: - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[7], TTCs, mostSevereIsMax=False)) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[6], collisionProbabilities)) - - # crossing zones and pPET - if computeCZ: - self.crossingZones = crossingZones - pPETs = {} - for i, cz in self.crossingZones.items(): - pPETs[i] = prediction.SafetyPoint.computeExpectedIndicator(cz) - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[9], pPETs, mostSevereIsMax=False)) - # TODO add probability of collision, and probability of successful evasive action - - def computePET(self, collisionDistanceThreshold): - pet, t1, t2= moving.MovingObject.computePET(self.roadUser1, self.roadUser2, collisionDistanceThreshold) - if pet is not None: - self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[10], {min(t1, t2): pet}, mostSevereIsMax = False)) - - def setCollision(self, collision): - '''indicates if it is a collision: argument should be boolean''' - self.collision = collision - - def isCollision(self): - if hasattr(self, 'collision'): - return self.collision - else: - return None - - def getCollisionPoints(self): - return self.collisionPoints - - def getCrossingZones(self): - return self.crossingZones - -def createInteractions(objects, _others = None): - '''Create all interactions of two co-existing road users''' - if _others is not None: - others = _others - - interactions = [] - num = 0 - for i in range(len(objects)): - if _others is None: - others = objects[:i] - for j in range(len(others)): - commonTimeInterval = objects[i].commonTimeInterval(others[j]) - if not commonTimeInterval.empty(): - interactions.append(Interaction(num, commonTimeInterval, objects[i].num, others[j].num, objects[i], others[j])) - num += 1 - return interactions - -def findInteraction(interactions, roadUserNum1, roadUserNum2): - 'Returns the right interaction in the set' - i=0 - while i<len(interactions) and set([roadUserNum1, roadUserNum2]) != interactions[i].getRoadUserNumbers(): - i+=1 - if i<len(interactions): - return interactions[i] - else: - return None - -def computeIndicators(interactions, computeMotionPrediction, computePET, predictionParameters, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, timeInterval = None): - for inter in interactions: - print('processing interaction {}'.format(inter.getNum())) # logging.debug('processing interaction {}'.format(inter.getNum())) - inter.computeIndicators() - if computeMotionPrediction: - inter.computeCrossingsCollisions(predictionParameters, collisionDistanceThreshold, timeHorizon, computeCZ, debug, timeInterval) - if computePET: - inter.computePET(collisionDistanceThreshold) - return interactions - -def aggregateSafetyPoints(interactions, pointType = 'collision'): - '''Put all collision points or crossing zones in a list for display''' - allPoints = [] - if pointType == 'collision': - for i in interactions: - for points in i.collisionPoints.values(): - allPoints += points - elif pointType == 'crossing': - for i in interactions: - for points in i.crossingZones.values(): - allPoints += points - else: - print('unknown type of point: '+pointType) - return allPoints - -def prototypeCluster(interactions, similarities, indicatorName, minSimilarity, similarityFunc = None, minClusterSize = None, randomInitialization = False): - return ml.prototypeCluster([inter.getIndicator(indicatorName) for inter in interactions], similarities, minSimilarity, similarityFunc, minClusterSize, randomInitialization) - -class Crossing(moving.STObject): - '''Class for the event of a street crossing - - TODO: detecter passage sur la chaussee - identifier origines et destination (ou uniquement chaussee dans FOV) - carac traversee - detecter proximite veh (retirer si trop similaire simultanement - carac interaction''' - - def __init__(self, roaduserNum = None, num = None, timeInterval = None): - moving.STObject.__init__(self, num, timeInterval) - self.roaduserNum = roaduserNum - - - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/events.txt') - #suite = doctest.DocTestSuite() - unittest.TextTestRunner().run(suite) -
--- a/python/indicators.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,258 +0,0 @@ -#! /usr/bin/env python -'''Class for indicators, temporal indicators, and safety indicators''' - -import moving -#import matplotlib.nxutils as nx -from matplotlib.pyplot import plot, ylim -from matplotlib.pylab import find -from numpy import array, arange, mean, floor, mean -from scipy import percentile - -def multivariateName(indicatorNames): - return '_'.join(indicatorNames) - -# need for a class representing the indicators, their units, how to print them in graphs... -class TemporalIndicator(object): - '''Class for temporal indicators - i.e. indicators that take a value at specific instants - - values should be - * a dict, for the values at specific time instants - * or a list with a time interval object if continuous measurements - - it should have more information like name, unit''' - - def __init__(self, name, values, timeInterval = None, maxValue = None): - self.name = name - if timeInterval is None: - self.values = values - instants = sorted(self.values.keys()) - if len(instants) > 0: - self.timeInterval = moving.TimeInterval(instants[0], instants[-1]) - else: - self.timeInterval = moving.TimeInterval() - else: - assert len(values) == timeInterval.length() - self.timeInterval = timeInterval - self.values = {} - for i in range(int(round(self.timeInterval.length()))): - self.values[self.timeInterval[i]] = values[i] - self.maxValue = maxValue - - def __len__(self): - return len(self.values) - - def empty(self): - return len(self.values) == 0 - - def __getitem__(self, t): - 'Returns the value at time t' - return self.values.get(t) - - def getIthValue(self, i): - sortedKeys = sorted(self.values.keys()) - if 0<=i<len(sortedKeys): - return self.values[sortedKeys[i]] - else: - return None - - def __iter__(self): - self.iterInstantNum = 0 # index in the interval or keys of the dict - return self - - def __next__(self): - if self.iterInstantNum >= len(self.values):#(self.timeInterval and self.iterInstantNum>=self.timeInterval.length())\ - # or (self.iterInstantNum >= self.values) - raise StopIteration - else: - self.iterInstantNum += 1 - return self.getIthValue(self.iterInstantNum-1) - - def getTimeInterval(self): - return self.timeInterval - - def getName(self): - return self.name - - def getValues(self): - return [self.__getitem__(t) for t in self.timeInterval] - - def plot(self, options = '', xfactor = 1., yfactor = 1., timeShift = 0, **kwargs): - if self.getTimeInterval().length() == 1: - marker = 'o' - else: - marker = '' - time = sorted(self.values.keys()) - plot([(x+timeShift)/xfactor for x in time], [self.values[i]/yfactor for i in time], options+marker, **kwargs) - if self.maxValue: - ylim(ymax = self.maxValue) - - @classmethod - def createMultivariate(cls, indicators): - '''Creates a new temporal indicator where the value at each instant is a list - of the indicator values at the instant, in the same order - the time interval will be the union of the time intervals of the indicators - name is concatenation of the indicator names''' - if len(indicators) < 2: - print('Error creating multivariate indicator with only {} indicator'.format(len(indicators))) - return None - - timeInterval = moving.TimeInterval.unionIntervals([indic.getTimeInterval() for indic in indicators]) - values = {} - for t in timeInterval: - tmpValues = [indic[t] for indic in indicators] - uniqueValues = set(tmpValues) - if len(uniqueValues) >= 2 or uniqueValues.pop() is not None: - values[t] = tmpValues - return cls(multivariateName([indic.name for indic in indicators]), values) - -# TODO static method avec class en parametre pour faire des indicateurs agrege, list par instant - -def l1Distance(x, y): # lambda x,y:abs(x-y) - if x is None or y is None: - return float('inf') - else: - return abs(x-y) - -def multiL1Matching(x, y, thresholds, proportionMatching=1.): - n = 0 - nDimensions = len(x) - for i in range(nDimensions): - if l1Distance(x[i], y[i]) <= thresholds[i]: - n += 1 - return n >= nDimensions*proportionMatching - -from utils import LCSS as utilsLCSS - -class LCSS(utilsLCSS): - '''Adapted LCSS class for indicators, same pattern''' - def __init__(self, similarityFunc, delta = float('inf'), minLength = 0, aligned = False, lengthFunc = min): - utilsLCSS.__init__(self, similarityFunc = similarityFunc, delta = delta, aligned = aligned, lengthFunc = lengthFunc) - self.minLength = minLength - - def checkIndicator(self, indicator): - return indicator is not None and len(indicator) >= self.minLength - - def compute(self, indicator1, indicator2, computeSubSequence = False): - if self.checkIndicator(indicator1) and self.checkIndicator(indicator2): - return self._compute(indicator1.getValues(), indicator2.getValues(), computeSubSequence) - else: - return 0 - - def computeNormalized(self, indicator1, indicator2, computeSubSequence = False): - if self.checkIndicator(indicator1) and self.checkIndicator(indicator2): - return self._computeNormalized(indicator1.getValues(), indicator2.getValues(), computeSubSequence) - else: - return 0. - - def computeDistance(self, indicator1, indicator2, computeSubSequence = False): - if self.checkIndicator(indicator1) and self.checkIndicator(indicator2): - return self._computeDistance(indicator1.getValues(), indicator2.getValues(), computeSubSequence) - else: - return 1. - -class SeverityIndicator(TemporalIndicator): - '''Class for severity indicators - field mostSevereIsMax is True - if the most severe value taken by the indicator is the maximum''' - - def __init__(self, name, values, timeInterval=None, mostSevereIsMax=True, maxValue = None): - TemporalIndicator.__init__(self, name, values, timeInterval, maxValue) - self.mostSevereIsMax = mostSevereIsMax - - def getMostSevereValue(self, minNInstants=1, centile=15.): - '''if there are more than minNInstants observations, - returns either the average of these maximum values - or if centile is not None the n% centile from the most severe value - - eg for TTC, 15 returns the 15th centile (value such that 15% of observations are lower)''' - if self.__len__() < minNInstants: - return None - else: - values = list(self.values.values()) - if centile is not None: - if self.mostSevereIsMax: - c = 100-centile - else: - c = centile - return percentile(values, c) - else: - values = sorted(values, reverse = self.mostSevereIsMax) # inverted if most severe is max -> take the first values - return mean(values[:minNInstants]) - - def getInstantOfMostSevereValue(self): - '''Returns the instant at which the indicator reaches its most severe value''' - if self.mostSevereIsMax: - return max(self.values, key=self.values.get) - else: - return min(self.values, key=self.values.get) - -# functions to aggregate discretized maps of indicators -# TODO add values in the cells between the positions (similar to discretizing vector graphics to bitmap) - -def indicatorMap(indicatorValues, trajectory, squareSize): - '''Returns a dictionary - with keys for the indices of the cells (squares) - in which the trajectory positions are located - at which the indicator values are attached - - ex: speeds and trajectory''' - - assert len(indicatorValues) == trajectory.length() - indicatorMap = {} - for k in range(trajectory.length()): - p = trajectory[k] - i = floor(p.x/squareSize) - j = floor(p.y/squareSize) - if (i,j) in indicatorMap: - indicatorMap[(i,j)].append(indicatorValues[k]) - else: - indicatorMap[(i,j)] = [indicatorValues[k]] - for k in indicatorMap: - indicatorMap[k] = mean(indicatorMap[k]) - return indicatorMap - -# def indicatorMapFromPolygon(value, polygon, squareSize): -# '''Fills an indicator map with the value within the polygon -# (array of Nx2 coordinates of the polygon vertices)''' -# points = [] -# for x in arange(min(polygon[:,0])+squareSize/2, max(polygon[:,0]), squareSize): -# for y in arange(min(polygon[:,1])+squareSize/2, max(polygon[:,1]), squareSize): -# points.append([x,y]) -# inside = nx.points_inside_poly(array(points), polygon) -# indicatorMap = {} -# for i in range(len(inside)): -# if inside[i]: -# indicatorMap[(floor(points[i][0]/squareSize), floor(points[i][1]/squareSize))] = 0 -# return indicatorMap - -def indicatorMapFromAxis(value, limits, squareSize): - '''axis = [xmin, xmax, ymin, ymax] ''' - indicatorMap = {} - for x in arange(limits[0], limits[1], squareSize): - for y in arange(limits[2], limits[3], squareSize): - indicatorMap[(floor(x/squareSize), floor(y/squareSize))] = value - return indicatorMap - -def combineIndicatorMaps(maps, squareSize, combinationFunction): - '''Puts many indicator maps together - (averaging the values in each cell - if more than one maps has a value)''' - indicatorMap = {} - for m in maps: - for k,v in m.items(): - if k in indicatorMap: - indicatorMap[k].append(v) - else: - indicatorMap[k] = [v] - for k in indicatorMap: - indicatorMap[k] = combinationFunction(indicatorMap[k]) - return indicatorMap - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/indicators.txt') - unittest.TextTestRunner().run(suite) -# #doctest.testmod() -# #doctest.testfile("example.txt")
--- a/python/metadata.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,415 +0,0 @@ -from datetime import datetime, timedelta -from os import path, listdir, sep -from math import floor - -from numpy import zeros, loadtxt, array - -from sqlalchemy import orm, create_engine, Column, Integer, Float, DateTime, String, ForeignKey, Boolean, Interval -from sqlalchemy.orm import relationship, backref, sessionmaker -from sqlalchemy.ext.declarative import declarative_base - -from utils import datetimeFormat, removeExtension, getExtension, TimeConverter -from cvutils import computeUndistortMaps, videoFilenameExtensions, infoVideo -from moving import TimeInterval - -""" -Metadata to describe how video data and configuration files for video analysis are stored - -Typical example is - -site1/view1/2012-06-01/video.avi - /2012-06-02/video.avi - ... - /view2/2012-06-01/video.avi - /2012-06-02/video.avi - ... - -- where site1 is the path to the directory containing all information pertaining to the site, -relative to directory of the SQLite file storing the metadata -represented by Site class -(can contain for example the aerial or map image of the site, used for projection) - -- view1 is the directory for the first camera field of view (camera fixed position) at site site1 -represented by CameraView class -(can contain for example the homography file, mask file and tracking configuration file) - -- YYYY-MM-DD is the directory containing all the video files for that day -with camera view view1 at site site1 - - -""" - -Base = declarative_base() - -class Site(Base): - __tablename__ = 'sites' - idx = Column(Integer, primary_key=True) - name = Column(String) # path to directory containing all site information (in subdirectories), relative to the database position - description = Column(String) # longer names, eg intersection of road1 and road2 - xcoordinate = Column(Float) # ideally moving.Point, but needs to be - ycoordinate = Column(Float) - mapImageFilename = Column(String) # path to map image file, relative to site name, ie sitename/mapImageFilename - nUnitsPerPixel = Column(Float) # number of units of distance per pixel in map image - worldDistanceUnit = Column(String, default = 'm') # make sure it is default in the database - - def __init__(self, name, description = "", xcoordinate = None, ycoordinate = None, mapImageFilename = None, nUnitsPerPixel = 1., worldDistanceUnit = 'm'): - self.name = name - self.description = description - self.xcoordinate = xcoordinate - self.ycoordinate = ycoordinate - self.mapImageFilename = mapImageFilename - self.nUnitsPerPixel = nUnitsPerPixel - self.worldDistanceUnit = worldDistanceUnit - - def getPath(self): - return self.name - - def getMapImageFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.getPath(), self.mapImageFilename) - else: - return self.mapImageFilename - - -class EnvironementalFactors(Base): - '''Represents any environmental factors that may affect the results, in particular - * changing weather conditions - * changing road configuration, geometry, signalization, etc. - ex: sunny, rainy, before counter-measure, after counter-measure''' - __tablename__ = 'environmental_factors' - idx = Column(Integer, primary_key=True) - startTime = Column(DateTime) - endTime = Column(DateTime) - description = Column(String) # eg sunny, before, after - siteIdx = Column(Integer, ForeignKey('sites.idx')) - - site = relationship("Site", backref = backref('environmentalFactors')) - - def __init__(self, startTime, endTime, description, site): - 'startTime is passed as string in utils.datetimeFormat, eg 2011-06-22 10:00:39' - self.startTime = datetime.strptime(startTime, datetimeFormat) - self.endTime = datetime.strptime(endTime, datetimeFormat) - self.description = description - self.site = site - -class CameraType(Base): - ''' Represents parameters of the specific camera used. - - Taken and adapted from tvalib''' - __tablename__ = 'camera_types' - idx = Column(Integer, primary_key=True) - name = Column(String) - resX = Column(Integer) - resY = Column(Integer) - frameRate = Column(Float) - frameRateTimeUnit = Column(String, default = 's') - intrinsicCameraMatrixStr = Column(String) - distortionCoefficientsStr = Column(String) - - def __init__(self, name, resX, resY, frameRate, frameRateTimeUnit = 's', trackingConfigurationFilename = None, intrinsicCameraFilename = None, intrinsicCameraMatrix = None, distortionCoefficients = None): - self.name = name - self.resX = resX - self.resY = resY - self.frameRate = frameRate - self.frameRateTimeUnit = frameRateTimeUnit - self.intrinsicCameraMatrix = None # should be np.array - self.distortionCoefficients = None # list - - if trackingConfigurationFilename is not None: - from storage import ProcessParameters - params = ProcessParameters(trackingConfigurationFilename) - self.intrinsicCameraMatrix = params.intrinsicCameraMatrix - self.distortionCoefficients = params.distortionCoefficients - elif intrinsicCameraFilename is not None: - self.intrinsicCameraMatrix = loadtxt(intrinsicCameraFilename) - self.distortionCoefficients = distortionCoefficients - else: - self.intrinsicCameraMatrix = intrinsicCameraMatrix - self.distortionCoefficients = distortionCoefficients - - if self.intrinsicCameraMatrix is not None: - self.intrinsicCameraMatrixStr = str(self.intrinsicCameraMatrix.tolist()) - if self.distortionCoefficients is not None and len(self.distortionCoefficients) == 5: - self.distortionCoefficientsStr = str(self.distortionCoefficients) - - @orm.reconstructor - def initOnLoad(self): - if self.intrinsicCameraMatrixStr is not None: - from ast import literal_eval - self.intrinsicCameraMatrix = array(literal_eval(self.intrinsicCameraMatrixStr)) - else: - self.intrinsicCameraMatrix = None - if self.distortionCoefficientsStr is not None: - self.distortionCoefficients = literal_eval(self.distortionCoefficientsStr) - else: - self.distortionCoefficients = None - - def computeUndistortMaps(self, undistortedImageMultiplication = None): - if undistortedImageMultiplication is not None and self.intrinsicCameraMatrix is not None and self.distortionCoefficients is not None: - [self.map1, self.map2], newCameraMatrix = computeUndistortMaps(self.resX, self.resY, undistortedImageMultiplication, self.intrinsicCameraMatrix, self.distortionCoefficients) - else: - self.map1 = None - self.map2 = None - - @staticmethod - def getCameraType(session, cameraTypeId, resX = None): - 'Returns the site(s) matching the index or the name' - if str.isdigit(cameraTypeId): - return session.query(CameraType).filter(CameraType.idx == int(cameraTypeId)).all() - else: - if resX is not None: - return session.query(CameraType).filter(CameraType.name.like('%'+cameraTypeId+'%')).filter(CameraType.resX == resX).all() - else: - return session.query(CameraType).filter(CameraType.name.like('%'+cameraTypeId+'%')).all() - -# class SiteDescription(Base): # list of lines and polygons describing the site, eg for sidewalks, center lines - -class CameraView(Base): - __tablename__ = 'camera_views' - idx = Column(Integer, primary_key=True) - description = Column(String) - homographyFilename = Column(String) # path to homograph file, relative to the site name - siteIdx = Column(Integer, ForeignKey('sites.idx')) - cameraTypeIdx = Column(Integer, ForeignKey('camera_types.idx')) - trackingConfigurationFilename = Column(String) # path to configuration .cfg file, relative to site name - maskFilename = Column(String) # path to mask file, relative to site name - virtual = Column(Boolean) # indicates it is not a real camera view, eg merged - - site = relationship("Site", backref = backref('cameraViews')) - cameraType = relationship('CameraType', backref = backref('cameraViews')) - - def __init__(self, description, homographyFilename, site, cameraType, trackingConfigurationFilename, maskFilename, virtual = False): - self.description = description - self.homographyFilename = homographyFilename - self.site = site - self.cameraType = cameraType - self.trackingConfigurationFilename = trackingConfigurationFilename - self.maskFilename = maskFilename - self.virtual = virtual - - def getHomographyFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.site.getPath(), self.homographyFilename) - else: - return self.homographyFilename - - def getTrackingConfigurationFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.site.getPath(), self.trackingConfigurationFilename) - else: - return self.trackingConfigurationFilename - - def getMaskFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.site.getPath(), self.maskFilename) - else: - return self.maskFilename - - def getTrackingParameters(self): - return ProcessParameters(self.getTrackingConfigurationFilename()) - - def getHomographyDistanceUnit(self): - return self.site.worldDistanceUnit - -# class Alignment(Base): -# __tablename__ = 'alignments' -# idx = Column(Integer, primary_key=True) -# siteIdx = Column(Integer, ForeignKey('sites.idx')) - -# cameraView = relationship("Site", backref = backref('alignments')) - -# def __init__(self, cameraView): -# self.cameraView = cameraView - -# class Point(Base): -# __tablename__ = 'points' -# alignmentIdx = Column(Integer, ForeignKey('alignments.idx'), primary_key=True) -# index = Column(Integer, primary_key=True) # order of points in this alignment -# x = Column(Float) -# y = Column(Float) - -# alignment = relationship("Alignment", backref = backref('points', order_by = index)) - -# def __init__(self, alignmentIdx, index, x, y): -# self.alignmentIdx = alignmentIdx -# self.index = index -# self.x = x -# self.y = y - -class VideoSequence(Base): - __tablename__ = 'video_sequences' - idx = Column(Integer, primary_key=True) - name = Column(String) # path to the video file relative to the the site name - startTime = Column(DateTime) - duration = Column(Interval) # video sequence duration - databaseFilename = Column(String) # path to the database file relative to the the site name - virtual = Column(Boolean) # indicates it is not a real video sequence (no video file), eg merged - cameraViewIdx = Column(Integer, ForeignKey('camera_views.idx')) - - cameraView = relationship("CameraView", backref = backref('videoSequences', order_by = idx)) - - def __init__(self, name, startTime, duration, cameraView, databaseFilename = None, virtual = False): - '''startTime is passed as string in utils.datetimeFormat, eg 2011-06-22 10:00:39 - duration is a timedelta object''' - self.name = name - if isinstance(startTime, str): - self.startTime = datetime.strptime(startTime, datetimeFormat) - else: - self.startTime = startTime - self.duration = duration - self.cameraView = cameraView - if databaseFilename is None and len(self.name) > 0: - self.databaseFilename = removeExtension(self.name)+'.sqlite' - else: - self.databaseFilename = databaseFilename - self.virtual = virtual - - def getVideoSequenceFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.cameraView.site.getPath(), self.name) - else: - return self.name - - def getDatabaseFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.cameraView.site.getPath(), self.databaseFilename) - else: - return self.databaseFilename - - def getTimeInterval(self): - return TimeInterval(self.startTime, self.startTime+self.duration) - - def containsInstant(self, instant): - 'instant is a datetime' - return self.startTime <= instant and self.startTime+self.duration - - def intersection(self, startTime, endTime): - 'returns the moving.TimeInterval intersection with [startTime, endTime]' - return TimeInterval.intersection(self.getTimeInterval(), TimeInterval(startTime, endTime)) - - def getFrameNum(self, instant): - 'Warning, there is no check of correct time units' - if self.containsInstant(instant): - return int(floor((instant-self.startTime).seconds*self.cameraView.cameraType.frameRate)) - else: - return None - -class TrackingAnnotation(Base): - __tablename__ = 'tracking_annotations' - idx = Column(Integer, primary_key=True) - description = Column(String) # description - groundTruthFilename = Column(String) - firstFrameNum = Column(Integer) # first frame num of annotated data (could be computed on less data) - lastFrameNum = Column(Integer) - videoSequenceIdx = Column(Integer, ForeignKey('video_sequences.idx')) - maskFilename = Column(String) # path to mask file (can be different from camera view, for annotations), relative to site name - undistorted = Column(Boolean) # indicates whether the annotations were done in undistorted video space - - videoSequence = relationship("VideoSequence", backref = backref('trackingAnnotations')) - - def __init__(self, description, groundTruthFilename, firstFrameNum, lastFrameNum, videoSequence, maskFilename, undistorted = True): - self.description = description - self.groundTruthFilename = groundTruthFilename - self.firstFrameNum = firstFrameNum - self.lastFrameNum = lastFrameNum - self.videoSequence = videoSequence - self.undistorted = undistorted - self.maskFilename = maskFilename - - def getGroundTruthFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.videoSequence.cameraView.site.getPath(), self.groundTruthFilename) - else: - return self.groundTruthFilename - - def getMaskFilename(self, relativeToSiteFilename = True): - if relativeToSiteFilename: - return path.join(self.videoSequence.cameraView.site.getPath(), self.maskFilename) - else: - return self.maskFilename - - def getTimeInterval(self): - return TimeInterval(self.firstFrameNum, self.lastFrameNum) - -# add class for Analysis: foreign key VideoSequenceId, dataFilename, configFilename (get the one from camera view by default), mask? (no, can be referenced in the tracking cfg file) - -# class Analysis(Base): # parameters necessary for processing the data: free form -# eg bounding box depends on camera view, tracking configuration depends on camera view -# results: sqlite - -def createDatabase(filename): - 'creates a session to query the filename' - engine = create_engine('sqlite:///'+filename) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - return Session() - -def connectDatabase(filename): - 'creates a session to query the filename' - engine = create_engine('sqlite:///'+filename) - Session = sessionmaker(bind=engine) - return Session() - -def getSite(session, siteId = None, name = None, description = None): - 'Returns the site(s) matching the index or the name' - if siteId is not None: - return session.query(Site).filter(Site.idx == int(siteId)).all() - elif name is not None: - return session.query(Site).filter(Site.description.like('%'+name+'%')).all() - elif description is not None: - return session.query(Site).filter(Site.description.like('%'+description+'%')).all() - else: - print('No siteId, name or description have been provided to the function') - return [] - -def getCameraView(session, viewId): - 'Returns the site(s) matching the index' - return session.query(CameraView).filter(CameraView.idx == int(viewId)).first() - -def initializeSites(session, directoryName, nViewsPerSite = 1): - '''Initializes default site objects and n camera views per site - - eg somedirectory/montreal/ contains intersection1, intersection2, etc. - The site names would be somedirectory/montreal/intersection1, somedirectory/montreal/intersection2, etc. - The views should be directories in somedirectory/montreal/intersection1''' - sites = [] - cameraViews = [] - names = sorted(listdir(directoryName)) - for name in names: - if path.isdir(directoryName+sep+name): - sites.append(Site(directoryName+sep+name, None)) - for cameraViewIdx in range(1, nViewsPerSite+1): - cameraViews.append(CameraView('view{}'.format(cameraViewIdx), None, sites[-1], None, None, None)) - session.add_all(sites) - session.add_all(cameraViews) - session.commit() - -def initializeVideos(session, cameraView, directoryName, startTime = None, datetimeFormat = None): - '''Initializes videos with time or tries to guess it from filename - directoryName should contain the videos to find and be the relative path from the site location''' - names = sorted(listdir(directoryName)) - videoSequences = [] - if datetimeFormat is not None: - timeConverter = TimeConverter(datetimeFormat) - for name in names: - prefix = removeExtension(name) - extension = getExtension(name) - if extension in videoFilenameExtensions: - if datetimeFormat is not None: - from argparse import ArgumentTypeError - try: - t1 = timeConverter.convert(name[:name.rfind('_')]) - print('DB time {} / Time from filename {}'.format(startTime, t1)) - except ArgumentTypeError as e: - print('File format error for time {} (prefix {})'.format(name, prefix)) - vidinfo = infoVideo(directoryName+sep+name) - duration = vidinfo['number of frames']#timedelta(minutes = 27, seconds = 33) - fps = vidinfo['fps'] - duration = timedelta(seconds=duration/fps) - videoSequences.append(VideoSequence(directoryName+sep+name, startTime, duration, cameraView, directoryName+sep+prefix+'.sqlite')) - startTime += duration - session.add_all(videoSequences) - session.commit() - -# management -# TODO need to be able to copy everything from a site from one sqlite to another, and delete everything attached to a site
--- a/python/ml.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,326 +0,0 @@ -#! /usr/bin/env python -'''Libraries for machine learning algorithms''' - -from os import path -from random import shuffle -from copy import copy, deepcopy - -import numpy as np -from matplotlib.pylab import text -import matplotlib as mpl -import matplotlib.pyplot as plt -from scipy.cluster.vq import kmeans, whiten, vq -from sklearn import mixture -try: - import cv2 - opencvAvailable = True -except ImportError: - print('OpenCV library could not be loaded (video replay functions will not be available)') # TODO change to logging module - opencvAvailable = False - -import utils - -##################### -# OpenCV ML models -##################### - -def computeConfusionMatrix(model, samples, responses): - '''computes the confusion matrix of the classifier (model) - - samples should be n samples by m variables''' - classifications = {} - predictions = model.predict(samples) - for predicted, y in zip(predictions, responses): - classifications[(y, predicted)] = classifications.get((y, predicted), 0)+1 - return classifications - -if opencvAvailable: - class SVM(object): - '''wrapper for OpenCV SimpleVectorMachine algorithm''' - def __init__(self, svmType = cv2.ml.SVM_C_SVC, kernelType = cv2.ml.SVM_RBF, degree = 0, gamma = 1, coef0 = 0, Cvalue = 1, nu = 0, p = 0): - self.model = cv2.ml.SVM_create() - self.model.setType(svmType) - self.model.setKernel(kernelType) - self.model.setDegree(degree) - self.model.setGamma(gamma) - self.model.setCoef0(coef0) - self.model.setC(Cvalue) - self.model.setNu(nu) - self.model.setP(p) - - def save(self, filename): - self.model.save(filename) - - def train(self, samples, layout, responses, computePerformance = False): - self.model.train(samples, layout, responses) - if computePerformance: - return computeConfusionMatrix(self, samples, responses) - - def predict(self, hog): - retval, predictions = self.model.predict(hog) - if hog.shape[0] == 1: - return predictions[0][0] - else: - return np.asarray(predictions, dtype = np.int).ravel().tolist() - - def SVM_load(filename): - if path.exists(filename): - svm = SVM() - svm.model = cv2.ml.SVM_load(filename) - return svm - else: - print('Provided filename {} does not exist: model not loaded!'.format(filename)) - -##################### -# Clustering -##################### - -class Centroid(object): - 'Wrapper around instances to add a counter' - - def __init__(self, instance, nInstances = 1): - self.instance = instance - self.nInstances = nInstances - - # def similar(instance2): - # return self.instance.similar(instance2) - - def add(self, instance2): - self.instance = self.instance.multiply(self.nInstances)+instance2 - self.nInstances += 1 - self.instance = self.instance.multiply(1/float(self.nInstances)) - - def average(c): - inst = self.instance.multiply(self.nInstances)+c.instance.multiply(instance.nInstances) - inst.multiply(1/(self.nInstances+instance.nInstances)) - return Centroid(inst, self.nInstances+instance.nInstances) - - def plot(self, options = ''): - self.instance.plot(options) - text(self.instance.position.x+1, self.instance.position.y+1, str(self.nInstances)) - -def kMedoids(similarityMatrix, initialCentroids = None, k = None): - '''Algorithm that clusters any dataset based on a similarity matrix - Either the initialCentroids or k are passed''' - pass - -def assignCluster(data, similarFunc, initialCentroids = None, shuffleData = True): - '''k-means algorithm with similarity function - Two instances should be in the same cluster if the sameCluster function returns true for two instances. It is supposed that the average centroid of a set of instances can be computed, using the function. - The number of clusters will be determined accordingly - - data: list of instances - averageCentroid: ''' - localdata = copy(data) # shallow copy to avoid modifying data - if shuffleData: - shuffle(localdata) - if initialCentroids is None: - centroids = [Centroid(localdata[0])] - else: - centroids = deepcopy(initialCentroids) - for instance in localdata[1:]: - i = 0 - while i<len(centroids) and not similarFunc(centroids[i].instance, instance): - i += 1 - if i == len(centroids): - centroids.append(Centroid(instance)) - else: - centroids[i].add(instance) - - return centroids - -# TODO recompute centroids for each cluster: instance that minimizes some measure to all other elements - -def spectralClustering(similarityMatrix, k, iter=20): - '''Spectral Clustering algorithm''' - n = len(similarityMatrix) - # create Laplacian matrix - rowsum = np.sum(similarityMatrix,axis=0) - D = np.diag(1 / np.sqrt(rowsum)) - I = np.identity(n) - L = I - np.dot(D,np.dot(similarityMatrix,D)) - # compute eigenvectors of L - U,sigma,V = np.linalg.svd(L) - # create feature vector from k first eigenvectors - # by stacking eigenvectors as columns - features = np.array(V[:k]).T - # k-means - features = whiten(features) - centroids,distortion = kmeans(features,k, iter) - code,distance = vq(features,centroids) # code starting from 0 (represent first cluster) to k-1 (last cluster) - return code,sigma - -def assignToPrototypeClusters(instances, prototypeIndices, similarities, minSimilarity, similarityFunc = None, minClusterSize = 0): - '''Assigns instances to prototypes - if minClusterSize is not 0, the clusters will be refined by removing iteratively the smallest clusters - and reassigning all elements in the cluster until no cluster is smaller than minClusterSize - - labels are indices in the prototypeIndices''' - if similarityFunc is None: - print('similarityFunc is None') - return None - - indices = [i for i in range(len(instances)) if i not in prototypeIndices] - labels = [-1]*len(instances) - assign = True - while assign: - for i in prototypeIndices: - labels[i] = i - for i in indices: - for j in prototypeIndices: - if similarities[i][j] < 0: - similarities[i][j] = similarityFunc(instances[i], instances[j]) - similarities[j][i] = similarities[i][j] - label = similarities[i][prototypeIndices].argmax() - if similarities[i][prototypeIndices[label]] >= minSimilarity: - labels[i] = prototypeIndices[label] - else: - labels[i] = -1 # outlier - clusterSizes = {i: sum(np.array(labels) == i) for i in prototypeIndices} - smallestClusterIndex = min(clusterSizes, key = clusterSizes.get) - assign = (clusterSizes[smallestClusterIndex] < minClusterSize) - if assign: - prototypeIndices.remove(smallestClusterIndex) - indices = [i for i in range(similarities.shape[0]) if labels[i] == smallestClusterIndex] - return prototypeIndices, labels - -def prototypeCluster(instances, similarities, minSimilarity, similarityFunc = None, optimizeCentroid = False, randomInitialization = False, initialPrototypeIndices = None): - '''Finds exemplar (prototype) instance that represent each cluster - Returns the prototype indices (in the instances list) - - the elements in the instances list must have a length (method __len__), or one can use the optimizeCentroid - the positions in the instances list corresponds to the similarities - if similarityFunc is provided, the similarities are calculated as needed (this is faster) if not in similarities (negative if not computed) - similarities must still be allocated with the right size - - if an instance is different enough (<minSimilarity), - it will become a new prototype. - Non-prototype instances will be assigned to an existing prototype - - if optimizeCentroid is True, each time an element is added, we recompute the centroid trajectory as the most similar to all in the cluster - - initialPrototypeIndices are indices in instances - - TODO: check how similarity evolves in clusters''' - if len(instances) == 0: - print('no instances to cluster (empty list)') - return None - if similarityFunc is None: - print('similarityFunc is None') - return None - - # sort instances based on length - indices = range(len(instances)) - if randomInitialization or optimizeCentroid: - indices = np.random.permutation(indices).tolist() - else: - def compare(i, j): - if len(instances[i]) > len(instances[j]): - return -1 - elif len(instances[i]) == len(instances[j]): - return 0 - else: - return 1 - indices.sort(compare) - # initialize clusters - clusters = [] - if initialPrototypeIndices is None: - prototypeIndices = [indices[0]] - else: - prototypeIndices = initialPrototypeIndices # think of the format: if indices, have to be in instances - for i in prototypeIndices: - clusters.append([i]) - indices.remove(i) - # go through all instances - for i in indices: - for j in prototypeIndices: - if similarities[i][j] < 0: - similarities[i][j] = similarityFunc(instances[i], instances[j]) - similarities[j][i] = similarities[i][j] - label = similarities[i][prototypeIndices].argmax() # index in prototypeIndices - if similarities[i][prototypeIndices[label]] < minSimilarity: - prototypeIndices.append(i) - clusters.append([]) - else: - clusters[label].append(i) - if optimizeCentroid: - if len(clusters[label]) >= 2: # no point if only one element in cluster - for j in clusters[label][:-1]: - if similarities[i][j] < 0: - similarities[i][j] = similarityFunc(instances[i], instances[j]) - similarities[j][i] = similarities[i][j] - clusterIndices = clusters[label] - clusterSimilarities = similarities[clusterIndices][:,clusterIndices] - newCentroidIdx = clusterIndices[clusterSimilarities.sum(0).argmax()] - if prototypeIndices[label] != newCentroidIdx: - prototypeIndices[label] = newCentroidIdx - elif len(instances[prototypeIndices[label]]) < len(instances[i]): # replace prototype by current instance i if longer # otherwise, possible to test if randomInitialization or initialPrototypes is not None - prototypeIndices[label] = i - return prototypeIndices - -def computeClusterSizes(labels, prototypeIndices, outlierIndex = -1): - clusterSizes = {i: sum(np.array(labels) == i) for i in prototypeIndices} - clusterSizes['outlier'] = sum(np.array(labels) == outlierIndex) - return clusterSizes - -def computeClusterStatistics(labels, prototypeIndices, instances, similarities, similarityFunc, clusters = None, outlierIndex = -1): - if clusters is None: - clusters = {protoId:[] for protoId in prototypeIndices+[-1]} - for i,l in enumerate(labels): - clusters[l].append(i) - clusters = [clusters[protoId] for protoId in prototypeIndices] - for i, cluster in enumerate(clusters): - n = len(cluster) - print('cluster {}: {} elements'.format(prototypeIndices[i], n)) - if n >=2: - for j,k in enumerate(cluster): - for l in cluster[:j]: - if similarities[k][l] < 0: - similarities[k][l] = similarityFunc(instances[k], instances[l]) - similarities[l][k] = similarities[k][l] - print('Mean similarity to prototype: {}'.format((similarities[prototypeIndices[i]][cluster].sum()+1)/(n-1))) - print('Mean overall similarity: {}'.format((similarities[cluster][:,cluster].sum()+n)/(n*(n-1)))) - -# Gaussian Mixture Models -def plotGMM(mean, covariance, gmmId, fig, color, alpha = 0.3): - v, w = np.linalg.eigh(covariance) - angle = 180*np.arctan2(w[0][1], w[0][0])/np.pi - v *= 4 - ell = mpl.patches.Ellipse(mean, v[0], v[1], 180+angle, color=color) - ell.set_clip_box(fig.bbox) - ell.set_alpha(alpha) - fig.axes[0].add_artist(ell) - plt.plot([mean[0]], [mean[1]], 'x'+color) - plt.annotate(str(gmmId), xy=(mean[0]+1, mean[1]+1)) - -def plotGMMClusters(model, labels = None, dataset = None, fig = None, colors = utils.colors, nUnitsPerPixel = 1., alpha = 0.3): - '''plot the ellipse corresponding to the Gaussians - and the predicted classes of the instances in the dataset''' - if fig is None: - fig = plt.figure() - if len(fig.get_axes()) == 0: - fig.add_subplot(111) - for i in range(model.n_components): - mean = model.means_[i]/nUnitsPerPixel - covariance = model.covariances_[i]/nUnitsPerPixel - # plot points - if dataset is not None: - tmpDataset = dataset/nUnitsPerPixel - plt.scatter(tmpDataset[labels == i, 0], tmpDataset[labels == i, 1], .8, color=colors[i]) - # plot an ellipse to show the Gaussian component - plotGMM(mean, covariance, i, fig, colors[i], alpha) - if dataset is None: # to address issues without points, the axes limits are not redrawn - minima = model.means_.min(0) - maxima = model.means_.max(0) - xwidth = 0.5*(maxima[0]-minima[0]) - ywidth = 0.5*(maxima[1]-minima[1]) - plt.xlim(minima[0]-xwidth,maxima[0]+xwidth) - plt.ylim(minima[1]-ywidth,maxima[1]+ywidth) - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/ml.txt') - unittest.TextTestRunner().run(suite) -# #doctest.testmod() -# #doctest.testfile("example.txt")
--- a/python/moving.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1957 +0,0 @@ -#! /usr/bin/env python -'''Libraries for moving objects, trajectories...''' - -import utils, cvutils -from base import VideoFilenameAddable - -from math import sqrt, atan2, cos, sin -from numpy import median, mean, array, arange, zeros, ones, hypot, NaN, std, floor, float32, argwhere, minimum -from matplotlib.pyplot import plot, text -from scipy.stats import scoreatpercentile -from scipy.spatial.distance import cdist -from scipy.signal import savgol_filter -import copy - -try: - from shapely.geometry import Polygon, Point as shapelyPoint - from shapely.prepared import prep, PreparedGeometry - shapelyAvailable = True -except ImportError: - print('Shapely library could not be loaded') - shapelyAvailable = False - - -class Interval(object): - '''Generic interval: a subset of real numbers (not iterable)''' - def __init__(self, first=0, last=-1, revert = False): - if revert and last<first: - self.first=last - self.last=first - else: - self.first=first - self.last=last - - def __str__(self): - return '[{0}, {1}]'.format(self.first, self.last) - - def __repr__(self): - return self.__str__() - - def __eq__(self, other): - return ((self.first == other.first) and (self.last == other.last)) or ((self.first == other.last) and (self.last == other.first)) - - def empty(self): - return self.first > self.last - - def center(self): - return (self.first+self.last)/2. - - def length(self): - '''Returns the length of the interval''' - return float(max(0,self.last-self.first)) - - def equal(self, i2): - return self.first==i2.first and self.last == i2.last - - def getList(self): - return [self.first, self.last] - - def contains(self, instant): - return (self.first<=instant and self.last>=instant) - - def inside(self, interval2): - '''Indicates if the temporal interval of self is comprised in interval2''' - return (self.first >= interval2.first) and (self.last <= interval2.last) - - def shift(self, offset): - self.first += offset - self.last += offset - - @classmethod - def union(cls, interval1, interval2): - '''Smallest interval comprising self and interval2''' - return cls(min(interval1.first, interval2.first), max(interval2.last, interval2.last)) - - @classmethod - def intersection(cls, interval1, interval2): - '''Largest interval comprised in both self and interval2''' - return cls(max(interval1.first, interval2.first), min(interval1.last, interval2.last)) - - def distance(self, interval2): - if not Interval.intersection(self, interval2).empty(): - return 0 - elif self.first > interval2.last: - return self.first - interval2.last - elif self.last < interval2.first: - return interval2.first - self.last - else: - return None - - @classmethod - def unionIntervals(cls, intervals): - 'returns the smallest interval containing all intervals' - inter = cls(intervals[0].first, intervals[0].last) - for i in intervals[1:]: - inter = cls.union(inter, i) - return inter - - -class TimeInterval(Interval): - '''Temporal interval: set of instants at fixed time step, between first and last, included - - For example: based on frame numbers (hence the modified length method) - It may be modified directly by setting first and last - It also (mostly) works with datetime.datetime''' - - def __init__(self, first=0, last=-1, revert = False): - super(TimeInterval, self).__init__(first, last, revert) - - @staticmethod - def fromInterval(inter): - return TimeInterval(inter.first, inter.last) - - def __getitem__(self, i): - if not self.empty(): - if isinstance(i, int): - return self.first+i - else: - raise TypeError("Invalid argument type.") - #elif isinstance( key, slice ): - - def __iter__(self): - self.iterInstantNum = -1 - return self - - def __next__(self): - if self.iterInstantNum >= self.length()-1: - raise StopIteration - else: - self.iterInstantNum += 1 - return self[self.iterInstantNum] - - def length(self): - '''Returns the length of the interval''' - return float(max(0,self.last-self.first+1)) - - def __len__(self): - return self.length() - -# class BoundingPolygon: -# '''Class for a polygon bounding a set of points -# with methods to create intersection, unions... -# ''' -# We will use the polygon class of Shapely - -class STObject(object): - '''Class for spatio-temporal object, i.e. with temporal and spatial existence - (time interval and bounding polygon for positions (e.g. rectangle)). - - It may not mean that the object is defined - for all time instants within the time interval''' - - def __init__(self, num = None, timeInterval = None, boundingPolygon = None): - self.num = num - self.timeInterval = timeInterval - self.boundingPolygon = boundingPolygon - - def empty(self): - return self.timeInterval.empty()# or not self.boudingPolygon - - def getNum(self): - return self.num - - def __len__(self): - return self.timeInterval.length() - - def length(self): - return self.timeInterval.length() - - def getFirstInstant(self): - return self.timeInterval.first - - def getLastInstant(self): - return self.timeInterval.last - - def setFirstInstant(self, t): - if t <= self.timeInterval.last: - self.timeInterval.first = t - else: - print('new first instant is after last, not changing') - - def setLastInstant(self, t): - if t >= self.timeInterval.first: - self.timeInterval.last = t - else: - print('new last instant is before first, not changing') - - def getTimeInterval(self): - return self.timeInterval - - def existsAtInstant(self, t): - return self.timeInterval.contains(t) - - def commonTimeInterval(self, obj2): - return TimeInterval.intersection(self.getTimeInterval(), obj2.getTimeInterval()) - - def shiftTimeInterval(self, offset): - self.timeInterval.shift(offset) - -class Point(object): - def __init__(self, x, y): - self.x = x - self.y = y - - def __str__(self): - return '({:f},{:f})'.format(self.x,self.y) - - def __repr__(self): - return self.__str__() - - def __eq__(self, other): - return (self.x == other.x) and (self.y == other.y) - - def __add__(self, other): - return Point(self.x+other.x, self.y+other.y) - - def __sub__(self, other): - return Point(self.x-other.x, self.y-other.y) - - def __neg__(self): - return Point(-self.x, -self.y) - - def __getitem__(self, i): - if i == 0: - return self.x - elif i == 1: - return self.y - else: - raise IndexError() - - def orthogonal(self, clockwise = True): - 'Returns the orthogonal vector' - if clockwise: - return Point(self.y, -self.x) - else: - return Point(-self.y, self.x) - - def projectLocal(self, v, clockwise = True): - 'Projects point projected on v, v.orthogonal()' - e1 = v/v.norm2() - e2 = e1.orthogonal(clockwise) - return Point(Point.dot(self, e1), Point.doc(self, e2)) - - def rotate(self, theta): - return Point(self.x*cos(theta)-self.y*sin(theta), self.x*sin(theta)+self.y*cos(theta)) - - def __mul__(self, alpha): - 'Warning, returns a new Point' - return Point(self.x*alpha, self.y*alpha) - - def divide(self, alpha): - 'Warning, returns a new Point' - return Point(self.x/alpha, self.y/alpha) - - def plot(self, options = 'o', **kwargs): - plot([self.x], [self.y], options, **kwargs) - - @staticmethod - def plotSegment(p1, p2, options = 'o', **kwargs): - plot([p1.x, p2.x], [p1.y, p2.y], options, **kwargs) - - def angle(self): - return atan2(self.y, self.x) - - def norm2Squared(self): - '''2-norm distance (Euclidean distance)''' - return self.x**2+self.y**2 - - def norm2(self): - '''2-norm distance (Euclidean distance)''' - return sqrt(self.norm2Squared()) - - def norm1(self): - return abs(self.x)+abs(self.y) - - def normMax(self): - return max(abs(self.x),abs(self.y)) - - def aslist(self): - return [self.x, self.y] - - def astuple(self): - return (self.x, self.y) - - def asint(self): - return Point(int(self.x), int(self.y)) - - if shapelyAvailable: - def asShapely(self): - return shapelyPoint(self.x, self.y) - - def homographyProject(self, homography): - projected = cvutils.homographyProject(array([[self.x], [self.y]]), homography) - return Point(projected[0], projected[1]) - - def inPolygon(self, polygon): - '''Indicates if the point x, y is inside the polygon - (array of Nx2 coordinates of the polygon vertices) - - taken from http://www.ariel.com.au/a/python-point-int-poly.html - - Use Polygon.contains if Shapely is installed''' - - n = polygon.shape[0]; - counter = 0; - - p1 = polygon[0,:]; - for i in range(n+1): - p2 = polygon[i % n,:]; - if self.y > min(p1[1],p2[1]): - if self.y <= max(p1[1],p2[1]): - if self.x <= max(p1[0],p2[0]): - if p1[1] != p2[1]: - xinters = (self.y-p1[1])*(p2[0]-p1[0])/(p2[1]-p1[1])+p1[0]; - if p1[0] == p2[0] or self.x <= xinters: - counter+=1; - p1=p2 - return (counter%2 == 1); - - @staticmethod - def fromList(p): - return Point(p[0], p[1]) - - @staticmethod - def dot(p1, p2): - 'Scalar product' - return p1.x*p2.x+p1.y*p2.y - - @staticmethod - def cross(p1, p2): - 'Cross product' - return p1.x*p2.y-p1.y*p2.x - - @staticmethod - def parallel(p1, p2): - return Point.cross(p1, p2) == 0. - - @staticmethod - def cosine(p1, p2): - return Point.dot(p1,p2)/(p1.norm2()*p2.norm2()) - - @staticmethod - def distanceNorm2(p1, p2): - return (p1-p2).norm2() - - @staticmethod - def plotAll(points, **kwargs): - from matplotlib.pyplot import scatter - scatter([p.x for p in points],[p.y for p in points], **kwargs) - - def similarOrientation(self, refDirection, cosineThreshold): - 'Indicates whether the cosine of the vector and refDirection is smaller than cosineThreshold' - return Point.cosine(self, refDirection) >= cosineThreshold - - @staticmethod - def timeToCollision(p1, p2, v1, v2, collisionThreshold): - '''Computes exact time to collision with a distance threshold - The unknown of the equation is the time to reach the intersection - between the relative trajectory of one road user - and the circle of radius collisionThreshold around the other road user''' - dv = v1-v2 - dp = p1-p2 - a = dv.norm2Squared()#(v1.x-v2.x)**2 + (v1.y-v2.y)**2 - b = 2*Point.dot(dv, dp)#2 * ((p1.x-p2.x) * (v1.x-v2.x) + (p1.y-p2.y) * (v1.y-v2.y)) - c = dp.norm2Squared() - collisionThreshold**2#(p1.x-p2.x)**2 + (p1.y-p2.y)**2 - collisionThreshold**2 - - delta = b**2 - 4*a*c - if delta >= 0: - deltaRoot = sqrt(delta) - ttc1 = (-b + deltaRoot)/(2*a) - ttc2 = (-b - deltaRoot)/(2*a) - if ttc1 >= 0 and ttc2 >= 0: - return min(ttc1,ttc2) - elif ttc1 >= 0: - return ttc1 - elif ttc2 >= 0: - return ttc2 - else: # ttc1 < 0 and ttc2 < 0: - return None - else: - return None - - @staticmethod - def midPoint(p1, p2): - 'Returns the middle of the segment [p1, p2]' - return Point(0.5*p1.x+0.5*p2.x, 0.5*p1.y+0.5*p2.y) - - @staticmethod - def agg(points, aggFunc = mean): - return Point(aggFunc([p.x for p in points]), aggFunc([p.y for p in points])) - -if shapelyAvailable: - def pointsInPolygon(points, polygon): - '''Optimized tests of a series of points within (Shapely) polygon (not prepared)''' - if type(polygon) == PreparedGeometry: - prepared_polygon = polygon - else: - prepared_polygon = prep(polygon) - return list(filter(prepared_polygon.contains, points)) - -# Functions for coordinate transformation -# From Paul St-Aubin's PVA tools -def subsec_spline_dist(splines): - ''' Prepare list of spline subsegments from a spline list. - - Output: - ======= - ss_spline_d[spline #][mode][station] - - where: - mode=0: incremental distance - mode=1: cumulative distance - mode=2: cumulative distance with trailing distance - ''' - ss_spline_d = [] - #Prepare subsegment distances - for spline in range(len(splines)): - ss_spline_d[spline]=[]#.append([[],[],[]]) - ss_spline_d[spline].append(zeros(len(splines[spline])-1)) #Incremental distance - ss_spline_d[spline].append(zeros(len(splines[spline])-1)) #Cumulative distance - ss_spline_d[spline].append(zeros(len(splines[spline]))) #Cumulative distance with trailing distance - for spline_p in range(len(splines[spline])): - if spline_p > (len(splines[spline]) - 2): - break - ss_spline_d[spline][0][spline_p] = utils.pointDistanceL2(splines[spline][spline_p][0],splines[spline][spline_p][1],splines[spline][(spline_p+1)][0],splines[spline][(spline_p+1)][1]) - ss_spline_d[spline][1][spline_p] = sum(ss_spline_d[spline][0][0:spline_p]) - ss_spline_d[spline][2][spline_p] = ss_spline_d[spline][1][spline_p]#sum(ss_spline_d[spline][0][0:spline_p]) - - ss_spline_d[spline][2][-1] = ss_spline_d[spline][2][-2] + ss_spline_d[spline][0][-1] - - return ss_spline_d - -def prepareSplines(splines): - 'Approximates slope singularity by giving some slope roundoff; account for roundoff error' - for spline in splines: - p1 = spline[0] - for i in range(len(spline)-1): - p2 = spline[i+1] - if(round(p1.x, 10) == round(p2.x, 10)): - p2.x += 0.0000000001 - if(round(p1.y, 10) == round(p2.y, 10)): - p2.y += 0.0000000001 - p1 = p2 - -def ppldb2p(qx,qy, p0x,p0y, p1x,p1y): - ''' Point-projection (Q) on line defined by 2 points (P0,P1). - http://cs.nyu.edu/~yap/classes/visual/03s/hw/h2/math.pdf - ''' - if(p0x == p1x and p0y == p1y): - return None - try: - #Approximate slope singularity by giving some slope roundoff; account for roundoff error - # if(round(p0x, 10) == round(p1x, 10)): - # p1x += 0.0000000001 - # if(round(p0y, 10) == round(p1y, 10)): - # p1y += 0.0000000001 - #make the calculation - Y = (-(qx)*(p0y-p1y)-(qy*(p0y-p1y)**2)/(p0x-p1x)+p0x**2*(p0y-p1y)/(p0x-p1x)-p0x*p1x*(p0y-p1y)/(p0x-p1x)-p0y*(p0x-p1x))/(p1x-p0x-(p0y-p1y)**2/(p0x-p1x)) - X = (-Y*(p1y-p0y)+qx*(p1x-p0x)+qy*(p1y-p0y))/(p1x-p0x) - except ZeroDivisionError: - print('Error: Division by zero in ppldb2p. Please report this error with the full traceback:') - print('qx={0}, qy={1}, p0x={2}, p0y={3}, p1x={4}, p1y={5}...'.format(qx, qy, p0x, p0y, p1x, p1y)) - import pdb; pdb.set_trace() - return Point(X,Y) - -def getSYfromXY(p, splines, goodEnoughSplineDistance = 0.5): - ''' Snap a point p to its nearest subsegment of it's nearest spline (from the list splines). - A spline is a list of points (class Point), most likely a trajectory. - - Output: - ======= - [spline index, - subsegment leading point index, - snapped point, - subsegment distance, - spline distance, - orthogonal point offset] - - or None - ''' - minOffsetY = float('inf') - #For each spline - for splineIdx in range(len(splines)): - #For each spline point index - for spline_p in range(len(splines[splineIdx])-1): - #Get closest point on spline - closestPoint = ppldb2p(p.x,p.y,splines[splineIdx][spline_p][0],splines[splineIdx][spline_p][1],splines[splineIdx][spline_p+1][0],splines[splineIdx][spline_p+1][1]) - if closestPoint is None: - print('Error: Spline {0}, segment {1} has identical bounds and therefore is not a vector. Projection cannot continue.'.format(splineIdx, spline_p)) - return None - # check if the projected point is in between the current segment of the alignment bounds - if utils.inBetween(splines[splineIdx][spline_p][0], splines[splineIdx][spline_p+1][0], closestPoint.x) and utils.inBetween(splines[splineIdx][spline_p][1], splines[splineIdx][spline_p+1][1], closestPoint.y): - offsetY = Point.distanceNorm2(closestPoint, p) - if offsetY < minOffsetY: - minOffsetY = offsetY - snappedSplineIdx = splineIdx - snappedSplineLeadingPoint = spline_p - snappedPoint = Point(closestPoint.x, closestPoint.y) - #Jump loop if significantly close - if offsetY < goodEnoughSplineDistance: - break - - #Get sub-segment distance - if minOffsetY != float('inf'): - subsegmentDistance = Point.distanceNorm2(snappedPoint, splines[snappedSplineIdx][snappedSplineLeadingPoint]) - #Get cumulative alignment distance (total segment distance) - splineDistanceS = splines[snappedSplineIdx].getCumulativeDistance(snappedSplineLeadingPoint) + subsegmentDistance - orthogonalSplineVector = (splines[snappedSplineIdx][snappedSplineLeadingPoint+1]-splines[snappedSplineIdx][snappedSplineLeadingPoint]).orthogonal() - offsetVector = p-snappedPoint - if Point.dot(orthogonalSplineVector, offsetVector) < 0: - minOffsetY = -minOffsetY - return [snappedSplineIdx, snappedSplineLeadingPoint, snappedPoint, subsegmentDistance, splineDistanceS, minOffsetY] - else: - print('Offset for point {} is infinite (check with prepareSplines if some spline segments are aligned with axes)'.format(p)) - return None - -def getXYfromSY(s, y, splineNum, splines, mode = 0): - ''' Find X,Y coordinate from S,Y data. - if mode = 0 : return Snapped X,Y - if mode !=0 : return Real X,Y - ''' - - #(buckle in, it gets ugly from here on out) - ss_spline_d = subsec_spline_dist(splines) - - #Find subsegment - snapped_x = None - snapped_y = None - for spline_ss_index in range(len(ss_spline_d[splineNum][1])): - if(s < ss_spline_d[splineNum][1][spline_ss_index]): - ss_value = s - ss_spline_d[splineNum][1][spline_ss_index-1] - #Get normal vector and then snap - vector_l_x = (splines[splineNum][spline_ss_index][0] - splines[splineNum][spline_ss_index-1][0]) - vector_l_y = (splines[splineNum][spline_ss_index][1] - splines[splineNum][spline_ss_index-1][1]) - magnitude = sqrt(vector_l_x**2 + vector_l_y**2) - n_vector_x = vector_l_x/magnitude - n_vector_y = vector_l_y/magnitude - snapped_x = splines[splineNum][spline_ss_index-1][0] + ss_value*n_vector_x - snapped_y = splines[splineNum][spline_ss_index-1][1] + ss_value*n_vector_y - - #Real values (including orthogonal projection of y)) - real_x = snapped_x - y*n_vector_y - real_y = snapped_y + y*n_vector_x - break - - if mode == 0 or (not snapped_x): - if(not snapped_x): - snapped_x = splines[splineNum][-1][0] - snapped_y = splines[splineNum][-1][1] - return [snapped_x,snapped_y] - else: - return [real_x,real_y] - - -class NormAngle(object): - '''Alternate encoding of a point, by its norm and orientation''' - - def __init__(self, norm, angle): - self.norm = norm - self.angle = angle - - @staticmethod - def fromPoint(p): - norm = p.norm2() - if norm > 0: - angle = p.angle() - else: - angle = 0. - return NormAngle(norm, angle) - - def __add__(self, other): - 'a norm cannot become negative' - return NormAngle(max(self.norm+other.norm, 0), self.angle+other.angle) - - def getPoint(self): - return Point(self.norm*cos(self.angle), self.norm*sin(self.angle)) - - -def predictPositionNoLimit(nTimeSteps, initialPosition, initialVelocity, initialAcceleration = Point(0,0)): - '''Predicts the position in nTimeSteps at constant speed/acceleration''' - return initialVelocity + initialAcceleration.__mul__(nTimeSteps),initialPosition+initialVelocity.__mul__(nTimeSteps) + initialAcceleration.__mul__(nTimeSteps**2*0.5) - -def predictPosition(position, speedOrientation, control, maxSpeed = None): - '''Predicts the position (moving.Point) at the next time step with given control input (deltaSpeed, deltaTheta) - speedOrientation is the other encoding of velocity, (speed, orientation) - speedOrientation and control are NormAngle''' - predictedSpeedTheta = speedOrientation+control - if maxSpeed is not None: - predictedSpeedTheta.norm = min(predictedSpeedTheta.norm, maxSpeed) - predictedPosition = position+predictedSpeedTheta.getPoint() - return predictedPosition, predictedSpeedTheta - - -class FlowVector(object): - '''Class to represent 4-D flow vectors, - ie a position and a velocity''' - def __init__(self, position, velocity): - 'position and velocity should be Point instances' - self.position = position - self.velocity = velocity - - def __add__(self, other): - return FlowVector(self.position+other.position, self.velocity+other.velocity) - - def __mul__(self, alpha): - return FlowVector(self.position.__mul__(alpha), self.velocity.__mul__(alpha)) - - def plot(self, options = '', **kwargs): - plot([self.position.x, self.position.x+self.velocity.x], [self.position.y, self.position.y+self.velocity.y], options, **kwargs) - self.position.plot(options+'x', **kwargs) - - @staticmethod - def similar(f1, f2, maxDistance2, maxDeltavelocity2): - return (f1.position-f2.position).norm2Squared()<maxDistance2 and (f1.velocity-f2.velocity).norm2Squared()<maxDeltavelocity2 - -def intersection(p1, p2, p3, p4): - ''' Intersection point (x,y) of lines formed by the vectors p1-p2 and p3-p4 - http://paulbourke.net/geometry/pointlineplane/''' - dp12 = p2-p1 - dp34 = p4-p3 - #det = (p4.y-p3.y)*(p2.x-p1.x)-(p4.x-p3.x)*(p2.y-p1.y) - det = float(dp34.y*dp12.x-dp34.x*dp12.y) - if det == 0.: - return None - else: - ua = (dp34.x*(p1.y-p3.y)-dp34.y*(p1.x-p3.x))/det - return p1+dp12.__mul__(ua) - -# def intersection(p1, p2, dp1, dp2): -# '''Returns the intersection point between the two lines -# defined by the respective vectors (dp) and origin points (p)''' -# from numpy import matrix -# from numpy.linalg import linalg -# A = matrix([[dp1.y, -dp1.x], -# [dp2.y, -dp2.x]]) -# B = matrix([[dp1.y*p1.x-dp1.x*p1.y], -# [dp2.y*p2.x-dp2.x*p2.y]]) - -# if linalg.det(A) == 0: -# return None -# else: -# intersection = linalg.solve(A,B) -# return Point(intersection[0,0], intersection[1,0]) - -def segmentIntersection(p1, p2, p3, p4): - '''Returns the intersecting point of the segments [p1, p2] and [p3, p4], None otherwise''' - - if (Interval.intersection(Interval(p1.x,p2.x,True), Interval(p3.x,p4.x,True)).empty()) or (Interval.intersection(Interval(p1.y,p2.y,True), Interval(p3.y,p4.y,True)).empty()): - return None - else: - inter = intersection(p1, p2, p3, p4) - if (inter is not None - and utils.inBetween(p1.x, p2.x, inter.x) - and utils.inBetween(p3.x, p4.x, inter.x) - and utils.inBetween(p1.y, p2.y, inter.y) - and utils.inBetween(p3.y, p4.y, inter.y)): - return inter - else: - return None - -def segmentLineIntersection(p1, p2, p3, p4): - '''Indicates if the line going through p1 and p2 intersects inside p3, p4''' - inter = intersection(p1, p2, p3, p4) - if inter is not None and utils.inBetween(p3.x, p4.x, inter.x) and utils.inBetween(p3.y, p4.y, inter.y): - return inter - else: - return None - - -class Trajectory(object): - '''Class for trajectories: temporal sequence of positions - - The class is iterable''' - - def __init__(self, positions=None): - if positions is not None: - self.positions = positions - else: - self.positions = [[],[]] - - @staticmethod - def generate(p, v, nPoints): - t = Trajectory() - p0 = Point(p.x, p.y) - t.addPosition(p0) - for i in range(nPoints-1): - p0 += v - t.addPosition(p0) - return t, Trajectory([[v.x]*nPoints, [v.y]*nPoints]) - - @staticmethod - def load(line1, line2): - return Trajectory([[float(n) for n in line1.split(' ')], - [float(n) for n in line2.split(' ')]]) - - @staticmethod - def fromPointList(points): - t = Trajectory() - if isinstance(points[0], list) or isinstance(points[0], tuple): - for p in points: - t.addPositionXY(p[0],p[1]) - else: - for p in points: - t.addPosition(p) - return t - - def __len__(self): - return len(self.positions[0]) - - def length(self): - return self.__len__() - - def empty(self): - return self.__len__() == 0 - - def __getitem__(self, i): - if isinstance(i, int): - return Point(self.positions[0][i], self.positions[1][i]) - elif isinstance(i, slice): - return Trajectory([self.positions[0][i],self.positions[1][i]]) - else: - raise TypeError("Invalid argument type.") - - def __str__(self): - return ' '.join([self.__getitem__(i).__str__() for i in range(self.length())]) - - def __repr__(self): - return self.__str__() - - def __iter__(self): - self.iterInstantNum = 0 - return self - - def __next__(self): - if self.iterInstantNum >= self.length(): - raise StopIteration - else: - self.iterInstantNum += 1 - return self[self.iterInstantNum-1] - - def __eq__(self, other): - if self.length() == other.length(): - result = True - for p, po in zip(self, other): - result = result and (p == po) - return result - else: - return False - - def setPositionXY(self, i, x, y): - if i < self.__len__(): - self.positions[0][i] = x - self.positions[1][i] = y - - def setPosition(self, i, p): - self.setPositionXY(i, p.x, p.y) - - def addPositionXY(self, x, y): - self.positions[0].append(x) - self.positions[1].append(y) - - def addPosition(self, p): - self.addPositionXY(p.x, p.y) - - def duplicateLastPosition(self): - self.positions[0].append(self.positions[0][-1]) - self.positions[1].append(self.positions[1][-1]) - - @staticmethod - def _plot(positions, options = '', withOrigin = False, lastCoordinate = None, timeStep = 1, objNum = None, **kwargs): - if lastCoordinate is None: - plot(positions[0][::timeStep], positions[1][::timeStep], options, **kwargs) - elif 0 <= lastCoordinate <= len(positions[0]): - plot(positions[0][:lastCoordinate:timeStep], positions[1][:lastCoordinate:timeStep], options, **kwargs) - if withOrigin: - plot([positions[0][0]], [positions[1][0]], 'ro', **kwargs) - if objNum is not None: - text(positions[0][0], positions[1][0], '{}'.format(objNum)) - - def homographyProject(self, homography): - return Trajectory(cvutils.homographyProject(array(self.positions), homography).tolist()) - - def newCameraProject(self, newCameraMatrix): - return Trajectory(cvutils.newCameraProject(array(self.positions), newCameraMatrix).tolist()) - - def plot(self, options = '', withOrigin = False, timeStep = 1, objNum = None, **kwargs): - Trajectory._plot(self.positions, options, withOrigin, None, timeStep, objNum, **kwargs) - - def plotAt(self, lastCoordinate, options = '', withOrigin = False, timeStep = 1, objNum = None, **kwargs): - Trajectory._plot(self.positions, options, withOrigin, lastCoordinate, timeStep, objNum, **kwargs) - - def plotOnWorldImage(self, nPixelsPerUnitDistance, options = '', withOrigin = False, timeStep = 1, objNum = None, **kwargs): - imgPositions = [[x*nPixelsPerUnitDistance for x in self.positions[0]], - [x*nPixelsPerUnitDistance for x in self.positions[1]]] - Trajectory._plot(imgPositions, options, withOrigin, None, timeStep, objNum, **kwargs) - - def getXCoordinates(self): - return self.positions[0] - - def getYCoordinates(self): - return self.positions[1] - - def asArray(self): - return array(self.positions) - - def xBounds(self): - # look for function that does min and max in one pass - return Interval(min(self.getXCoordinates()), max(self.getXCoordinates())) - - def yBounds(self): - # look for function that does min and max in one pass - return Interval(min(self.getYCoordinates()), max(self.getYCoordinates())) - - def add(self, traj2): - '''Returns a new trajectory of the same length''' - if self.length() != traj2.length(): - print('Trajectories of different lengths') - return None - else: - return Trajectory([[a+b for a,b in zip(self.getXCoordinates(),traj2.getXCoordinates())], - [a+b for a,b in zip(self.getYCoordinates(),traj2.getYCoordinates())]]) - - def subtract(self, traj2): - '''Returns a new trajectory of the same length''' - if self.length() != traj2.length(): - print('Trajectories of different lengths') - return None - else: - return Trajectory([[a-b for a,b in zip(self.getXCoordinates(),traj2.getXCoordinates())], - [a-b for a,b in zip(self.getYCoordinates(),traj2.getYCoordinates())]]) - - def __mul__(self, alpha): - '''Returns a new trajectory of the same length''' - return Trajectory([[alpha*x for x in self.getXCoordinates()], - [alpha*y for y in self.getYCoordinates()]]) - - def differentiate(self, doubleLastPosition = False): - diff = Trajectory() - for i in range(1, self.length()): - diff.addPosition(self[i]-self[i-1]) - if doubleLastPosition: - diff.addPosition(diff[-1]) - return diff - - def differentiateSG(self, window_length, polyorder, deriv=0, delta=1.0, axis=-1, mode='interp', cval=0.0, nInstantsIgnoredAtEnds = 2): - '''Differentiates the trajectory using the Savitsky Golay filter - - window_length : The length of the filter window (i.e. the number of coefficients). window_length must be a positive odd integer. - polyorder : The order of the polynomial used to fit the samples. polyorder must be less than window_length. - deriv : The order of the derivative to compute. This must be a nonnegative integer. The default is 0, which means to filter the data without differentiating. - delta : The spacing of the samples to which the filter will be applied. This is only used if deriv > 0. Default is 1.0. - axis : The axis of the array x along which the filter is to be applied. Default is -1. - mode : Must be mirror, constant, nearest, wrap or interp. This determines the type of extension to use for the padded signal to which the filter is applied. When mode is constant, the padding value is given by cval. See the Notes for more details on mirror, constant, wrap, and nearest. When the interp mode is selected (the default), no extension is used. Instead, a degree polyorder polynomial is fit to the last window_length values of the edges, and this polynomial is used to evaluate the last window_length // 2 output values. - cval : Value to fill past the edges of the input if mode is constant. Default is 0.0. - - https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.savgol_filter.html#scipy.signal.savgol_filter''' - if removeBothEnds >=1: - pos = [self.positions[0][nInstantsIgnoredAtEnds:-nInstantsIgnoredAtEnds], - self.positions[1][nInstantsIgnoredAtEnds:-nInstantsIgnoredAtEnds]] - else: - pos = self.positions - filtered = savgol_filter(pos, window_length, polyorder, deriv, delta, axis, mode, cval) - return Trajectory(filtered) - - def norm(self): - '''Returns the list of the norms at each instant''' - return hypot(self.positions[0], self.positions[1]) - - def computeCumulativeDistances(self): - '''Computes the distance from each point to the next and the cumulative distance up to the point - Can be accessed through getDistance(idx) and getCumulativeDistance(idx)''' - self.distances = [] - self.cumulativeDistances = [0.] - p1 = self[0] - cumulativeDistance = 0. - for i in range(self.length()-1): - p2 = self[i+1] - self.distances.append(Point.distanceNorm2(p1,p2)) - cumulativeDistance += self.distances[-1] - self.cumulativeDistances.append(cumulativeDistance) - p1 = p2 - - def getDistance(self,i): - '''Return the distance between points i and i+1''' - if i < self.length()-1: - return self.distances[i] - else: - print('Index {} beyond trajectory length {}-1'.format(i, self.length())) - - def getCumulativeDistance(self, i): - '''Return the cumulative distance between the beginning and point i''' - if i < self.length(): - return self.cumulativeDistances[i] - else: - print('Index {} beyond trajectory length {}'.format(i, self.length())) - - def getMaxDistance(self, metric): - 'Returns the maximum distance between points in the trajectory' - positions = self.getPositions().asArray().T - return cdist(positions, positions, metric = metric).max() - - def getClosestPoint(self, p1, maxDist2 = None): - '''Returns the instant of the closest position in trajectory to p1 (and the point) - if maxDist is not None, will check the distance is smaller - TODO: could use cdist for different metrics''' - distances2 = [] - minDist2 = float('inf') - i = -1 - for p2 in self: - distances2.append(Point.distanceNorm2(p1, p2)) - if distances2[-1] < minDist2: - minDist2 = distances2[-1] - i = len(distances2)-1 - if maxDist2 is not None and minDist2 < maxDist2: - return None - else: - return i - - def similarOrientation(self, refDirection, cosineThreshold, minProportion = 0.5): - '''Indicates whether the minProportion (<=1.) (eg half) of the trajectory elements (vectors for velocity) - have a cosine with refDirection is smaller than cosineThreshold''' - count = 0 - lengthThreshold = float(self.length())*minProportion - for p in self: - if p.similarOrientation(refDirection, cosineThreshold): - count += 1 - return count >= lengthThreshold - - def wiggliness(self): - straightDistance = Point.distanceNorm2(self.__getitem__(0),self.__getitem__(self.length()-1)) - if straightDistance > 0: - return self.getCumulativeDistance(self.length()-1)/float(straightDistance) - else: - return None - - def getIntersections(self, p1, p2): - '''Returns a list of the indices at which the trajectory - intersects with the segment of extremities p1 and p2 - Returns an empty list if there is no crossing''' - indices = [] - intersections = [] - - for i in range(self.length()-1): - q1=self.__getitem__(i) - q2=self.__getitem__(i+1) - p = segmentIntersection(q1, q2, p1, p2) - if p is not None: - if q1.x != q2.x: - ratio = (p.x-q1.x)/(q2.x-q1.x) - elif q1.y != q2.y: - ratio = (p.y-q1.y)/(q2.y-q1.y) - else: - ratio = 0 - indices.append(i+ratio) - intersections.append(p) - return indices, intersections - - def getLineIntersections(self, p1, p2): - '''Returns a list of the indices at which the trajectory - intersects with the line going through p1 and p2 - Returns an empty list if there is no crossing''' - indices = [] - intersections = [] - - for i in range(self.length()-1): - q1=self.__getitem__(i) - q2=self.__getitem__(i+1) - p = segmentLineIntersection(p1, p2, q1, q2) - if p is not None: - if q1.x != q2.x: - ratio = (p.x-q1.x)/(q2.x-q1.x) - elif q1.y != q2.y: - ratio = (p.y-q1.y)/(q2.y-q1.y) - else: - ratio = 0 - indices.append(i+ratio) - intersections.append(p) - return indices, intersections - - def getTrajectoryInInterval(self, inter): - 'Returns all position between index inter.first and index.last (included)' - if inter.first >=0 and inter.last<= self.length(): - return Trajectory([self.positions[0][inter.first:inter.last+1], - self.positions[1][inter.first:inter.last+1]]) - else: - return None - - def subSample(self, step): - 'Returns the positions very step' - return Trajectory([self.positions[0][::step], - self.positions[1][::step]]) - - if shapelyAvailable: - def getInstantsInPolygon(self, polygon): - '''Returns the list of instants at which the trajectory is in the polygon''' - instants = [] - n = self.length() - for t, x, y in zip(range(n), self.positions[0], self.positions[1]): - if polygon.contains(shapelyPoint(x, y)): - instants.append(t) - return instants - - def getTrajectoryInPolygon(self, polygon, t2 = None): - '''Returns the trajectory built with the set of points inside the (shapely) polygon - The polygon could be a prepared polygon (faster) from prepared.prep - - t2 is another trajectory (could be velocities) - which is filtered based on the first (self) trajectory''' - traj = Trajectory() - inPolygon = [] - for x, y in zip(self.positions[0], self.positions[1]): - inPolygon.append(polygon.contains(shapelyPoint(x, y))) - if inPolygon[-1]: - traj.addPositionXY(x, y) - traj2 = Trajectory() - if t2 is not None: - for inp, x, y in zip(inPolygon, t2.positions[0], t2.positions[1]): - if inp: - traj2.addPositionXY(x, y) - return traj, traj2 - - def proportionInPolygon(self, polygon, minProportion = 0.5): - instants = self.getInstantsInPolygon(polygon) - lengthThreshold = float(self.length())*minProportion - return len(instants) >= lengthThreshold - else: - def getTrajectoryInPolygon(self, polygon, t2 = None): - '''Returns the trajectory built with the set of points inside the polygon - (array of Nx2 coordinates of the polygon vertices)''' - traj = Trajectory() - inPolygon = [] - for p in self: - inPolygon.append(p.inPolygon(polygon)) - if inPolygon[-1]: - traj.addPosition(p) - traj2 = Trajectory() - if t2 is not None: - for inp, x, y in zip(inPolygon, t2.positions[0], t2.positions[1]): - if inp: - traj2.addPositionXY(p.x, p.y) - return traj, traj2 - - def proportionInPolygon(self, polygon, minProportion = 0.5): - inPolygon = [p.inPolygon(polygon) for p in self] - lengthThreshold = float(self.length())*minProportion - return sum(inPolygon) >= lengthThreshold - - @staticmethod - def lcss(t1, t2, lcss): - return lcss.compute(t1, t2) - -class CurvilinearTrajectory(Trajectory): - '''Sub class of trajectory for trajectories with curvilinear coordinates and lane assignements - longitudinal coordinate is stored as first coordinate (exterior name S) - lateral coordiante is stored as second coordinate''' - - def __init__(self, S = None, Y = None, lanes = None): - if S is None or Y is None or len(S) != len(Y): - self.positions = [[],[]] - if S is not None and Y is not None and len(S) != len(Y): - print("S and Y coordinates of different lengths\nInitializing to empty lists") - else: - self.positions = [S,Y] - if lanes is None or len(lanes) != self.length(): - self.lanes = [] - else: - self.lanes = lanes - - def __getitem__(self,i): - if isinstance(i, int): - return [self.positions[0][i], self.positions[1][i], self.lanes[i]] - else: - raise TypeError("Invalid argument type.") - #elif isinstance( key, slice ): - - def getSCoordinates(self): - return self.getXCoordinates() - - def getLanes(self): - return self.lanes - - def addPositionSYL(self, s, y, lane): - self.addPositionXY(s,y) - self.lanes.append(lane) - - def addPosition(self, p): - 'Adds position in the point format for curvilinear of list with 3 values' - self.addPositionSYL(p[0], p[1], p[2]) - - def setPosition(self, i, s, y, lane): - self.setPositionXY(i, s, y) - if i < self.__len__(): - self.lanes[i] = lane - - def differentiate(self, doubleLastPosition = False): - diff = CurvilinearTrajectory() - p1 = self[0] - for i in range(1, self.length()): - p2 = self[i] - diff.addPositionSYL(p2[0]-p1[0], p2[1]-p1[1], p1[2]) - p1=p2 - if doubleLastPosition and self.length() > 1: - diff.addPosition(diff[-1]) - return diff - - def getIntersections(self, S1, lane = None): - '''Returns a list of the indices at which the trajectory - goes past the curvilinear coordinate S1 - (in provided lane if lane is not None) - Returns an empty list if there is no crossing''' - indices = [] - for i in range(self.length()-1): - q1=self.__getitem__(i) - q2=self.__getitem__(i+1) - if q1[0] <= S1 < q2[0] and (lane is None or (self.lanes[i] == lane and self.lanes[i+1] == lane)): - indices.append(i+(S1-q1[0])/(q2[0]-q1[0])) - return indices - -################## -# Moving Objects -################## - -userTypeNames = ['unknown', - 'car', - 'pedestrian', - 'motorcycle', - 'bicycle', - 'bus', - 'truck'] - -userType2Num = utils.inverseEnumeration(userTypeNames) - -class CarClassifier: - def predict(self, hog): - return userType2Num['car'] -carClassifier = CarClassifier() - -class MovingObject(STObject, VideoFilenameAddable): - '''Class for moving objects: a spatio-temporal object - with a trajectory and a geometry (constant volume over time) - and a usertype (e.g. road user) coded as a number (see userTypeNames) - ''' - - def __init__(self, num = None, timeInterval = None, positions = None, velocities = None, geometry = None, userType = userType2Num['unknown']): - super(MovingObject, self).__init__(num, timeInterval) - self.positions = positions - self.velocities = velocities - self.geometry = geometry - self.userType = userType - self.features = None - # compute bounding polygon from trajectory - - @staticmethod - def aggregateTrajectories(features, interval = None, aggFunc = mean): - 'Computes the aggregate trajectory from list of MovingObject features' - positions = Trajectory() - velocities = Trajectory() - if interval is None: - inter = TimeInterval.unionIntervals([f.getTimeInterval() for f in features]) - else: - inter = interval - for t in inter: - points = [] - vels = [] - for f in features: - if f.existsAtInstant(t): - points.append(f.getPositionAtInstant(t)) - vels.append(f.getVelocityAtInstant(t)) - positions.addPosition(Point.agg(points, aggFunc)) - velocities.addPosition(Point.agg(vels, aggFunc)) - return inter, positions, velocities - - @staticmethod - def generate(num, p, v, timeInterval): - positions, velocities = Trajectory.generate(p, v, int(timeInterval.length())) - return MovingObject(num = num, timeInterval = timeInterval, positions = positions, velocities = velocities) - - def updatePositions(self): - inter, self.positions, self.velocities = MovingObject.aggregateTrajectories(self.features, self.getTimeInterval()) - - @staticmethod - def concatenate(obj1, obj2, num = None, newFeatureNum = None, computePositions = False): - '''Concatenates two objects, whether overlapping temporally or not - - Positions will be recomputed only if computePositions is True - Otherwise, only featureNumbers and/or features will be merged''' - if num is None: - newNum = obj1.getNum() - else: - newNum = num - commonTimeInterval = obj1.commonTimeInterval(obj2) - if commonTimeInterval.empty(): - #print('The two objects\' time intervals do not overlap: obj1 {} and obj2 {}'.format(obj1.getTimeInterval(), obj2.getTimeInterval())) - emptyInterval = TimeInterval(min(obj1.getLastInstant(),obj2.getLastInstant()), max(obj1.getFirstInstant(),obj2.getFirstInstant())) - if obj1.existsAtInstant(emptyInterval.last): - firstObject = obj2 - secondObject = obj1 - else: - firstObject = obj1 - secondObject = obj2 - v = (secondObject.getPositionAtInstant(emptyInterval.last)-firstObject.getPositionAtInstant(emptyInterval.first)).divide(emptyInterval.length()-1) - positions = copy.deepcopy(firstObject.getPositions()) - velocities = copy.deepcopy(firstObject.getPositions()) - featurePositions = Trajectory() - featureVelocities = Trajectory() - p = firstObject.getPositionAtInstant(emptyInterval.first)+v - for t in range(emptyInterval.first+1, emptyInterval.last): - positions.addPosition(p) - velocities.addPosition(v) - featurePositions.addPosition(p) - featureVelocities.addPosition(v) - p=p+v - for t in secondObject.getTimeInterval(): - positions.addPosition(secondObject.getPositionAtInstant(t)) - velocities.addPosition(secondObject.getVelocityAtInstant(t)) - newObject = MovingObject(newNum, TimeInterval(firstObject.getFirstInstant(), secondObject.getLastInstant()), positions, velocities) - if hasattr(obj1, 'featureNumbers') and hasattr(obj2, 'featureNumbers'): - if newFeatureNum is not None: - newObject.featureNumbers = obj1.featureNumbers+obj2.featureNumbers+[newFeatureNum] - else: - print('Issue, new created feature has no num id') - if obj1.hasFeatures() and obj2.hasFeatures(): - newObject.features = obj1.getFeatures()+obj2.getFeatures()+[MovingObject(newFeatureNum, TimeInterval(emptyInterval.first+1, emptyInterval.last-1), featurePositions, featureVelocities)] - else: # time intervals overlap - newTimeInterval = TimeInterval.union(obj1.getTimeInterval(), obj2.getTimeInterval()) - newObject = MovingObject(newNum, newTimeInterval) - if hasattr(obj1, 'featureNumbers') and hasattr(obj2, 'featureNumbers'): - newObject.featureNumbers = obj1.featureNumbers+obj2.featureNumbers - if obj1.hasFeatures() and obj2.hasFeatures(): - newObject.features = obj1.getFeatures()+obj2.getFeatures() - newObject.updatePositions() - else: - print('Cannot update object positions without features') - # user type - if obj1.getUserType() != obj2.getUserType(): - print('The two moving objects have different user types: obj1 {} obj2 {}'.format(userTypeNames[obj1.getUserType()], userTypeNames[obj2.getUserType()])) - newObject.setUserType(obj1.getUserType()) - return newObject - - def getObjectInTimeInterval(self, inter): - '''Returns a new object extracted from self, - restricted to time interval inter''' - intersection = TimeInterval.intersection(inter, self.getTimeInterval()) - if not intersection.empty(): - trajectoryInterval = TimeInterval(intersection.first-self.getFirstInstant(), intersection.last-self.getFirstInstant()) - obj = MovingObject(self.num, intersection, self.positions.getTrajectoryInInterval(trajectoryInterval), self.geometry, self.userType) - if self.velocities is not None: - obj.velocities = self.velocities.getTrajectoryInInterval(trajectoryInterval) - return obj - else: - print('The object does not exist at {}'.format(inter)) - return None - - def getObjectsInMask(self, mask, homography = None, minLength = 1): - '''Returns new objects made of the positions in the mask - mask is in the destination of the homography space''' - if homography is not None: - self.projectedPositions = self.positions.homographyProject(homography) - else: - self.projectedPositions = self.positions - def inMask(positions, i, mask): - p = positions[i] - return mask[int(p.y), int(p.x)] != 0. - - #subTimeIntervals self.getFirstInstant()+i - filteredIndices = [inMask(self.projectedPositions, i, mask) for i in range(int(self.length()))] - # 'connected components' in subTimeIntervals - l = 0 - intervalLabels = [] - prev = True - for i in filteredIndices: - if i: - if not prev: # new interval - l += 1 - intervalLabels.append(l) - else: - intervalLabels.append(-1) - prev = i - intervalLabels = array(intervalLabels) - subObjects = [] - for l in set(intervalLabels): - if l >= 0: - if sum(intervalLabels == l) >= minLength: - times = [self.getFirstInstant()+i for i in range(len(intervalLabels)) if intervalLabels[i] == l] - subTimeInterval = TimeInterval(min(times), max(times)) - subObjects.append(self.getObjectInTimeInterval(subTimeInterval)) - - return subObjects - - def getPositions(self): - return self.positions - - def getVelocities(self): - return self.velocities - - def getUserType(self): - return self.userType - - def computeCumulativeDistances(self): - self.positions.computeCumulativeDistances() - - def getCurvilinearPositions(self): - if hasattr(self, 'curvilinearPositions'): - return self.curvilinearPositions - else: - return None - - def plotCurvilinearPositions(self, lane = None, options = '', withOrigin = False, **kwargs): - if hasattr(self, 'curvilinearPositions'): - if lane is None: - plot(list(self.getTimeInterval()), self.curvilinearPositions.positions[0], options, **kwargs) - if withOrigin: - plot([self.getFirstInstant()], [self.curvilinearPositions.positions[0][0]], 'ro', **kwargs) - else: - instants = [] - coords = [] - for t, p in zip(self.getTimeInterval(), self.curvilinearPositions): - if p[2] == lane: - instants.append(t) - coords.append(p[0]) - else: - instants.append(NaN) - coords.append(NaN) - plot(instants, coords, options, **kwargs) - if withOrigin and len(instants)>0: - plot([instants[0]], [coords[0]], 'ro', **kwargs) - else: - print('Object {} has no curvilinear positions'.format(self.getNum())) - - def setUserType(self, userType): - self.userType = userType - - def setFeatures(self, features, featuresOrdered = False): - '''Sets the features in the features field based on featureNumbers - if not all features are loaded from 0, one needs to renumber in a dict''' - if featuresOrdered: - tmp = features - else: - tmp = {f.getNum():f for f in features} - self.features = [tmp[i] for i in self.featureNumbers] - - def getFeatures(self): - return self.features - - def hasFeatures(self): - return (self.features is not None) - - def getFeature(self, i): - if self.hasFeatures() and i<len(self.features): - return self.features[i] - else: - return None - - def getNLongestFeatures(self, nFeatures = 1): - if self.features is None: - return [] - else: - tmp = utils.sortByLength(self.getFeatures(), reverse = True) - return tmp[:min(len(tmp), nFeatures)] - - def getFeatureNumbers(self): - '''Returns the number of features at each instant - dict instant -> number of features''' - if self.hasFeatures(): - featureNumbers = {} - for t in self.getTimeInterval(): - n = 0 - for f in self.getFeatures(): - if f.existsAtInstant(t): - n += 1 - featureNumbers[t]=n - return featureNumbers - else: - print('Object {} has no features loaded.'.format(self.getNum())) - return None - - def getSpeeds(self, nInstantsIgnoredAtEnds = 0): - speeds = self.getVelocities().norm() - if nInstantsIgnoredAtEnds > 0: - n = min(nInstantsIgnoredAtEnds, int(floor(self.length()/2.))) - return speeds[n:-n] - else: - return speeds - - def getAccelerations(self, window_length, polyorder, delta=1.0, axis=-1, mode='interp', cval=0.0, speeds = None, nInstantsIgnoredAtEnds = 0): - '''Returns the 1-D acceleration from the 1-D speeds - Caution about previously filtered data''' - if speeds is None: - speeds = self.getSpeeds(nInstantsIgnoredAtEnds) - return savgol_filter(speeds, window_length, polyorder, 1, delta, axis, mode, cval) - - def getSpeedIndicator(self): - from indicators import SeverityIndicator - return SeverityIndicator('Speed', {t:self.getVelocityAtInstant(t).norm2() for t in self.getTimeInterval()}) - - def getPositionAt(self, i): - return self.positions[i] - - def getVelocityAt(self, i): - return self.velocities[i] - - def getPositionAtInstant(self, i): - return self.positions[i-self.getFirstInstant()] - - def getVelocityAtInstant(self, i): - return self.velocities[i-self.getFirstInstant()] - - def getXCoordinates(self): - return self.positions.getXCoordinates() - - def getYCoordinates(self): - return self.positions.getYCoordinates() - - def plot(self, options = '', withOrigin = False, timeStep = 1, withFeatures = False, withIds = False, **kwargs): - if withIds: - objNum = self.getNum() - else: - objNum = None - if withFeatures and self.hasFeatures(): - for f in self.getFeatures(): - f.positions.plot('r', True, timeStep, **kwargs) - self.positions.plot('bx-', True, timeStep, objNum, **kwargs) - else: - self.positions.plot(options, withOrigin, timeStep, objNum, **kwargs) - - def plotOnWorldImage(self, nPixelsPerUnitDistance, options = '', withOrigin = False, timeStep = 1, withIds = False, **kwargs): - if withIds: - self.positions.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, self.getNum(), **kwargs) - else: - self.positions.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, None, **kwargs) - - def play(self, videoFilename, homography = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1.): - cvutils.displayTrajectories(videoFilename, [self], homography = homography, firstFrameNum = self.getFirstInstant(), lastFrameNumArg = self.getLastInstant(), undistort = undistort, intrinsicCameraMatrix = intrinsicCameraMatrix, distortionCoefficients = distortionCoefficients, undistortedImageMultiplication = undistortedImageMultiplication) - - def speedDiagnostics(self, framerate = 1., display = False, nInstantsIgnoredAtEnds=0): - speeds = framerate*self.getSpeeds(nInstantsIgnoredAtEnds) - coef = utils.linearRegression(list(range(len(speeds))), speeds) - print('min/5th perc speed: {} / {}\nspeed diff: {}\nspeed stdev: {}\nregression: {}'.format(min(speeds), scoreatpercentile(speeds, 5), speeds[-2]-speeds[1], std(speeds), coef[0])) - if display: - from matplotlib.pyplot import figure, axis - figure(1) - self.plot() - axis('equal') - figure(2) - plot(list(self.getTimeInterval()), speeds) - figure(3) - plot(list(self.getTimeInterval()), self.getAccelerations(9, 3, speeds = speeds)) # arbitrary parameter - - @staticmethod - def minMaxDistance(obj1, obj2): - '''Computes the min max distance used for feature grouping''' - commonTimeInterval = obj1.commonTimeInterval(obj2) - if not commonTimeInterval.empty(): - minDistance = (obj1.getPositionAtInstant(commonTimeInterval.first)-obj2.getPositionAtInstant(commonTimeInterval.first)).norm2() - maxDistance = minDistance - for t in list(commonTimeInterval)[1:]: - d = (obj1.getPositionAtInstant(t)-obj2.getPositionAtInstant(t)).norm2() - if d<minDistance: - minDistance = d - elif d>maxDistance: - maxDistance = d - return int(commonTimeInterval.length()), minDistance, maxDistance - else: - return int(commonTimeInterval.length()), None, None - - @staticmethod - def distances(obj1, obj2, instant1, _instant2 = None): - '''Returns the distances between all features of the 2 objects - at the same instant instant1 - or at instant1 and instant2''' - if _instant2 is None: - instant2 = instant1 - else: - instant2 = _instant2 - positions1 = [f.getPositionAtInstant(instant1).astuple() for f in obj1.features if f.existsAtInstant(instant1)] - positions2 = [f.getPositionAtInstant(instant2).astuple() for f in obj2.features if f.existsAtInstant(instant2)] - return cdist(positions1, positions2, metric = 'euclidean') - - @staticmethod - def minDistance(obj1, obj2, instant1, instant2 = None): - return MovingObject.distances(obj1, obj2, instant1, instant2).min() - - @staticmethod - def maxDistance(obj1, obj2, instant, instant2 = None): - return MovingObject.distances(obj1, obj2, instant1, instant2).max() - - def maxSize(self): - '''Returns the max distance between features - at instant there are the most features''' - if hasattr(self, 'features'): - nFeatures = -1 - tMaxFeatures = 0 - for t in self.getTimeInterval(): - n = len([f for f in self.features if f.existsAtInstant(t)]) - if n > nFeatures: - nFeatures = n - tMaxFeatures = t - return MovingObject.maxDistance(self, self, tMaxFeatures) - else: - print('Load features to compute a maximum size') - return None - - def setRoutes(self, startRouteID, endRouteID): - self.startRouteID = startRouteID - self.endRouteID = endRouteID - - def getInstantsCrossingLane(self, p1, p2): - '''Returns the instant(s) - at which the object passes from one side of the segment to the other - empty list if there is no crossing''' - indices, intersections = self.positions.getIntersections(p1, p2) - return [t+self.getFirstInstant() for t in indices] - - def computeTrajectorySimilarities(self, prototypes, lcss): - 'Computes the similarities to the prototypes using the LCSS' - if not hasattr(self, 'prototypeSimilarities'): - self.prototypeSimilarities = [] - for proto in prototypes: - lcss.similarities(proto.getMovingObject().getPositions().asArray().T, self.getPositions().asArray().T) - similarities = lcss.similarityTable[-1, :-1].astype(float) - self.prototypeSimilarities.append(similarities/minimum(arange(1.,len(similarities)+1), proto.getMovingObject().length()*ones(len(similarities)))) - - @staticmethod - def computePET(obj1, obj2, collisionDistanceThreshold): - '''Post-encroachment time based on distance threshold - - Returns the smallest time difference when the object positions are within collisionDistanceThreshold - and the instants at which each object is passing through its corresponding position''' - positions1 = [p.astuple() for p in obj1.getPositions()] - positions2 = [p.astuple() for p in obj2.getPositions()] - n1 = len(positions1) - n2 = len(positions2) - pets = zeros((n1, n2)) - for i,t1 in enumerate(obj1.getTimeInterval()): - for j,t2 in enumerate(obj2.getTimeInterval()): - pets[i,j] = abs(t1-t2) - distances = cdist(positions1, positions2, metric = 'euclidean') - smallDistances = (distances <= collisionDistanceThreshold) - if smallDistances.any(): - smallPets = pets[smallDistances] - petIdx = smallPets.argmin() - distanceIndices = argwhere(smallDistances)[petIdx] - return smallPets[petIdx], obj1.getFirstInstant()+distanceIndices[0], obj2.getFirstInstant()+distanceIndices[1] - else: - return None, None, None - - def predictPosition(self, instant, nTimeSteps, externalAcceleration = Point(0,0)): - '''Predicts the position of object at instant+deltaT, - at constant speed''' - return predictPositionNoLimit(nTimeSteps, self.getPositionAtInstant(instant), self.getVelocityAtInstant(instant), externalAcceleration) - - def projectCurvilinear(self, alignments, ln_mv_av_win=3): - ''' Add, for every object position, the class 'moving.CurvilinearTrajectory()' - (curvilinearPositions instance) which holds information about the - curvilinear coordinates using alignment metadata. - From Paul St-Aubin's PVA tools - ====== - - Input: - ====== - alignments = a list of alignments, where each alignment is a list of - points (class Point). - ln_mv_av_win = moving average window (in points) in which to smooth - lane changes. As per tools_math.cat_mvgavg(), this term - is a search *radius* around the center of the window. - - ''' - - self.curvilinearPositions = CurvilinearTrajectory() - - #For each point - for i in range(int(self.length())): - result = getSYfromXY(self.getPositionAt(i), alignments) - - # Error handling - if(result is None): - print('Warning: trajectory {} at point {} {} has alignment errors (spline snapping)\nCurvilinear trajectory could not be computed'.format(self.getNum(), i, self.getPositionAt(i))) - else: - [align, alignPoint, snappedPoint, subsegmentDistance, S, Y] = result - self.curvilinearPositions.addPositionSYL(S, Y, align) - - ## Go back through points and correct lane - #Run through objects looking for outlier point - smoothed_lanes = utils.cat_mvgavg(self.curvilinearPositions.getLanes(),ln_mv_av_win) - ## Recalculate projected point to new lane - lanes = self.curvilinearPositions.getLanes() - if(lanes != smoothed_lanes): - for i in range(len(lanes)): - if(lanes[i] != smoothed_lanes[i]): - result = getSYfromXY(self.getPositionAt(i),[alignments[smoothed_lanes[i]]]) - - # Error handling - if(result is None): - ## This can be triggered by tracking errors when the trajectory jumps around passed another alignment. - print(' Warning: trajectory {} at point {} {} has alignment errors during trajectory smoothing and will not be corrected.'.format(self.getNum(), i, self.getPositionAt(i))) - else: - [align, alignPoint, snappedPoint, subsegmentDistance, S, Y] = result - self.curvilinearPositions.setPosition(i, S, Y, align) - - def computeSmoothTrajectory(self, minCommonIntervalLength): - '''Computes the trajectory as the mean of all features - if a feature exists, its position is - - Warning work in progress - TODO? not use the first/last 1-.. positions''' - nFeatures = len(self.features) - if nFeatures == 0: - print('Empty object features\nCannot compute smooth trajectory') - else: - # compute the relative position vectors - relativePositions = {} # relativePositions[(i,j)] is the position of j relative to i - for i in range(nFeatures): - for j in range(i): - fi = self.features[i] - fj = self.features[j] - inter = fi.commonTimeInterval(fj) - if inter.length() >= minCommonIntervalLength: - xi = array(fi.getXCoordinates()[inter.first-fi.getFirstInstant():int(fi.length())-(fi.getLastInstant()-inter.last)]) - yi = array(fi.getYCoordinates()[inter.first-fi.getFirstInstant():int(fi.length())-(fi.getLastInstant()-inter.last)]) - xj = array(fj.getXCoordinates()[inter.first-fj.getFirstInstant():int(fj.length())-(fj.getLastInstant()-inter.last)]) - yj = array(fj.getYCoordinates()[inter.first-fj.getFirstInstant():int(fj.length())-(fj.getLastInstant()-inter.last)]) - relativePositions[(i,j)] = Point(median(xj-xi), median(yj-yi)) - relativePositions[(j,i)] = -relativePositions[(i,j)] - - ### - # User Type Classification - ### - def classifyUserTypeSpeedMotorized(self, threshold, aggregationFunc = median, nInstantsIgnoredAtEnds = 0): - '''Classifies slow and fast road users - slow: non-motorized -> pedestrians - fast: motorized -> cars - - aggregationFunc can be any function that can be applied to a vector of speeds, including percentile: - aggregationFunc = lambda x: percentile(x, percentileFactor) # where percentileFactor is 85 for 85th percentile''' - speeds = self.getSpeeds(nInstantsIgnoredAtEnds) - if aggregationFunc(speeds) >= threshold: - self.setUserType(userType2Num['car']) - else: - self.setUserType(userType2Num['pedestrian']) - - def classifyUserTypeSpeed(self, speedProbabilities, aggregationFunc = median, nInstantsIgnoredAtEnds = 0): - '''Classifies road user per road user type - speedProbabilities are functions return P(speed|class) - in a dictionary indexed by user type names - Returns probabilities for each class - - for simple threshold classification, simply pass non-overlapping indicator functions (membership) - e.g. def indic(x): - if abs(x-mu) < sigma: - return 1 - else: - return x''' - if not hasattr(self, 'aggregatedSpeed'): - self.aggregatedSpeed = aggregationFunc(self.getSpeeds(nInstantsIgnoredAtEnds)) - userTypeProbabilities = {} - for userTypename in speedProbabilities: - userTypeProbabilities[userType2Num[userTypename]] = speedProbabilities[userTypename](self.aggregatedSpeed) - self.setUserType(utils.argmaxDict(userTypeProbabilities)) - return userTypeProbabilities - - def initClassifyUserTypeHoGSVM(self, aggregationFunc, pedBikeCarSVM, bikeCarSVM = None, pedBikeSpeedTreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), nInstantsIgnoredAtEnds = 0, homography = None, intrinsicCameraMatrix = None, distortionCoefficients = None): - '''Initializes the data structures for classification - - TODO? compute speed for longest feature?''' - self.aggregatedSpeed = aggregationFunc(self.getSpeeds(nInstantsIgnoredAtEnds)) - if self.aggregatedSpeed < pedBikeSpeedTreshold or bikeCarSVM is None: - self.appearanceClassifier = pedBikeCarSVM - elif self.aggregatedSpeed < bikeCarSpeedThreshold: - self.appearanceClassifier = bikeCarSVM - else: - self.appearanceClassifier = carClassifier - # project feature positions - if self.hasFeatures(): - for f in self.getFeatures(): - pp = cvutils.worldToImageProject(f.getPositions().asArray(), intrinsicCameraMatrix, distortionCoefficients, homography).tolist() - f.positions = Trajectory(pp) - self.userTypes = {} - - def classifyUserTypeHoGSVMAtInstant(self, img, instant, width, height, px, py, minNPixels, rescaleSize, orientations, pixelsPerCell, cellsPerBlock, blockNorm): - '''Extracts the image box around the object - (of square size max(width, height) of the box around the features, - with an added px or py for width and height (around the box)) - computes HOG on this cropped image (with parameters rescaleSize, orientations, pixelsPerCell, cellsPerBlock) - and applies the SVM model on it''' - croppedImg = cvutils.imageBox(img, self, instant, width, height, px, py, minNPixels) - if croppedImg is not None and len(croppedImg) > 0: - hog = cvutils.HOG(croppedImg, rescaleSize, orientations, pixelsPerCell, cellsPerBlock, blockNorm) - self.userTypes[instant] = self.appearanceClassifier.predict(hog.reshape(1,hog.size)) - else: - self.userTypes[instant] = userType2Num['unknown'] - - def classifyUserTypeHoGSVM(self, pedBikeCarSVM = None, width = 0, height = 0, homography = None, images = None, bikeCarSVM = None, pedBikeSpeedTreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), minSpeedEquiprobable = -1, speedProbabilities = None, aggregationFunc = median, maxPercentUnknown = 0.5, nInstantsIgnoredAtEnds = 0, px = 0.2, py = 0.2, minNPixels = 800, rescaleSize = (64, 64), orientations = 9, pixelsPerCell = (8,8), cellsPerBlock = (2,2)): - '''Agregates SVM detections in each image and returns probability - (proportion of instants with classification in each category) - - images is a dictionary of images indexed by instant - With default parameters, the general (ped-bike-car) classifier will be used - - Considered categories are the keys of speedProbabilities''' - if not hasattr(self, 'aggregatedSpeed') or not hasattr(self, 'userTypes'): - print('Initializing the data structures for classification by HoG-SVM') - self.initClassifyUserTypeHoGSVM(aggregationFunc, pedBikeCarSVM, bikeCarSVM, pedBikeSpeedTreshold, bikeCarSpeedThreshold, nInstantsIgnoredAtEnds) - - if len(self.userTypes) != self.length() and images is not None: # if classification has not been done previously - for t in self.getTimeInterval(): - if t not in self.userTypes: - self.classifyUserTypeHoGSVMAtInstant(images[t], t, homography, width, height, px, py, minNPixels, rescaleSize, orientations, pixelsPerCell, cellsPerBlock) - # compute P(Speed|Class) - if speedProbabilities is None or self.aggregatedSpeed < minSpeedEquiprobable: # equiprobable information from speed - userTypeProbabilities = {userType2Num['car']: 1., userType2Num['pedestrian']: 1., userType2Num['bicycle']: 1.} - else: - userTypeProbabilities = {userType2Num[userTypename]: speedProbabilities[userTypename](self.aggregatedSpeed) for userTypename in speedProbabilities} - # compute P(Class|Appearance) - nInstantsUserType = {userTypeNum: 0 for userTypeNum in userTypeProbabilities}# number of instants the object is classified as userTypename - nInstantsUserType[userType2Num['unknown']] = 0 - for t in self.userTypes: - nInstantsUserType[self.userTypes[t]] += 1 #nInstantsUserType.get(self.userTypes[t], 0) + 1 - # result is P(Class|Appearance) x P(Speed|Class) - if nInstantsUserType[userType2Num['unknown']] < maxPercentUnknown*self.length(): # if not too many unknowns - for userTypeNum in userTypeProbabilities: - userTypeProbabilities[userTypeNum] *= nInstantsUserType[userTypeNum] - # class is the user type that maximizes usertype probabilities - if nInstantsUserType[userType2Num['unknown']] >= maxPercentUnknown*self.length() and (speedProbabilities is None or self.aggregatedSpeed < minSpeedEquiprobable): # if no speed information and too many unknowns - self.setUserType(userType2Num['unknown']) - else: - self.setUserType(utils.argmaxDict(userTypeProbabilities)) - - def classifyUserTypeArea(self, areas, homography): - '''Classifies the object based on its location (projected to image space) - areas is a dictionary of matrix of the size of the image space - for different road users possible locations, indexed by road user type names - - TODO: areas could be a wrapper object with a contains method that would work for polygons and images (with wrapper class) - skip frames at beginning/end?''' - print('not implemented/tested yet') - if not hasattr(self, projectedPositions): - if homography is not None: - self.projectedPositions = obj.positions.homographyProject(homography) - else: - self.projectedPositions = obj.positions - possibleUserTypes = {userType: 0 for userType in range(len(userTypenames))} - for p in self.projectedPositions: - for userTypename in areas: - if areas[userTypename][p.x, p.y] != 0: - possibleUserTypes[userType2Enum[userTypename]] += 1 - # what to do: threshold for most common type? self.setUserType() - return possibleUserTypes - - @staticmethod - def collisionCourseDotProduct(movingObject1, movingObject2, instant): - 'A positive result indicates that the road users are getting closer' - deltap = movingObject1.getPositionAtInstant(instant)-movingObject2.getPositionAtInstant(instant) - deltav = movingObject2.getVelocityAtInstant(instant)-movingObject1.getVelocityAtInstant(instant) - return Point.dot(deltap, deltav) - - @staticmethod - def collisionCourseCosine(movingObject1, movingObject2, instant): - 'A positive result indicates that the road users are getting closer' - return Point.cosine(movingObject1.getPositionAtInstant(instant)-movingObject2.getPositionAtInstant(instant), #deltap - movingObject2.getVelocityAtInstant(instant)-movingObject1.getVelocityAtInstant(instant)) #deltav - - -class Prototype(object): - 'Class for a prototype' - - def __init__(self, filename, num, trajectoryType, nMatchings = None): - self.filename = filename - self.num = num - self.trajectoryType = trajectoryType - self.nMatchings = nMatchings - self.movingObject = None - - def getFilename(self): - return self.filename - def getNum(self): - return self.num - def getTrajectoryType(self): - return self.trajectoryType - def getNMatchings(self): - return self.nMatchings - def getMovingObject(self): - return self.movingObject - def setMovingObject(self, o): - self.movingObject = o - - -################## -# Annotations -################## - -class BBMovingObject(MovingObject): - '''Class for a moving object represented as a bounding box - used for series of ground truth annotations using bounding boxes - and for the output of Urban Tracker http://www.jpjodoin.com/urbantracker/ - - By default in image space - - Its center is the center of the box (generalize to other shapes?) - (computed after projecting if homography available) - ''' - - def __init__(self, num = None, timeInterval = None, topLeftPositions = None, bottomRightPositions = None, userType = userType2Num['unknown']): - super(BBMovingObject, self).__init__(num, timeInterval, userType = userType) - self.topLeftPositions = topLeftPositions.getPositions() - self.bottomRightPositions = bottomRightPositions.getPositions() - - def computeCentroidTrajectory(self, homography = None): - self.positions = self.topLeftPositions.add(self.bottomRightPositions).__mul__(0.5) - if homography is not None: - self.positions = self.positions.homographyProject(homography) - - def matches(self, obj, instant, matchingDistance): - '''Indicates if the annotation matches obj (MovingObject) - with threshold matchingDistance - Returns distance if below matchingDistance, matchingDistance+1 otherwise - (returns an actual value, otherwise munkres does not terminate)''' - d = Point.distanceNorm2(self.getPositionAtInstant(instant), obj.getPositionAtInstant(instant)) - if d < matchingDistance: - return d - else: - return matchingDistance + 1 - -def computeClearMOT(annotations, objects, matchingDistance, firstInstant, lastInstant, returnMatches = False, debug = False): - '''Computes the CLEAR MOT metrics - - Reference: - Keni, Bernardin, and Stiefelhagen Rainer. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008 (2008) - - objects and annotations are supposed to in the same space - current implementation is BBMovingObject (bounding boxes) - mathingDistance is threshold on matching between annotation and object - - TO: tracker output (objects) - GT: ground truth (annotations) - - Output: returns motp, mota, mt, mme, fpt, gt - mt number of missed GT.frames (sum of the number of GT not detected in each frame) - mme number of mismatches - fpt number of false alarm.frames (tracker objects without match in each frame) - gt number of GT.frames - - if returnMatches is True, return as 2 new arguments the GT and TO matches - matches is a dict - matches[i] is the list of matches for GT/TO i - the list of matches is a dict, indexed by time, for the TO/GT id matched at time t - (an instant t not present in matches[i] at which GT/TO exists means a missed detection or false alarm) - - TODO: Should we use the distance as weights or just 1/0 if distance below matchingDistance? - (add argument useDistanceForWeights = False)''' - from munkres import Munkres - - munk = Munkres() - dist = 0. # total distance between GT and TO - ct = 0 # number of associations between GT and tracker output in each frame - gt = 0 # number of GT.frames - mt = 0 # number of missed GT.frames (sum of the number of GT not detected in each frame) - fpt = 0 # number of false alarm.frames (tracker objects without match in each frame) - mme = 0 # number of mismatches - matches = {} # match[i] is the tracker track associated with GT i (using object references) - if returnMatches: - gtMatches = {a.getNum():{} for a in annotations} - toMatches = {o.getNum():{} for o in objects} - else: - gtMatches = None - toMatches = None - for t in range(firstInstant, lastInstant+1): - previousMatches = matches.copy() - # go through currently matched GT-TO and check if they are still matched withing matchingDistance - toDelete = [] - for a in matches: - if a.existsAtInstant(t) and matches[a].existsAtInstant(t): - d = a.matches(matches[a], t, matchingDistance) - if d < matchingDistance: - dist += d - else: - toDelete.append(a) - else: - toDelete.append(a) - for a in toDelete: - del matches[a] - - # match all unmatched GT-TO - matchedGTs = list(matches.keys()) - matchedTOs = list(matches.values()) - costs = [] - unmatchedGTs = [a for a in annotations if a.existsAtInstant(t) and a not in matchedGTs] - unmatchedTOs = [o for o in objects if o.existsAtInstant(t) and o not in matchedTOs] - nGTs = len(matchedGTs)+len(unmatchedGTs) - nTOs = len(matchedTOs)+len(unmatchedTOs) - if len(unmatchedTOs) > 0: - for a in unmatchedGTs: - costs.append([a.matches(o, t, matchingDistance) for o in unmatchedTOs]) - if len(costs) > 0: - newMatches = munk.compute(costs) - for k,v in newMatches: - if costs[k][v] < matchingDistance: - matches[unmatchedGTs[k]]=unmatchedTOs[v] - dist += costs[k][v] - if debug: - print('{} '.format(t)+', '.join(['{} {}'.format(k.getNum(), v.getNum()) for k,v in matches.items()])) - if returnMatches: - for a,o in matches.items(): - gtMatches[a.getNum()][t] = o.getNum() - toMatches[o.getNum()][t] = a.getNum() - - # compute metrics elements - ct += len(matches) - mt += nGTs-len(matches) - fpt += nTOs-len(matches) - gt += nGTs - # compute mismatches - # for gt that do not appear in both frames, check if the corresponding to was matched to another gt in previous/next frame - mismatches = [] - for a in matches: - if a in previousMatches: - if matches[a] != previousMatches[a]: - mismatches.append(a) - elif matches[a] in list(previousMatches.values()): - mismatches.append(matches[a]) - for a in previousMatches: - if a not in matches and previousMatches[a] in list(matches.values()): - mismatches.append(previousMatches[a]) - if debug: - for mm in set(mismatches): - print('{} {}'.format(type(mm), mm.getNum())) - # some object mismatches may appear twice - mme += len(set(mismatches)) - - if ct > 0: - motp = dist/ct - else: - motp = None - if gt > 0: - mota = 1.-float(mt+fpt+mme)/gt - else: - mota = None - return motp, mota, mt, mme, fpt, gt, gtMatches, toMatches - -def plotRoadUsers(objects, colors): - '''Colors is a PlottingPropertyValues instance''' - from matplotlib.pyplot import figure, axis - figure() - for obj in objects: - obj.plot(colors.get(obj.userType)) - axis('equal') - - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/moving.txt') - #suite = doctest.DocTestSuite() - unittest.TextTestRunner().run(suite) - #doctest.testmod() - #doctest.testfile("example.txt") - if shapelyAvailable: - suite = doctest.DocFileSuite('tests/moving_shapely.txt') - unittest.TextTestRunner().run(suite)
--- a/python/objectsmoothing.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,246 +0,0 @@ -import storage, moving, utils - -from math import atan2, degrees, sin, cos, pi -from numpy import median - -import matplotlib.pyplot as plt - -def findNearest(feat, featureSet,t,reverse=True): - dist={} - for f in featureSet: - if reverse: - dist[f]= moving.Point.distanceNorm2(feat.getPositionAtInstant(t+1),f.getPositionAtInstant(t)) - else: - dist[f]= moving.Point.distanceNorm2(feat.getPositionAtInstant(t-1),f.getPositionAtInstant(t)) - return min(dist, key=dist.get) # = utils.argmaxDict(dist) - -def getFeatures(obj, featureID): - currentFeature = obj.getFeature(featureID) - first = currentFeature.getFirstInstant() - last = currentFeature.getLastInstant() - featureList=[[currentFeature,first,last,moving.Point(0,0)]] - # find the features to fill in the beginning of the object existence - while first != obj.getFirstInstant(): - delta=featureList[-1][3] - featureSet = [f for f in obj.getFeatures() if f.existsAtInstant(first-1)] - feat = findNearest(currentFeature,featureSet,first-1,reverse=True) - if feat.existsAtInstant(first): - featureList.append([feat,feat.getFirstInstant(),first-1,(currentFeature.getPositionAtInstant(first)-feat.getPositionAtInstant(first))+delta]) - else: - featureList.append([feat,feat.getFirstInstant(),first-1,(currentFeature.getPositionAtInstant(first)-feat.getPositionAtInstant(first-1))+delta]) - currentFeature = feat - first= feat.getFirstInstant() - # find the features to fill in the end of the object existence - delta=moving.Point(0,0) - currentFeature = obj.getFeature(featureID) # need to reinitialize - while last!= obj.getLastInstant(): - featureSet = [f for f in obj.getFeatures() if f.existsAtInstant(last+1)] - feat = findNearest(currentFeature,featureSet,last+1,reverse=False) - if feat.existsAtInstant(last): - featureList.append([feat,last+1,feat.getLastInstant(),(currentFeature.getPositionAtInstant(last)-feat.getPositionAtInstant(last))+delta]) - else: - featureList.append([feat,last+1,feat.getLastInstant(),(currentFeature.getPositionAtInstant(last)-feat.getPositionAtInstant(last+1))+delta]) - currentFeature = feat - last= feat.getLastInstant() - delta=featureList[-1][3] - return featureList - -def buildFeature(obj, featureID, num = 1): - featureList= getFeatures(obj, featureID) - tmp={} - delta={} - for i in featureList: - for t in range(i[1],i[2]+1): - tmp[t]=[i[0],i[3]] - newTraj = moving.Trajectory() - - for instant in obj.getTimeInterval(): - newTraj.addPosition(tmp[instant][0].getPositionAtInstant(instant)+tmp[instant][1]) - newFeature= moving.MovingObject(num,timeInterval=obj.getTimeInterval(),positions=newTraj) - return newFeature - -def getBearing(p1,p2,p3): - angle = degrees(atan2(p3.y -p1.y, p3.x -p1.x)) - bearing1 = (90 - angle) % 360 - angle2 = degrees(atan2(p2.y -p1.y, p2.x -p1.x)) - bearing2 = (90 - angle2) % 360 - dist= moving.Point.distanceNorm2(p1, p2) - return [dist,bearing1,bearing2,bearing2-bearing1] - -#Quantitative analysis "CSJ" functions -def computeVelocities(obj, smoothing=True, halfWidth=3): #compute velocities from positions - velocities={} - for i in list(obj.timeInterval)[:-1]: - p1= obj.getPositionAtInstant(i) - p2= obj.getPositionAtInstant(i+1) - velocities[i]=p2-p1 - velocities[obj.getLastInstant()]= velocities[obj.getLastInstant()-1] # duplicate last point - if smoothing: - velX= [velocities[y].aslist()[0] for y in sorted(velocities.keys())] - velY= [velocities[y].aslist()[1] for y in sorted(velocities.keys())] - v1= list(utils.filterMovingWindow(velX, halfWidth)) - v2= list(utils.filterMovingWindow(velY, halfWidth)) - smoothedVelocity={} - for t,i in enumerate(sorted(velocities.keys())): - smoothedVelocity[i]=moving.Point(v1[t], v2[t]) - velocities=smoothedVelocity - return velocities - -def computeAcceleration(obj,fromPosition=True): - acceleration={} - if fromPosition: - velocities=computeVelocities(obj,False,1) - for i in sorted(velocities.keys()): - if i != sorted(velocities.keys())[-1]: - acceleration[i]= velocities[i+1]-velocities[i] - else: - for i in list(obj.timeInterval)[:-1]: - v1= obj.getVelocityAtInstant(i) - v2= obj.getVelocityAtInstant(i+1) - acceleration[i]= v2-v1 - return acceleration - -def computeJerk(obj,fromPosition=True): - jerk={} - acceleration=computeAcceleration(obj,fromPosition=fromPosition) - for i in sorted(acceleration.keys()): - if i != sorted(acceleration.keys())[-1]: - jerk[i] = (acceleration[i+1]-acceleration[i]).norm2() - return jerk - -def sumSquaredJerk(obj,fromPosition=True): - jerk= computeJerk(obj,fromPosition=fromPosition) - t=0 - for i in sorted(jerk.keys()): - t+= jerk[i]* jerk[i] - return t - -def smoothObjectTrajectory(obj, featureID,newNum,smoothing=False,halfWidth=3,create=False): - results=[] - bearing={} - if create: - feature = buildFeature(obj, featureID , num=1) # why num=1 - else: - feature = obj.getFeature(featureID) - for t in feature.getTimeInterval(): - p1= feature.getPositionAtInstant(t) - p2= obj.getPositionAtInstant(t) - if t!=feature.getLastInstant(): - p3= feature.getPositionAtInstant(t+1) - else: - p1= feature.getPositionAtInstant(t-1) - p3= feature.getPositionAtInstant(t) - bearing[t]= getBearing(p1,p2,p3)[1] - results.append(getBearing(p1,p2,p3)) - - medianResults=median(results,0) - dist= medianResults[0] - angle= medianResults[3] - - for i in sorted(bearing.keys()): - bearing[i]= bearing[i]+angle - - if smoothing: - bearingInput=[] - for i in sorted(bearing.keys()): - bearingInput.append(bearing[i]) - import utils - bearingOut=utils.filterMovingWindow(bearingInput, halfWidth) - for t,i in enumerate(sorted(bearing.keys())): - bearing[i]=bearingOut[t] - - #solve a smoothing problem in case of big drop in computing bearing (0,360) - for t,i in enumerate(sorted(bearing.keys())): - if i!= max(bearing.keys()) and abs(bearingInput[t] - bearingInput[t+1])>=340: - for x in range(max(i-halfWidth,min(bearing.keys())),min(i+halfWidth,max(bearing.keys()))+1): - bearing[x]=bearingInput[t-i+x] - - translated = moving.Trajectory() - for t in feature.getTimeInterval(): - p1= feature.getPositionAtInstant(t) - p1.x = p1.x + dist*sin(bearing[t]*pi/180) - p1.y = p1.y + dist*cos(bearing[t]*pi/180) - translated.addPosition(p1) - - #modify first and last un-smoothed positions (half width) - if smoothing: - d1= translated[halfWidth]- feature.positions[halfWidth] - d2= translated[-halfWidth-1]- feature.positions[-halfWidth-1] - for i in range(halfWidth): - p1= feature.getPositionAt(i)+d1 - p2= feature.getPositionAt(-i-1)+d2 - translated.setPosition(i,p1) - translated.setPosition(-i-1,p2) - - newObj= moving.MovingObject(newNum,timeInterval=feature.getTimeInterval(),positions=translated) - return newObj - -def smoothObject(obj, newNum, minLengthParam = 0.7, smoothing = False, plotResults = True, halfWidth = 3, _computeVelocities = True, optimize = True, create = False): - '''Computes a smoother trajectory for the object - and optionnally smoother velocities - - The object should have its features in obj.features - TODO: check whether features are necessary''' - if not obj.hasFeatures(): - print('Object {} has an empty list of features: please load and add them using obj.setFeatures(features)'.format(obj.getNum())) - from sys import exit - exit() - - featureList=[i for i,f in enumerate(obj.getFeatures()) if f.length() >= minLengthParam*obj.length()] - if featureList==[]: - featureList.append(utils.argmaxDict({i:f.length() for i,f in enumerate(obj.getFeatures())})) - create = True - newObjects = [] - for featureID in featureList: # featureID should be the index in the list of obj.features - newObjects.append(smoothObjectTrajectory(obj, featureID, newNum, smoothing = smoothing, halfWidth = halfWidth, create = create)) - - newTranslated = moving.Trajectory() - newInterval = [] - for t in obj.getTimeInterval(): - xCoord=[] - yCoord=[] - for i in newObjects: - if i.existsAtInstant(t): - p1= i.getPositionAtInstant(t) - xCoord.append(p1.x) - yCoord.append(p1.y) - if xCoord != []: - tmp= moving.Point(median(xCoord), median(yCoord)) - newInterval.append(t) - newTranslated.addPosition(tmp) - - newObj= moving.MovingObject(newNum, timeInterval = moving.TimeInterval(min(newInterval),max(newInterval)),positions=newTranslated) - - if _computeVelocities: - tmpTraj = moving.Trajectory() - velocities= computeVelocities(newObj,True,5) - for i in sorted(velocities.keys()): - tmpTraj.addPosition(velocities[i]) - newObj.velocities=tmpTraj - else: - newObj.velocities=obj.velocities - - if optimize: - csj1= sumSquaredJerk(obj,fromPosition=True) - csj2= sumSquaredJerk(newObj,fromPosition=True) - if csj1<csj2: - newObj=obj - newObj.velocities=obj.velocities - if _computeVelocities and csj1>=csj2: - csj3= sumSquaredJerk(obj,fromPosition=False) - csj4= sumSquaredJerk(newObj,fromPosition=False) - if csj4<=csj3: - newObj.velocities= obj.velocities - - newObj.featureNumbers=obj.featureNumbers - newObj.features=obj.getFeatures() - newObj.userType=obj.userType - - if plotResults: - plt.figure() - plt.title('objects_id = {}'.format(obj.num)) - for i in featureList: - obj.getFeature(i).plot('cx-') - obj.plot('rx-') - newObj.plot('gx-') - return newObj
--- a/python/pavement.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,313 +0,0 @@ -#! /usr/bin/env python -'''Tools for processing and analyzing pavement marking data''' - -import utils - -import numpy as np - - -paintTypes = {0: "Non-existant", - 1: "Eau", - 2: "Epoxy", - 3: "Alkyde", - 4: "Autre"} - -durabilities = {1: 98, #96 to 100 - 2: 85, #75 to 96 - 3: 62, #50 to 75 - 4: 32, #15 to 50 - 5: 7 #0 to 15 - } - -roadFunctionalClasses = {40: "Collectrice", - 20: "Nationale", - 30: "Regionale", - 10: "Autoroute", - 60: "Acces ressources", - 51: "Local 1", - 52: "Local 2", - 53: "Local 3", - 15: "Aut (PRN)", - 25: "Nat (PRN)", - 70: "Acces isolees", - 99: "Autres"} - -def caracteristiques(rtss, maintenanceLevel, rtssWeatherStation, fmr, paintType): - '''Computes characteristic data for the RTSS (class rtss) - maintenanceLevel = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\BDD_access\\analyse_donnees_deneigement\\exigence_circuits.txt', delimiter = ';') - rtssWeatherStation = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\stations_environnement_canada\\rtssWeatherStation\juste_pour_rtss_avec_donnees_entretien_hiv\\rtssWeatherStation_EC3.txt', delimiter = ',') - fmr = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\BDD_access\\analyse_donnees_deneigement\\fmr.txt', delimiter = ';') - paintType = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\BDD_access\\analyse_donnees_deneigement\\type_peinture.txt', delimiter = ';') - ''' - # determination exigence deneigement - if rtss.id in maintenanceLevel['rtss_debut']: - for i in range(len(maintenanceLevel)): - if maintenanceLevel['rtss_debut'][i] == rtss.id: - exigence = maintenanceLevel['exigence'][i] - else: - exigence = '' - - # determination x/y - if rtss.id in rtssWeatherStation['rtss']: - for i in range(len(rtssWeatherStation)): - if rtssWeatherStation['rtss'][i] == rtss.id: - x_moy = rtssWeatherStation['x_moy'][i] - y_moy = rtssWeatherStation['y_moy'][i] - else: - x_moy, y_moy = '','' - - # determination info fmr - age_revtm, classe_fonct, type_revtm, milieu, djma, pourc_camions, vit_max = [], [], [], [], [], [], [] - if rtss.id in fmr['rtss_debut']: - for i in range(len(fmr)): - if fmr['rtss_debut'][i] == rtss.id: - age_revtm.append(fmr['age_revtm'][i]) - classe_fonct.append(fmr['des_clasf_fonct'][i]) - type_revtm.append(fmr['des_type_revtm'][i]) - milieu.append(fmr['des_cod_mil'][i]) - djma.append(fmr['val_djma'][i]) - pourc_camions.append(fmr['val_pourc_camns'][i]) - vit_max.append(fmr['val_limt_vitss'][i]) - age_revtm = utils.mostCommon(age_revtm) - classe_fonct = utils.mostCommon(classe_fonct) - type_revtm = utils.mostCommon(type_revtm) - milieu = utils.mostCommon(milieu) - djma = utils.mostCommon(djma) - vit_max = utils.mostCommon(vit_max) - if vit_max < 0: - vit_max = '' - pourc_camions = utils.mostCommon(pourc_camions) - if pourc_camions == "" or pourc_camions < 0: - djma_camions = "" - else: - djma_camions = pourc_camions*djma/100 - else: - age_revtm, classe_fonct, type_revtm, milieu, djma, djma_camions, vit_max = '','','','','','','' - - # determination type peinture - peinture_rd, peinture_rg, peinture_cl = [], [], [] - peinture_lrd, peinture_lrg, peinture_lc = 0,0,0 - if rtss.id in paintType['rtss_debut_orig']: - for i in range(len(paintType)): - if paintType['rtss_debut_orig'][i] == rtss.id: - peinture_rd.append((paintType['peinture_rd'][i])) - peinture_rg.append((paintType['peinture_rg'][i])) - peinture_cl.append((paintType['peinture_cl'][i])) - peinture_lrd = utils.mostCommon(peinture_rd) - peinture_lrg = utils.mostCommon(peinture_rg) - peinture_lc = utils.mostCommon(peinture_cl) - else: - peinture_lrd, peinture_lrg, peinture_lc = '','','' - - return (exigence, x_moy, y_moy, age_revtm, classe_fonct, type_revtm, milieu, djma, djma_camions, vit_max, peinture_lrd, peinture_lrg, peinture_lc) - -def winterMaintenanceIndicators(data, startDate, endDate, circuitReference, snowThreshold): - '''Computes several winter maintenance indicators - data = entretien_hivernal = pylab.csv2rec('C:\\Users\Alexandre\Documents\Cours\Poly\Projet\mesures_entretien_hivernal\mesures_deneigement.txt', delimiter = ',')''' - import datetime - somme_eau, somme_neige, somme_abrasif, somme_sel, somme_lc, somme_lrg, somme_lrd, compteur_premiere_neige, compteur_somme_abrasif = 0,0,0,0,0,0,0,0,0 - - if circuitReference in data['ref_circuit']: - for i in range(len(data)): - if data['ref_circuit'][i] == circuitReference and (data['date'][i] + datetime.timedelta(days = 6)) <= endDate and (data['date'][i] + datetime.timedelta(days = 6)) > startDate: - compteur_premiere_neige += float(data['premiere_neige'][i]) - somme_neige += float(data['neige'][i]) - somme_eau += float(data['eau'][i]) - somme_abrasif += float(data['abrasif'][i]) - somme_sel += float(data['sel'][i]) - somme_lc += float(data['lc'][i]) - somme_lrg += float(data['lrg'][i]) - somme_lrd += float(data['lrd'][i]) - compteur_somme_abrasif += float(data['autre_abrasif_binaire'][i]) - if compteur_premiere_neige >= 1: - premiere_neige = 1 - else: - premiere_neige = 0 - if compteur_somme_abrasif >= 1: - autres_abrasifs = 1 - else: - autres_abrasifs = 0 - if somme_neige < snowThreshold: - neigeMTQ_sup_seuil = 0 - else: - neigeMTQ_sup_seuil = 1 - else: - somme_eau, somme_neige, somme_abrasif, somme_sel, somme_lc, somme_lrg, somme_lrd, premiere_neige, autres_abrasifs, neigeMTQ_sup_seuil = '','','','','','','','','','' - - return (somme_eau, somme_neige, neigeMTQ_sup_seuil, somme_abrasif, somme_sel, somme_lc, somme_lrg, somme_lrd, premiere_neige, autres_abrasifs) - -def weatherIndicators(data, startDate, endDate, snowThreshold, weatherDatatype, minProportionMeasures = 0.): - '''Computes the indicators from Environment Canada files - (loaded as a recarray using csv2rec in data), - between start and end dates (datetime.datetime objects) - - weatherDataType is to indicate Environnement Canada data ('ec') or else MTQ - minProportionMeasures is proportion of measures necessary to consider the indicators''' - from matplotlib.mlab import find - nbre_jours_T_negatif,nbre_jours_gel_degel,pluie_tot,neige_tot,ecart_type_T = 0,0,0,0,0 - compteur,nbre_jours_gel_consecutifs=0,0 - tmoys = [] - seuils_T = [20,15,10,5] - deltas_T = [0,0,0,0] - startIndex = find(data['date'] == startDate) - nDays = int((endDate - startDate).days)+1 - if len(startIndex) > 0 and startIndex+nDays <= len(data): - startIndex = startIndex[0] - for i in range(startIndex, startIndex+nDays): - if not np.isnan(data['tmax'][i]): - tmax = data['tmax'][i] - else: - tmax = None - if not np.isnan(data['tmin'][i]): - tmin = data['tmin'][i] - else: - tmin = None - if weatherDatatype == 'ec': - if data['pluie_tot'][i] is not None and not np.isnan(data['pluie_tot'][i]): - pluie_tot += data['pluie_tot'][i] - if data['neige_tot'][i] is not None and not np.isnan(data['neige_tot'][i]): - neige_tot += data['neige_tot'][i] - if tmax is not None: - if tmax < 0: - nbre_jours_T_negatif += 1 - if tmax is not None and tmin is not None: - if tmax > 0 and tmin < 0: - nbre_jours_gel_degel += 1 - for l in range(len(seuils_T)): - if tmax - tmin >=seuils_T[l]: - deltas_T[l] += 1 - if not np.isnan(data['tmoy'][i]): - tmoys.append(data['tmoy'][i]) - if tmax is not None: - if tmax < 0: - compteur += 1 - elif tmax >= 0 and compteur >= nbre_jours_gel_consecutifs: - nbre_jours_gel_consecutifs = compteur - compteur = 0 - else: - compteur = 0 - nbre_jours_gel_consecutifs = max(nbre_jours_gel_consecutifs,compteur) - if len(tmoys) > 0 and float(len(tmoys))/nDays >= minProportionMeasures: - if tmoys != []: - ecart_type_T = np.std(tmoys) - else: - ecart_type = None - if neige_tot < snowThreshold: - neigeEC_sup_seuil = 0 - else: - neigeEC_sup_seuil = 1 - return (nbre_jours_T_negatif,nbre_jours_gel_degel, deltas_T, nbre_jours_gel_consecutifs, pluie_tot, neige_tot, neigeEC_sup_seuil, ecart_type_T) - else: - return [None]*2+[[None]*len(seuils_T)]+[None]*5 - -def mtqWeatherIndicators(data, startDate, endDate,tmax,tmin,tmoy): - print("Deprecated, use weatherIndicators") - from matplotlib.mlab import find - nbre_jours_T_negatif,nbre_jours_gel_degel,ecart_type_T = 0,0,0 - compteur,nbre_jours_gel_consecutifs=0,0 - tmoys = [] - seuils_T = [20,15,10,5] - deltas_T = [0,0,0,0] - startIndex = find(data['date'] == startDate) - nDays = (endDate - startDate).days+1 - for i in range(startIndex, startIndex+nDays): - if tmax[i] < 0: - nbre_jours_T_negatif += 1 - if tmax[i] > 0 and tmin[i] < 0: - nbre_jours_gel_degel += 1 - for l in range(len(seuils_T)): - if tmax[i] - tmin[i] >=seuils_T[l]: - deltas_T[l] += 1 - tmoys.append(tmoy[i]) - if tmax[i] < 0: - compteur += 1 - elif tmax[i] >= 0 and compteur >= nbre_jours_gel_consecutifs: - nbre_jours_gel_consecutifs = compteur - compteur = 0 - else: - compteur = 0 - nbre_jours_gel_consecutifs = max(nbre_jours_gel_consecutifs,compteur) - if tmoys != []: - ecart_type_T = np.std(tmoys) - else: - ecart_type = None - - return (nbre_jours_T_negatif,nbre_jours_gel_degel, deltas_T, nbre_jours_gel_consecutifs, ecart_type_T) - -class RTSS(object): - '''class for data related to a RTSS: - - agregating pavement marking measurements - - RTSS characteristics from FMR: pavement type, age, AADT, truck AADT - - winter maintenance level from V155 - - If divided highway, the RTSS ends with G or D and are distinct: there is no ambiguity - - retroreflectivity types: there are CB, RJ and RB - If undivided, ending with C - - durability is fine: ETAT_MARQG_RG ETAT_MARQG_CL ETAT_MARQG_RD (+SG/SD, but recent) - - retroreflectivity: CJ is center line, RB and SB are left/right if DEBUT-FIN>0 or <0 - ''' - - def __init__(self, _id, name, data): - self.id = _id - self.name = name - self.data = data - -class MarkingTest(object): - '''class for a test site for a given product - - including the series of measurements over the years''' - - def __init__(self, _id, paintingDate, paintingType, color, data): - self.id = _id - self.paintingDate = paintingDate - self.paintingType = paintingType - self.color = color - self.data = data - self.nMeasures = len(data) - - def getSite(self): - return int(self.id[:2]) - - def getTestAttributes(self): - return [self.paintingType, self.color, self.paintingDate.year] - - def plot(self, measure, options = 'o', dayRatio = 1., **kwargs): - from matplotlib.pyplot import plot - plot(self.data['jours']/float(dayRatio), - self.data[measure], options, **kwargs) - - def getMarkingMeasures(self, dataLabel): - nonZeroIndices = ~np.isnan(self.data[dataLabel]) - return self.data[nonZeroIndices]['jours'], self.data[nonZeroIndices][dataLabel] - - def plotMarkingMeasures(self, measure, options = 'o', dayRatio = 1., **kwargs): - for i in range(1,7): - self.plot('{}_{}'.format(measure, i), options, dayRatio, **kwargs) - - def computeMarkingMeasureVariations(self, dataLabel, lanePositions, weatherData, snowThreshold, weatherDataType = 'ec', minProportionMeasures = 0.): - '''Computes for each successive measurement - lanePositions = None - measure variation, initial measure, time duration, weather indicators - - TODO if measurements per lane, add a variable for lane position (position1 to 6) - lanePositions = list of integers (range(1,7)) - measure variation, initial measure, time duration, lane position1, weather indicators - measure variation, initial measure, time duration, lane position2, weather indicators - ...''' - variationData = [] - if lanePositions is None: - nonZeroIndices = ~np.isnan(self.data[dataLabel]) - days = self.data[nonZeroIndices]['jours'] - dates = self.data[nonZeroIndices]['date_mesure'] - measures = self.data[nonZeroIndices][dataLabel] - for i in range(1, len(dates)): - nDaysTNegative, nDaysThawFreeze, deltaTemp, nConsecutiveFrozenDays, totalRain, totalSnow, snowAboveThreshold, stdevTemp = weatherIndicators(weatherData, dates[i-1], dates[i], snowThreshold, weatherDataType, minProportionMeasures) - if dates[i-1].year+1 == dates[i].year: - winter = 1 - if days[i-1]<365: - firstWinter = 1 - else: - winter = 0 - firstWinter = 0 - variationData.append([measures[i-1]-measures[i], measures[i-1], days[i]-days[i-1], days[i-1], winter, firstWinter, nDaysTNegative, nDaysThawFreeze] + deltaTemp + [nConsecutiveFrozenDays, totalRain, totalSnow, snowAboveThreshold, stdevTemp]) - return variationData
--- a/python/poly-utils.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,125 +0,0 @@ -#! /usr/bin/env python -'''Various utilities to load data saved by the POLY new output(s)''' - -from moving import TimeInterval -from indicators import SeverityIndicator - -import sys, utils -import numpy as np - - -def loadNewInteractions(videoFilename,interactionType,dirname, extension, indicatorsNames, roaduserNum1,roaduserNum2, selectedIndicators=[]): - '''Loads interactions from the POLY traffic event format''' - from events import Interaction - filename= dirname + videoFilename + extension - #filename= dirname + interactionType+ '-' + videoFilename + extension # case of min distance todo: change the saving format to be matched with all outputs - file = utils.openCheck(filename) - if (not file): - return [] - #interactions = [] - interactionNum = 0 - data= np.loadtxt(filename) - indicatorFrameNums= data[:,0] - inter = Interaction(interactionNum, TimeInterval(indicatorFrameNums[0],indicatorFrameNums[-1]), roaduserNum1, roaduserNum2) - inter.addVideoFilename(videoFilename) - inter.addInteractionType(interactionType) - for key in indicatorsNames: - values= {} - for i,t in enumerate(indicatorFrameNums): - values[t] = data[i,key] - inter.addIndicator(SeverityIndicator(indicatorsNames[key], values)) - if selectedIndicators !=[]: - values= {} - for i,t in enumerate(indicatorFrameNums): - values[t] = [data[i,index] for index in selectedIndicators] - inter.addIndicator(SeverityIndicator('selectedIndicators', values)) - - #interactions.append(inter) - file.close() - #return interactions - return inter - -# Plotting results - -frameRate = 15. - -# To run in directory that contains the directories that contain the results (Miss-xx and Incident-xx) -#dirname = '/home/nicolas/Research/Data/kentucky-db/' - -interactingRoadUsers = {'Miss/0404052336': [(0,3)] # 0,2 and 1 vs 3 - #, - #'Incident/0306022035': [(1,3)] - #, - #'Miss/0208030956': [(4,5),(5,7)] - } - - -def getIndicatorName(filename, withUnit = False): - if withUnit: - unit = ' (s)' - else: - unit = '' - if 'collision-point' in filename: - return 'TTC'+unit - elif 'crossing' in filename: - return 'pPET'+unit - elif 'probability' in filename: - return 'P(UEA)' - -def getMethodName(fileprefix): - if fileprefix == 'constant-velocity': - return 'Con. Vel.' - elif fileprefix == 'normal-adaptation': - return 'Norm. Ad.' - elif fileprefix == 'point-set': - return 'Pos. Set' - elif fileprefix == 'evasive-action': - return 'Ev. Act.' - elif fileprefix == 'point-set-evasive-action': - return 'Pos. Set' - -indicator2TimeIdx = {'TTC':2,'pPET':2, 'P(UEA)':3} - -def getDataAtInstant(data, i): - return data[data[:,2] == i] - -def getPointsAtInstant(data, i): - return getDataAtInstant(i)[3:5] - -def getIndicator(data, roadUserNumbers, indicatorName): - if data.ndim ==1: - data.shape = (1,data.shape[0]) - - # find the order for the roadUserNumbers - uniqueObj1 = np.unique(data[:,0]) - uniqueObj2 = np.unique(data[:,1]) - found = False - if roadUserNumbers[0] in uniqueObj1 and roadUserNumbers[1] in uniqueObj2: - objNum1 = roadUserNumbers[0] - objNum2 = roadUserNumbers[1] - found = True - if roadUserNumbers[1] in uniqueObj1 and roadUserNumbers[0] in uniqueObj2: - objNum1 = roadUserNumbers[1] - objNum2 = roadUserNumbers[0] - found = True - - # get subset of data for road user numbers - if found: - roadUserData = data[np.logical_and(data[:,0] == objNum1, data[:,1] == objNum2),:] - if roadUserData.size > 0: - time = np.unique(roadUserData[:,indicator2TimeIdx[indicatorName]]) - values = {} - if indicatorName == 'P(UEA)': - tmp = roadUserData[:,4] - for k,v in zip(time, tmp): - values[k]=v - return SeverityIndicator(indicatorName, values, mostSevereIsMax = False, maxValue = 1.), roadUserData - else: - for i in range(time[0],time[-1]+1): - try: - tmp = getDataAtInstant(roadUserData, i) - values[i] = np.sum(tmp[:,5]*tmp[:,6])/np.sum(tmp[:,5])/frameRate - except IOError: - values[i] = np.inf - return SeverityIndicator(indicatorName, values, mostSevereIsMax = False), roadUserData - return None, None
--- a/python/prediction.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,598 +0,0 @@ -#! /usr/bin/env python -'''Library for motion prediction methods''' - -import moving -from utils import LCSS - -import math, random -from copy import copy -import numpy as np -#from multiprocessing import Pool - - -class PredictedTrajectory(object): - '''Class for predicted trajectories with lazy evaluation - if the predicted position has not been already computed, compute it - - it should also have a probability''' - - def __init__(self): - self.probability = 0. - self.predictedPositions = {} - self.predictedSpeedOrientations = {} - #self.collisionPoints = {} - #self.crossingZones = {} - - def predictPosition(self, nTimeSteps): - if nTimeSteps > 0 and not nTimeSteps in self.predictedPositions: - self.predictPosition(nTimeSteps-1) - self.predictedPositions[nTimeSteps], self.predictedSpeedOrientations[nTimeSteps] = moving.predictPosition(self.predictedPositions[nTimeSteps-1], self.predictedSpeedOrientations[nTimeSteps-1], self.getControl(), self.maxSpeed) - return self.predictedPositions[nTimeSteps] - - def getPredictedTrajectory(self): - return moving.Trajectory.fromPointList(list(self.predictedPositions.values())) - - def getPredictedSpeeds(self): - return [so.norm for so in self.predictedSpeedOrientations.values()] - - def plot(self, options = '', withOrigin = False, timeStep = 1, **kwargs): - self.getPredictedTrajectory().plot(options, withOrigin, timeStep, **kwargs) - -class PredictedTrajectoryConstant(PredictedTrajectory): - '''Predicted trajectory at constant speed or acceleration - TODO generalize by passing a series of velocities/accelerations''' - - def __init__(self, initialPosition, initialVelocity, control = moving.NormAngle(0,0), probability = 1., maxSpeed = None): - self.control = control - self.maxSpeed = maxSpeed - self.probability = probability - self.predictedPositions = {0: initialPosition} - self.predictedSpeedOrientations = {0: moving.NormAngle.fromPoint(initialVelocity)} - - def getControl(self): - return self.control - -class PredictedTrajectoryPrototype(PredictedTrajectory): - '''Predicted trajectory that follows a prototype trajectory - The prototype is in the format of a moving.Trajectory: it could be - 1. an observed trajectory (extracted from video) - 2. a generic polyline (eg the road centerline) that a vehicle is supposed to follow - - Prediction can be done - 1. at constant speed (the instantaneous user speed) - 2. following the trajectory path, at the speed of the user - (applying a constant ratio equal - to the ratio of the user instantaneous speed and the trajectory closest speed)''' - - def __init__(self, initialPosition, initialVelocity, prototype, constantSpeed = False, nFramesIgnore = 3, probability = 1.): - ''' prototype is a MovingObject - - Prediction at constant speed will not work for unrealistic trajectories - that do not follow a slowly changing velocity (eg moving object trajectories, - but is good for realistic motion (eg features)''' - self.prototype = prototype - self.constantSpeed = constantSpeed - self.nFramesIgnore = nFramesIgnore - self.probability = probability - self.predictedPositions = {0: initialPosition} - self.closestPointIdx = prototype.getPositions().getClosestPoint(initialPosition) - self.deltaPosition = initialPosition-prototype.getPositionAt(self.closestPointIdx) #should be computed in relative coordinates to position - self.theta = prototype.getVelocityAt(self.closestPointIdx).angle() - self.initialSpeed = initialVelocity.norm2() - if not constantSpeed: - self.ratio = self.initialSpeed/prototype.getVelocityAt(self.closestPointIdx).norm2() - - def predictPosition(self, nTimeSteps): - if nTimeSteps > 0 and not nTimeSteps in self.predictedPositions: - deltaPosition = copy(self.deltaPosition) - if self.constantSpeed: - traj = self.prototype.getPositions() - trajLength = traj.length() - traveledDistance = nTimeSteps*self.initialSpeed + traj.getCumulativeDistance(self.closestPointIdx) - i = self.closestPointIdx - while i < trajLength and traj.getCumulativeDistance(i) < traveledDistance: - i += 1 - if i == trajLength: - v = self.prototype.getVelocityAt(-1-self.nFramesIgnore) - self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[i-1]+v*((traveledDistance-traj.getCumulativeDistance(i-1))/v.norm2()) - else: - v = self.prototype.getVelocityAt(min(i-1, int(self.prototype.length())-1-self.nFramesIgnore)) - self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[i-1]+(traj[i]-traj[i-1])*((traveledDistance-traj.getCumulativeDistance(i-1))/traj.getDistance(i-1)) - else: - traj = self.prototype.getPositions() - trajLength = traj.length() - nSteps = self.ratio*nTimeSteps+self.closestPointIdx - i = int(np.floor(nSteps)) - if nSteps < trajLength-1: - v = self.prototype.getVelocityAt(min(i, int(self.prototype.length())-1-self.nFramesIgnore)) - self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[i]+(traj[i+1]-traj[i])*(nSteps-i) - else: - v = self.prototype.getVelocityAt(-1-self.nFramesIgnore) - self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[-1]+v*(nSteps-trajLength+1) - return self.predictedPositions[nTimeSteps] - -class PredictedTrajectoryRandomControl(PredictedTrajectory): - '''Random vehicle control: suitable for normal adaptation''' - def __init__(self, initialPosition, initialVelocity, accelerationDistribution, steeringDistribution, probability = 1., maxSpeed = None): - '''Constructor - accelerationDistribution and steeringDistribution are distributions - that return random numbers drawn from them''' - self.accelerationDistribution = accelerationDistribution - self.steeringDistribution = steeringDistribution - self.maxSpeed = maxSpeed - self.probability = probability - self.predictedPositions = {0: initialPosition} - self.predictedSpeedOrientations = {0: moving.NormAngle.fromPoint(initialVelocity)} - - def getControl(self): - return moving.NormAngle(self.accelerationDistribution(),self.steeringDistribution()) - -class SafetyPoint(moving.Point): - '''Can represent a collision point or crossing zone - with respective safety indicator, TTC or pPET''' - def __init__(self, p, probability = 1., indicator = -1): - self.x = p.x - self.y = p.y - self.probability = probability - self.indicator = indicator - - def __str__(self): - return '{0} {1} {2} {3}'.format(self.x, self.y, self.probability, self.indicator) - - @staticmethod - def save(out, points, predictionInstant, objNum1, objNum2): - for p in points: - out.write('{0} {1} {2} {3}\n'.format(objNum1, objNum2, predictionInstant, p)) - - @staticmethod - def computeExpectedIndicator(points): - return np.sum([p.indicator*p.probability for p in points])/sum([p.probability for p in points]) - -def computeCollisionTime(predictedTrajectory1, predictedTrajectory2, collisionDistanceThreshold, timeHorizon): - '''Computes the first instant - at which two predicted trajectories are within some distance threshold - Computes all the times including timeHorizon - - User has to check the first variable collision to know about a collision''' - t = 1 - p1 = predictedTrajectory1.predictPosition(t) - p2 = predictedTrajectory2.predictPosition(t) - collision = (p1-p2).norm2() <= collisionDistanceThreshold - while t < timeHorizon and not collision: - t += 1 - p1 = predictedTrajectory1.predictPosition(t) - p2 = predictedTrajectory2.predictPosition(t) - collision = (p1-p2).norm2() <= collisionDistanceThreshold - return collision, t, p1, p2 - -def savePredictedTrajectoriesFigure(currentInstant, obj1, obj2, predictedTrajectories1, predictedTrajectories2, timeHorizon, printFigure = True): - from matplotlib.pyplot import figure, axis, title, clf, savefig - if printFigure: - clf() - else: - figure() - for et in predictedTrajectories1: - for t in range(int(np.round(timeHorizon))): - et.predictPosition(t) - et.plot('rx') - for et in predictedTrajectories2: - for t in range(int(np.round(timeHorizon))): - et.predictPosition(t) - et.plot('bx') - obj1.plot('r', withOrigin = True) - obj2.plot('b', withOrigin = True) - title('instant {0}'.format(currentInstant)) - axis('equal') - if printFigure: - savefig('predicted-trajectories-t-{0}.png'.format(currentInstant)) - -def calculateProbability(nMatching,similarity,objects): - sumFrequencies=sum([nMatching[p] for p in similarity]) - prototypeProbability={} - for i in similarity: - prototypeProbability[i]= similarity[i] * float(nMatching[i])/sumFrequencies - sumProbabilities= sum([prototypeProbability[p] for p in prototypeProbability]) - probabilities={} - for i in prototypeProbability: - probabilities[objects[i]]= float(prototypeProbability[i])/sumProbabilities - return probabilities - -def findPrototypes(prototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity=0.1,mostMatched=None,spatialThreshold=1.0, delta=180): - ''' behaviour prediction first step''' - if route[0] not in noiseEntryNums: - prototypesRoutes= [ x for x in sorted(prototypes.keys()) if route[0]==x[0]] - elif route[1] not in noiseExitNums: - prototypesRoutes=[ x for x in sorted(prototypes.keys()) if route[1]==x[1]] - else: - prototypesRoutes=[x for x in sorted(prototypes.keys())] - lcss = LCSS(similarityFunc=lambda x,y: (distanceForLCSS(x,y) <= spatialThreshold),delta=delta) - similarity={} - for y in prototypesRoutes: - if y in prototypes: - prototypesIDs=prototypes[y] - for x in prototypesIDs: - s=lcss.computeNormalized(partialObjPositions, objects[x].positions) - if s >= minSimilarity: - similarity[x]=s - - if mostMatched==None: - probabilities= calculateProbability(nMatching,similarity,objects) - return probabilities - else: - mostMatchedValues=sorted(similarity.values(),reverse=True)[:mostMatched] - keys=[k for k in similarity if similarity[k] in mostMatchedValues] - newSimilarity={} - for i in keys: - newSimilarity[i]=similarity[i] - probabilities= calculateProbability(nMatching,newSimilarity,objects) - return probabilities - -def findPrototypesSpeed(prototypes,secondStepPrototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity=0.1,mostMatched=None,useDestination=True,spatialThreshold=1.0, delta=180): - if useDestination: - prototypesRoutes=[route] - else: - if route[0] not in noiseEntryNums: - prototypesRoutes= [ x for x in sorted(prototypes.keys()) if route[0]==x[0]] - elif route[1] not in noiseExitNums: - prototypesRoutes=[ x for x in sorted(prototypes.keys()) if route[1]==x[1]] - else: - prototypesRoutes=[x for x in sorted(prototypes.keys())] - lcss = LCSS(similarityFunc=lambda x,y: (distanceForLCSS(x,y) <= spatialThreshold),delta=delta) - similarity={} - for y in prototypesRoutes: - if y in prototypes: - prototypesIDs=prototypes[y] - for x in prototypesIDs: - s=lcss.computeNormalized(partialObjPositions, objects[x].positions) - if s >= minSimilarity: - similarity[x]=s - - newSimilarity={} - for i in similarity: - if i in secondStepPrototypes: - for j in secondStepPrototypes[i]: - newSimilarity[j]=similarity[i] - probabilities= calculateProbability(nMatching,newSimilarity,objects) - return probabilities - -def getPrototypeTrajectory(obj,route,currentInstant,prototypes,secondStepPrototypes,nMatching,objects,noiseEntryNums,noiseExitNums,minSimilarity=0.1,mostMatched=None,useDestination=True,useSpeedPrototype=True): - partialInterval=moving.Interval(obj.getFirstInstant(),currentInstant) - partialObjPositions= obj.getObjectInTimeInterval(partialInterval).positions - if useSpeedPrototype: - prototypeTrajectories=findPrototypesSpeed(prototypes,secondStepPrototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity,mostMatched,useDestination) - else: - prototypeTrajectories=findPrototypes(prototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity,mostMatched) - return prototypeTrajectories - - -class PredictionParameters(object): - def __init__(self, name, maxSpeed): - self.name = name - self.maxSpeed = maxSpeed - - def __str__(self): - return '{0} {1}'.format(self.name, self.maxSpeed) - - def generatePredictedTrajectories(self, obj, instant): - return None - - def computeCrossingsCollisionsAtInstant(self, currentInstant, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False): - '''returns the lists of collision points and crossing zones''' - predictedTrajectories1 = self.generatePredictedTrajectories(obj1, currentInstant) - predictedTrajectories2 = self.generatePredictedTrajectories(obj2, currentInstant) - - collisionPoints = [] - if computeCZ: - crossingZones = [] - else: - crossingZones = None - for et1 in predictedTrajectories1: - for et2 in predictedTrajectories2: - collision, t, p1, p2 = computeCollisionTime(et1, et2, collisionDistanceThreshold, timeHorizon) - if collision: - collisionPoints.append(SafetyPoint((p1+p2)*0.5, et1.probability*et2.probability, t)) - elif computeCZ: # check if there is a crossing zone - # TODO same computation as PET with metric + concatenate past trajectory with future trajectory - cz = None - t1 = 0 - while not cz and t1 < timeHorizon: # t1 <= timeHorizon-1 - t2 = 0 - while not cz and t2 < timeHorizon: - cz = moving.segmentIntersection(et1.predictPosition(t1), et1.predictPosition(t1+1), et2.predictPosition(t2), et2.predictPosition(t2+1)) - if cz is not None: - deltaV= (et1.predictPosition(t1)- et1.predictPosition(t1+1) - et2.predictPosition(t2)+ et2.predictPosition(t2+1)).norm2() - crossingZones.append(SafetyPoint(cz, et1.probability*et2.probability, abs(t1-t2)-(float(collisionDistanceThreshold)/deltaV))) - t2 += 1 - t1 += 1 - - if debug: - savePredictedTrajectoriesFigure(currentInstant, obj1, obj2, predictedTrajectories1, predictedTrajectories2, timeHorizon) - - return collisionPoints, crossingZones - - def computeCrossingsCollisions(self, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, timeInterval = None):#, nProcesses = 1): - '''Computes all crossing and collision points at each common instant for two road users. ''' - collisionPoints = {} - if computeCZ: - crossingZones = {} - else: - crossingZones = None - if timeInterval is not None: - commonTimeInterval = timeInterval - else: - commonTimeInterval = obj1.commonTimeInterval(obj2) - #if nProcesses == 1: - for i in list(commonTimeInterval)[:-1]: # do not look at the 1 last position/velocities, often with errors - cp, cz = self.computeCrossingsCollisionsAtInstant(i, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ, debug) - if len(cp) != 0: - collisionPoints[i] = cp - if computeCZ and len(cz) != 0: - crossingZones[i] = cz - return collisionPoints, crossingZones - - def computeCollisionProbability(self, obj1, obj2, collisionDistanceThreshold, timeHorizon, debug = False, timeInterval = None): - '''Computes only collision probabilities - Returns for each instant the collision probability and number of samples drawn''' - collisionProbabilities = {} - if timeInterval is not None: - commonTimeInterval = timeInterval - else: - commonTimeInterval = obj1.commonTimeInterval(obj2) - for i in list(commonTimeInterval)[:-1]: - nCollisions = 0 - predictedTrajectories1 = self.generatePredictedTrajectories(obj1, i) - predictedTrajectories2 = self.generatePredictedTrajectories(obj2, i) - for et1 in predictedTrajectories1: - for et2 in predictedTrajectories2: - collision, t, p1, p2 = computeCollisionTime(et1, et2, collisionDistanceThreshold, timeHorizon) - if collision: - nCollisions += 1 - # take into account probabilities ?? - nSamples = float(len(predictedTrajectories1)*len(predictedTrajectories2)) - collisionProbabilities[i] = [nSamples, float(nCollisions)/nSamples] - - if debug: - savePredictedTrajectoriesFigure(i, obj1, obj2, predictedTrajectories1, predictedTrajectories2, timeHorizon) - - return collisionProbabilities - -class ConstantPredictionParameters(PredictionParameters): - def __init__(self, maxSpeed): - PredictionParameters.__init__(self, 'constant velocity', maxSpeed) - - def generatePredictedTrajectories(self, obj, instant): - return [PredictedTrajectoryConstant(obj.getPositionAtInstant(instant), obj.getVelocityAtInstant(instant), maxSpeed = self.maxSpeed)] - -class NormalAdaptationPredictionParameters(PredictionParameters): - def __init__(self, maxSpeed, nPredictedTrajectories, accelerationDistribution, steeringDistribution, useFeatures = False): - '''An example of acceleration and steering distributions is - lambda: random.triangular(-self.maxAcceleration, self.maxAcceleration, 0.) - ''' - if useFeatures: - name = 'point set normal adaptation' - else: - name = 'normal adaptation' - PredictionParameters.__init__(self, name, maxSpeed) - self.nPredictedTrajectories = nPredictedTrajectories - self.useFeatures = useFeatures - self.accelerationDistribution = accelerationDistribution - self.steeringDistribution = steeringDistribution - - def __str__(self): - return PredictionParameters.__str__(self)+' {0} {1} {2}'.format(self.nPredictedTrajectories, - self.maxAcceleration, - self.maxSteering) - - def generatePredictedTrajectories(self, obj, instant): - predictedTrajectories = [] - if self.useFeatures and obj.hasFeatures(): - features = [f for f in obj.getFeatures() if f.existsAtInstant(instant)] - positions = [f.getPositionAtInstant(instant) for f in features] - velocities = [f.getVelocityAtInstant(instant) for f in features] - else: - positions = [obj.getPositionAtInstant(instant)] - velocities = [obj.getVelocityAtInstant(instant)] - probability = 1./float(len(positions)*self.nPredictedTrajectories) - for i in range(self.nPredictedTrajectories): - for initialPosition,initialVelocity in zip(positions, velocities): - predictedTrajectories.append(PredictedTrajectoryRandomControl(initialPosition, - initialVelocity, - self.accelerationDistribution, - self.steeringDistribution, - probability, - maxSpeed = self.maxSpeed)) - return predictedTrajectories - -class PointSetPredictionParameters(PredictionParameters): - def __init__(self, maxSpeed): - PredictionParameters.__init__(self, 'point set', maxSpeed) - - def generatePredictedTrajectories(self, obj, instant): - predictedTrajectories = [] - if obj.hasFeatures(): - features = [f for f in obj.getFeatures() if f.existsAtInstant(instant)] - positions = [f.getPositionAtInstant(instant) for f in features] - velocities = [f.getVelocityAtInstant(instant) for f in features] - probability = 1./float(len(positions)) - for initialPosition,initialVelocity in zip(positions, velocities): - predictedTrajectories.append(PredictedTrajectoryConstant(initialPosition, initialVelocity, probability = probability, maxSpeed = self.maxSpeed)) - return predictedTrajectories - else: - print('Object {} has no features'.format(obj.getNum())) - return None - - -class EvasiveActionPredictionParameters(PredictionParameters): - def __init__(self, maxSpeed, nPredictedTrajectories, accelerationDistribution, steeringDistribution, useFeatures = False): - '''Suggested acceleration distribution may not be symmetric, eg - lambda: random.triangular(self.minAcceleration, self.maxAcceleration, 0.)''' - - if useFeatures: - name = 'point set evasive action' - else: - name = 'evasive action' - PredictionParameters.__init__(self, name, maxSpeed) - self.nPredictedTrajectories = nPredictedTrajectories - self.useFeatures = useFeatures - self.accelerationDistribution = accelerationDistribution - self.steeringDistribution = steeringDistribution - - def __str__(self): - return PredictionParameters.__str__(self)+' {0} {1} {2} {3}'.format(self.nPredictedTrajectories, self.minAcceleration, self.maxAcceleration, self.maxSteering) - - def generatePredictedTrajectories(self, obj, instant): - predictedTrajectories = [] - if self.useFeatures and obj.hasFeatures(): - features = [f for f in obj.getFeatures() if f.existsAtInstant(instant)] - positions = [f.getPositionAtInstant(instant) for f in features] - velocities = [f.getVelocityAtInstant(instant) for f in features] - else: - positions = [obj.getPositionAtInstant(instant)] - velocities = [obj.getVelocityAtInstant(instant)] - probability = 1./float(self.nPredictedTrajectories) - for i in range(self.nPredictedTrajectories): - for initialPosition,initialVelocity in zip(positions, velocities): - predictedTrajectories.append(PredictedTrajectoryConstant(initialPosition, - initialVelocity, - moving.NormAngle(self.accelerationDistribution(), - self.steeringDistribution()), - probability, - self.maxSpeed)) - return predictedTrajectories - - -class CVDirectPredictionParameters(PredictionParameters): - '''Prediction parameters of prediction at constant velocity - using direct computation of the intersecting point - Warning: the computed time to collision may be higher than timeHorizon (not used)''' - - def __init__(self): - PredictionParameters.__init__(self, 'constant velocity (direct computation)', None) - - def computeCrossingsCollisionsAtInstant(self, currentInstant, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, *kwargs): - collisionPoints = [] - if computeCZ: - crossingZones = [] - else: - crossingZones = None - - p1 = obj1.getPositionAtInstant(currentInstant) - p2 = obj2.getPositionAtInstant(currentInstant) - if (p1-p2).norm2() <= collisionDistanceThreshold: - collisionPoints = [SafetyPoint((p1+p2)*0.5, 1., 0.)] - else: - v1 = obj1.getVelocityAtInstant(currentInstant) - v2 = obj2.getVelocityAtInstant(currentInstant) - intersection = moving.intersection(p1, p1+v1, p2, p2+v2) - - if intersection is not None: - dp1 = intersection-p1 - dp2 = intersection-p2 - dot1 = moving.Point.dot(dp1, v1) - dot2 = moving.Point.dot(dp2, v2) - if (computeCZ and (dot1 > 0 or dot2 > 0)) or (dot1 > 0 and dot2 > 0): # if the road users are moving towards the intersection or if computing pPET - dist1 = dp1.norm2() - dist2 = dp2.norm2() - s1 = math.copysign(v1.norm2(), dot1) - s2 = math.copysign(v2.norm2(), dot2) - halfCollisionDistanceThreshold = collisionDistanceThreshold/2. - timeInterval1 = moving.TimeInterval(max(0,dist1-halfCollisionDistanceThreshold)/s1, (dist1+halfCollisionDistanceThreshold)/s1) - timeInterval2 = moving.TimeInterval(max(0,dist2-halfCollisionDistanceThreshold)/s2, (dist2+halfCollisionDistanceThreshold)/s2) - collisionTimeInterval = moving.TimeInterval.intersection(timeInterval1, timeInterval2) - - if collisionTimeInterval.empty(): - if computeCZ: - crossingZones = [SafetyPoint(intersection, 1., timeInterval1.distance(timeInterval2))] - else: - collisionPoints = [SafetyPoint(intersection, 1., collisionTimeInterval.center())] - - if debug and intersection is not None: - from matplotlib.pyplot import plot, figure, axis, title - figure() - plot([p1.x, intersection.x], [p1.y, intersection.y], 'r') - plot([p2.x, intersection.x], [p2.y, intersection.y], 'b') - intersection.plot() - obj1.plot('r') - obj2.plot('b') - title('instant {0}'.format(currentInstant)) - axis('equal') - - return collisionPoints, crossingZones - -class CVExactPredictionParameters(PredictionParameters): - '''Prediction parameters of prediction at constant velocity - using direct computation of the intersecting point (solving the equation) - Warning: the computed time to collision may be higher than timeHorizon (not used)''' - - def __init__(self): - PredictionParameters.__init__(self, 'constant velocity (direct exact computation)', None) - - def computeCrossingsCollisionsAtInstant(self, currentInstant, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, *kwargs): - 'TODO compute pPET' - collisionPoints = [] - crossingZones = [] - - p1 = obj1.getPositionAtInstant(currentInstant) - p2 = obj2.getPositionAtInstant(currentInstant) - v1 = obj1.getVelocityAtInstant(currentInstant) - v2 = obj2.getVelocityAtInstant(currentInstant) - #intersection = moving.intersection(p1, p1+v1, p2, p2+v2) - - if not moving.Point.parallel(v1, v2): - ttc = moving.Point.timeToCollision(p1, p2, v1, v2, collisionDistanceThreshold) - if ttc is not None: - collisionPoints = [SafetyPoint((p1+(v1*ttc)+p2+(v2*ttc))*0.5, 1., ttc)] - else: - pass # compute pPET - - return collisionPoints, crossingZones - -class PrototypePredictionParameters(PredictionParameters): - def __init__(self, prototypes, nPredictedTrajectories, pointSimilarityDistance, minSimilarity, lcssMetric = 'cityblock', minFeatureTime = 10, constantSpeed = False, useFeatures = True): - PredictionParameters.__init__(self, 'prototypes', None) - self.prototypes = prototypes - self.nPredictedTrajectories = nPredictedTrajectories - self.lcss = LCSS(metric = lcssMetric, epsilon = pointSimilarityDistance) - self.minSimilarity = minSimilarity - self.minFeatureTime = minFeatureTime - self.constantSpeed = constantSpeed - self.useFeatures = useFeatures - - def getLcss(self): - return self.lcss - - def addPredictedTrajectories(self, predictedTrajectories, obj, instant): - obj.computeTrajectorySimilarities(self.prototypes, self.lcss) - for proto, similarities in zip(self.prototypes, obj.prototypeSimilarities): - if similarities[instant-obj.getFirstInstant()] >= self.minSimilarity: - initialPosition = obj.getPositionAtInstant(instant) - initialVelocity = obj.getVelocityAtInstant(instant) - predictedTrajectories.append(PredictedTrajectoryPrototype(initialPosition, initialVelocity, proto.getMovingObject(), constantSpeed = self.constantSpeed, probability = proto.getNMatchings())) - - def generatePredictedTrajectories(self, obj, instant): - predictedTrajectories = [] - if instant-obj.getFirstInstant()+1 >= self.minFeatureTime: - if self.useFeatures and obj.hasFeatures(): - if not hasattr(obj, 'currentPredictionFeatures'): - obj.currentPredictionFeatures = [] - else: - obj.currentPredictionFeatures[:] = [f for f in obj.currentPredictionFeatures if f.existsAtInstant(instant)] - firstInstants = [(f,f.getFirstInstant()) for f in obj.getFeatures() if f.existsAtInstant(instant) and f not in obj.currentPredictionFeatures] - firstInstants.sort(key = lambda t: t[1]) - for f,t1 in firstInstants[:min(self.nPredictedTrajectories, len(firstInstants), self.nPredictedTrajectories-len(obj.currentPredictionFeatures))]: - obj.currentPredictionFeatures.append(f) - for f in obj.currentPredictionFeatures: - self.addPredictedTrajectories(predictedTrajectories, f, instant) - else: - self.addPredictedTrajectories(predictedTrajectories, obj, instant) - return predictedTrajectories - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/prediction.txt') - #suite = doctest.DocTestSuite() - unittest.TextTestRunner().run(suite) - #doctest.testmod() - #doctest.testfile("example.txt") -
--- a/python/processing.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,20 +0,0 @@ -#! /usr/bin/env python -'''Algorithms to process trajectories and moving objects''' - -import moving - -import numpy as np - - -def extractSpeeds(objects, zone): - speeds = {} - objectsNotInZone = [] - import matplotlib.nxutils as nx - for o in objects: - inPolygon = nx.points_inside_poly(o.getPositions().asArray().T, zone.T) - if inPolygon.any(): - objspeeds = [o.getVelocityAt(i).norm2() for i in range(int(o.length()-1)) if inPolygon[i]] - speeds[o.num] = np.mean(objspeeds) # km/h - else: - objectsNotInZone.append(o) - return speeds, objectsNotInZone
--- a/python/requirements.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ -matplotlib -numpy - -The following libraries are optional. They are necessary for (sometimes very) specific classes/functions. - -Computer Vision (cvutils.py): opencv, scikit-image -Statistics and machine learning (ml.py): scipy, scikit-learn -Moving object geometry (currently commented) (moving.py) and plotting shapely polygons (utils.py): shapely -Tabular data loading/processing (storage.py): pandas
--- a/python/run-tests.sh Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,10 +0,0 @@ -#!/bin/sh -# for file in tests/*... basename -for f in ./*.py -do - python3 $f -done -for f in ./tests/*.py -do - python3 $f -done
--- a/python/sensors.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,65 +0,0 @@ -#! /usr/bin/env python -'''Libraries for detecting, counting, etc., road users''' - -from numpy import mean, isnan - -import moving - -# TODO graphical user interface for creation - -class Sensor: - def detect(self, o): - print("Detect method not implemented") - return False - - def detectInstants(self, o): - print("DetectInstants method not implemented") - return [] - -class BoxSensor(Sensor): - def __init__(self, polygon, minNPointsInBox = 1): - self.polygon = polygon # check 2xN? - self.minNPointsInBox = minNPointsInBox - - def detectInstants(self, obj): - indices = obj.getPositions().getInstantsInPolygon(self.polygon) - firstInstant = obj.getFirstInstant() - return [i+firstInstant for i in indices] - - def detect(self, obj): - instants = self.detectInstants(obj) - return len(instants) >= self.minNPointsInBox - -def detectAnd(sensors, obj): - 'Returns True if all sensors detect the object' - result = True - for s in sensors: - result = result and s.detect(obj) - if not result: - return result - return result - -def detectOr(sensors, obj): - 'Returns True if any sensor detects the object' - result = False - for s in sensors: - result = result or s.detect(obj) - if result: - return result - return result - -def detectAndOrder(sensors, obj): - 'Returns True if all sensors are detected and in their order' - detectionInstants = [] - for s in sensors: - instants = s.detectInstants(obj) - if len(instants) == 0: - return False - else: - detectionInstants.append(mean(instants)) - result = True - for i in range(len(sensors)-1): - result = result and (detectionInstants[i] <= detectionInstants[i+1]) - if not result: - return result - return result
--- a/python/storage.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1477 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -'''Various utilities to save and load data''' - -import utils, moving, events, indicators, shutil -from base import VideoFilenameAddable - -from pathlib import Path -from copy import copy -import sqlite3, logging -from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg -from pandas import read_csv, merge - - -commentChar = '#' - -delimiterChar = '%'; - -ngsimUserTypes = {'twowheels':1, - 'car':2, - 'truck':3} - -tableNames = {'feature':'positions', - 'object': 'objects', - 'objectfeatures': 'positions'} - -######################### -# Sqlite -######################### - -# utils -def printDBError(error): - print('DB Error: {}'.format(error)) - -def dropTables(connection, tableNames): - 'deletes the table with names in tableNames' - try: - cursor = connection.cursor() - for tableName in tableNames: - cursor.execute('DROP TABLE IF EXISTS '+tableName) - except sqlite3.OperationalError as error: - printDBError(error) - -def deleteFromSqlite(filename, dataType): - 'Deletes (drops) some tables in the filename depending on type of data' - if Path(filename).is_file(): - with sqlite3.connect(filename) as connection: - if dataType == 'object': - dropTables(connection, ['objects', 'objects_features']) - elif dataType == 'interaction': - dropTables(connection, ['interactions', 'indicators']) - elif dataType == 'bb': - dropTables(connection, ['bounding_boxes']) - elif dataType == 'pois': - dropTables(connection, ['gaussians2d', 'objects_pois']) - elif dataType == 'prototype': - dropTables(connection, ['prototypes', 'objects_prototypes']) - else: - print('Unknown data type {} to delete from database'.format(dataType)) - else: - print('{} does not exist'.format(filename)) - -def tableExists(connection, tableName): - 'indicates if the table exists in the database' - try: - cursor = connection.cursor() - cursor.execute('SELECT COUNT(*) FROM SQLITE_MASTER WHERE type = \'table\' AND name = \''+tableName+'\'') - return cursor.fetchone()[0] == 1 - except sqlite3.OperationalError as error: - printDBError(error) - -def createTrajectoryTable(cursor, tableName): - if tableName.endswith('positions') or tableName.endswith('velocities'): - cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (trajectory_id INTEGER, frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") - else: - print('Unallowed name {} for trajectory table'.format(tableName)) - -def createObjectsTable(cursor): - cursor.execute("CREATE TABLE IF NOT EXISTS objects (object_id INTEGER, road_user_type INTEGER, n_objects INTEGER, PRIMARY KEY(object_id))") - -def createAssignmentTable(cursor, objectType1, objectType2, objectIdColumnName1, objectIdColumnName2): - cursor.execute("CREATE TABLE IF NOT EXISTS "+objectType1+"s_"+objectType2+"s ("+objectIdColumnName1+" INTEGER, "+objectIdColumnName2+" INTEGER, PRIMARY KEY("+objectIdColumnName1+","+objectIdColumnName2+"))") - -def createObjectsFeaturesTable(cursor): - cursor.execute("CREATE TABLE IF NOT EXISTS objects_features (object_id INTEGER, trajectory_id INTEGER, PRIMARY KEY(object_id, trajectory_id))") - - -def createCurvilinearTrajectoryTable(cursor): - cursor.execute("CREATE TABLE IF NOT EXISTS curvilinear_positions (trajectory_id INTEGER, frame_number INTEGER, s_coordinate REAL, y_coordinate REAL, lane TEXT, PRIMARY KEY(trajectory_id, frame_number))") - -def createFeatureCorrespondenceTable(cursor): - cursor.execute('CREATE TABLE IF NOT EXISTS feature_correspondences (trajectory_id INTEGER, source_dbname VARCHAR, db_trajectory_id INTEGER, PRIMARY KEY(trajectory_id))') - -def createInteractionTable(cursor): - cursor.execute('CREATE TABLE IF NOT EXISTS interactions (id INTEGER PRIMARY KEY, object_id1 INTEGER, object_id2 INTEGER, first_frame_number INTEGER, last_frame_number INTEGER, FOREIGN KEY(object_id1) REFERENCES objects(id), FOREIGN KEY(object_id2) REFERENCES objects(id))') - -def createIndicatorTable(cursor): - cursor.execute('CREATE TABLE IF NOT EXISTS indicators (interaction_id INTEGER, indicator_type INTEGER, frame_number INTEGER, value REAL, FOREIGN KEY(interaction_id) REFERENCES interactions(id), PRIMARY KEY(interaction_id, indicator_type, frame_number))') - -def insertTrajectoryQuery(tableName): - return "INSERT INTO "+tableName+" VALUES (?,?,?,?)" - -def insertObjectQuery(): - return "INSERT INTO objects VALUES (?,?,?)" - -def insertObjectFeatureQuery(): - return "INSERT INTO objects_features VALUES (?,?)" - -def createIndex(connection, tableName, columnName, unique = False): - '''Creates an index for the column in the table - I will make querying with a condition on this column faster''' - try: - cursor = connection.cursor() - s = "CREATE " - if unique: - s += "UNIQUE " - cursor.execute(s+"INDEX IF NOT EXISTS "+tableName+"_"+columnName+"_index ON "+tableName+"("+columnName+")") - connection.commit() - #connection.close() - except sqlite3.OperationalError as error: - printDBError(error) - -def getNumberRowsTable(connection, tableName, columnName = None): - '''Returns the number of rows for the table - If columnName is not None, means we want the number of distinct values for that column - (otherwise, we can just count(*))''' - try: - cursor = connection.cursor() - if columnName is None: - cursor.execute("SELECT COUNT(*) from "+tableName) - else: - cursor.execute("SELECT COUNT(DISTINCT "+columnName+") from "+tableName) - return cursor.fetchone()[0] - except sqlite3.OperationalError as error: - printDBError(error) - -def getMinMax(connection, tableName, columnName, minmax): - '''Returns max/min or both for given column in table - minmax must be string max, min or minmax''' - try: - cursor = connection.cursor() - if minmax == 'min' or minmax == 'max': - cursor.execute("SELECT "+minmax+"("+columnName+") from "+tableName) - elif minmax == 'minmax': - cursor.execute("SELECT MIN("+columnName+"), MAX("+columnName+") from "+tableName) - else: - print("Argument minmax unknown: {}".format(minmax)) - return cursor.fetchone()[0] - except sqlite3.OperationalError as error: - printDBError(error) - -def getObjectCriteria(objectNumbers): - if objectNumbers is None: - query = '' - elif type(objectNumbers) == int: - query = '<= {0}'.format(objectNumbers-1) - elif type(objectNumbers) == list: - query = 'in ('+', '.join([str(n) for n in objectNumbers])+')' - else: - print('objectNumbers {} are not a known type ({})'.format(objectNumbers, type(objectNumbers))) - query = '' - return query - -def loadTrajectoriesFromTable(connection, tableName, trajectoryType, objectNumbers = None, timeStep = None): - '''Loads trajectories (in the general sense) from the given table - can be positions or velocities - - returns a moving object''' - cursor = connection.cursor() - - try: - objectCriteria = getObjectCriteria(objectNumbers) - queryStatement = None - if trajectoryType == 'feature': - queryStatement = 'SELECT * from '+tableName - if objectNumbers is not None and timeStep is not None: - queryStatement += ' WHERE trajectory_id '+objectCriteria+' AND frame_number%{} = 0'.format(timeStep) - elif objectNumbers is not None: - queryStatement += ' WHERE trajectory_id '+objectCriteria - elif timeStep is not None: - queryStatement += ' WHERE frame_number%{} = 0'.format(timeStep) - queryStatement += ' ORDER BY trajectory_id, frame_number' - elif trajectoryType == 'object': - queryStatement = 'SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from '+tableName+' P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id' - if objectNumbers is not None: - queryStatement += ' AND OF.object_id '+objectCriteria - if timeStep is not None: - queryStatement += ' AND P.frame_number%{} = 0'.format(timeStep) - queryStatement += ' GROUP BY OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number' - elif trajectoryType in ['bbtop', 'bbbottom']: - if trajectoryType == 'bbtop': - corner = 'top_left' - elif trajectoryType == 'bbbottom': - corner = 'bottom_right' - queryStatement = 'SELECT object_id, frame_number, x_'+corner+', y_'+corner+' FROM '+tableName - if objectNumbers is not None and timeStep is not None: - queryStatement += ' WHERE object_id '+objectCriteria+' AND frame_number%{} = 0'.format(timeStep) - elif objectNumbers is not None: - queryStatement += ' WHERE object_id '+objectCriteria - elif timeStep is not None: - queryStatement += ' WHERE frame_number%{} = 0'.format(timeStep) - queryStatement += ' ORDER BY object_id, frame_number' - else: - print('Unknown trajectory type {}'.format(trajectoryType)) - if queryStatement is not None: - cursor.execute(queryStatement) - logging.debug(queryStatement) - except sqlite3.OperationalError as error: - printDBError(error) - return [] - - objId = -1 - obj = None - objects = [] - for row in cursor: - if row[0] != objId: - objId = row[0] - if obj is not None and (obj.length() == obj.positions.length() or (timeStep is not None and npceil(obj.length()/timeStep) == obj.positions.length())): - objects.append(obj) - elif obj is not None: - print('Object {} is missing {} positions'.format(obj.getNum(), int(obj.length())-obj.positions.length())) - obj = moving.MovingObject(row[0], timeInterval = moving.TimeInterval(row[1], row[1]), positions = moving.Trajectory([[row[2]],[row[3]]])) - else: - obj.timeInterval.last = row[1] - obj.positions.addPositionXY(row[2],row[3]) - - if obj is not None and (obj.length() == obj.positions.length() or (timeStep is not None and npceil(obj.length()/timeStep) == obj.positions.length())): - objects.append(obj) - elif obj is not None: - print('Object {} is missing {} positions'.format(obj.getNum(), int(obj.length())-obj.positions.length())) - - return objects - -def loadUserTypesFromTable(cursor, objectNumbers): - objectCriteria = getObjectCriteria(objectNumbers) - queryStatement = 'SELECT object_id, road_user_type FROM objects' - if objectNumbers is not None: - queryStatement += ' WHERE object_id '+objectCriteria - cursor.execute(queryStatement) - userTypes = {} - for row in cursor: - userTypes[row[0]] = row[1] - return userTypes - -def loadTrajectoriesFromSqlite(filename, trajectoryType, objectNumbers = None, withFeatures = False, timeStep = None, tablePrefix = None): - '''Loads the trajectories (in the general sense, - either features, objects (feature groups) or bounding box series) - The number loaded is either the first objectNumbers objects, - or the indices in objectNumbers from the database''' - objects = [] - with sqlite3.connect(filename) as connection: - if tablePrefix is None: - prefix = '' - else: - prefix = tablePrefix + '_' - objects = loadTrajectoriesFromTable(connection, prefix+'positions', trajectoryType, objectNumbers, timeStep) - objectVelocities = loadTrajectoriesFromTable(connection, prefix+'velocities', trajectoryType, objectNumbers, timeStep) - - if len(objectVelocities) > 0: - for o,v in zip(objects, objectVelocities): - if o.getNum() == v.getNum(): - o.velocities = v.positions - o.velocities.duplicateLastPosition() # avoid having velocity shorter by one position than positions - else: - print('Could not match positions {0} with velocities {1}'.format(o.getNum(), v.getNum())) - - if trajectoryType == 'object': - cursor = connection.cursor() - try: - # attribute feature numbers to objects - queryStatement = 'SELECT trajectory_id, object_id FROM objects_features' - if objectNumbers is not None: - queryStatement += ' WHERE object_id '+getObjectCriteria(objectNumbers) - queryStatement += ' ORDER BY object_id' # order is important to group all features per object - logging.debug(queryStatement) - cursor.execute(queryStatement) - - featureNumbers = {} - for row in cursor: - objId = row[1] - if objId not in featureNumbers: - featureNumbers[objId] = [row[0]] - else: - featureNumbers[objId].append(row[0]) - - for obj in objects: - obj.featureNumbers = featureNumbers[obj.getNum()] - - # load userType - userTypes = loadUserTypesFromTable(cursor, objectNumbers) - for obj in objects: - obj.userType = userTypes[obj.getNum()] - - if withFeatures: - nFeatures = 0 - for obj in objects: - nFeatures = max(nFeatures, max(obj.featureNumbers)) - features = loadTrajectoriesFromSqlite(filename, 'feature', nFeatures+1, timeStep = timeStep) - for obj in objects: - obj.setFeatures(features) - - except sqlite3.OperationalError as error: - printDBError(error) - return objects - -def loadObjectFeatureFrameNumbers(filename, objectNumbers = None): - 'Loads the feature frame numbers for each object' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - queryStatement = 'SELECT OF.object_id, TL.trajectory_id, TL.length FROM (SELECT trajectory_id, max(frame_number)-min(frame_number) AS length FROM positions GROUP BY trajectory_id) TL, objects_features OF WHERE TL.trajectory_id = OF.trajectory_id' - if objectNumbers is not None: - queryStatement += ' AND object_id '+getObjectCriteria(objectNumbers) - queryStatement += ' ORDER BY OF.object_id, TL.length DESC' - logging.debug(queryStatement) - cursor.execute(queryStatement) - objectFeatureNumbers = {} - for row in cursor: - objId = row[0] - if objId in objectFeatureNumbers: - objectFeatureNumbers[objId].append(row[1]) - else: - objectFeatureNumbers[objId] = [row[1]] - return objectFeatureNumbers - except sqlite3.OperationalError as error: - printDBError(error) - return None - -def addCurvilinearTrajectoriesFromSqlite(filename, objects): - '''Adds curvilinear positions (s_coordinate, y_coordinate, lane) - from a database to an existing MovingObject dict (indexed by each objects's num)''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from curvilinear_positions order by trajectory_id, frame_number') - except sqlite3.OperationalError as error: - printDBError(error) - return [] - - missingObjectNumbers = [] - objNum = None - for row in cursor: - if objNum != row[0]: - objNum = row[0] - if objNum in objects: - objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() - else: - missingObjectNumbers.append(objNum) - if objNum in objects: - objects[objNum].curvilinearPositions.addPositionSYL(row[2],row[3],row[4]) - if len(missingObjectNumbers) > 0: - print('List of missing objects to attach corresponding curvilinear trajectories: {}'.format(missingObjectNumbers)) - -def saveTrajectoriesToTable(connection, objects, trajectoryType, tablePrefix = None): - 'Saves trajectories in table tableName' - cursor = connection.cursor() - # Parse feature and/or object structure and commit to DB - if(trajectoryType == 'feature' or trajectoryType == 'object'): - # Extract features from objects - if trajectoryType == 'object': - features = [] - for obj in objects: - if obj.hasFeatures(): - features += obj.getFeatures() - if len(features) == 0: - print('Warning, objects have no features') # todo save centroid trajectories? - elif trajectoryType == 'feature': - features = objects - # Setup feature queries - if tablePrefix is None: - prefix = '' - else: - prefix = tablePrefix+'_' - createTrajectoryTable(cursor, prefix+"positions") - createTrajectoryTable(cursor, prefix+"velocities") - positionQuery = insertTrajectoryQuery(prefix+"positions") - velocityQuery = insertTrajectoryQuery(prefix+"velocities") - # Setup object queries - if trajectoryType == 'object': - createObjectsTable(cursor) - createObjectsFeaturesTable(cursor) - objectQuery = insertObjectQuery() - objectFeatureQuery = insertObjectFeatureQuery() - for feature in features: - num = feature.getNum() - frameNum = feature.getFirstInstant() - for p in feature.getPositions(): - cursor.execute(positionQuery, (num, frameNum, p.x, p.y)) - frameNum += 1 - velocities = feature.getVelocities() - if velocities is not None: - frameNum = feature.getFirstInstant() - for v in velocities[:-1]: - cursor.execute(velocityQuery, (num, frameNum, v.x, v.y)) - frameNum += 1 - if trajectoryType == 'object': - for obj in objects: - if obj.hasFeatures(): - for feature in obj.getFeatures(): - featureNum = feature.getNum() - cursor.execute(objectFeatureQuery, (obj.getNum(), featureNum)) - cursor.execute(objectQuery, (obj.getNum(), obj.getUserType(), 1)) - # Parse curvilinear position structure - elif(trajectoryType == 'curvilinear'): - createCurvilinearTrajectoryTable(cursor) - curvilinearQuery = "INSERT INTO curvilinear_positions VALUES (?,?,?,?,?)" - for obj in objects: - num = obj.getNum() - frameNum = obj.getFirstInstant() - for p in obj.getCurvilinearPositions(): - cursor.execute(curvilinearQuery, (num, frameNum, p[0], p[1], p[2])) - frameNum += 1 - else: - print('Unknown trajectory type {}'.format(trajectoryType)) - connection.commit() - -def saveTrajectoriesToSqlite(outputFilename, objects, trajectoryType): - '''Writes features, ie the trajectory positions (and velocities if exist) - with their instants to a specified sqlite file - Either feature positions (and velocities if they exist) - or curvilinear positions will be saved at a time''' - - with sqlite3.connect(outputFilename) as connection: - try: - saveTrajectoriesToTable(connection, objects, trajectoryType, None) - except sqlite3.OperationalError as error: - printDBError(error) - -def setRoadUserTypes(filename, objects): - '''Saves the user types of the objects in the sqlite database stored in filename - The objects should exist in the objects table''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - for obj in objects: - cursor.execute('update objects set road_user_type = {} WHERE object_id = {}'.format(obj.getUserType(), obj.getNum())) - connection.commit() - -def loadBBMovingObjectsFromSqlite(filename, objectType = 'bb', objectNumbers = None, timeStep = None): - '''Loads bounding box moving object from an SQLite - (format of SQLite output by the ground truth annotation tool - or Urban Tracker - - Load descriptions?''' - objects = [] - with sqlite3.connect(filename) as connection: - if objectType == 'bb': - topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep) - bottomCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbbottom', objectNumbers, timeStep) - userTypes = loadUserTypesFromTable(connection.cursor(), objectNumbers) # string format is same as object - - for t, b in zip(topCorners, bottomCorners): - num = t.getNum() - if t.getNum() == b.getNum(): - annotation = moving.BBMovingObject(num, t.getTimeInterval(), t, b, userTypes[num]) - objects.append(annotation) - else: - print ('Unknown type of bounding box {}'.format(objectType)) - return objects - -def saveInteraction(cursor, interaction): - roadUserNumbers = list(interaction.getRoadUserNumbers()) - cursor.execute('INSERT INTO interactions VALUES({}, {}, {}, {}, {})'.format(interaction.getNum(), roadUserNumbers[0], roadUserNumbers[1], interaction.getFirstInstant(), interaction.getLastInstant())) - -def saveInteractionsToSqlite(filename, interactions): - 'Saves the interactions in the table' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - createInteractionTable(cursor) - for inter in interactions: - saveInteraction(cursor, inter) - except sqlite3.OperationalError as error: - printDBError(error) - connection.commit() - -def saveIndicator(cursor, interactionNum, indicator): - for instant in indicator.getTimeInterval(): - if indicator[instant]: - cursor.execute('INSERT INTO indicators VALUES({}, {}, {}, {})'.format(interactionNum, events.Interaction.indicatorNameToIndices[indicator.getName()], instant, indicator[instant])) - -def saveIndicatorsToSqlite(filename, interactions, indicatorNames = events.Interaction.indicatorNames): - 'Saves the indicator values in the table' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - createInteractionTable(cursor) - createIndicatorTable(cursor) - for inter in interactions: - saveInteraction(cursor, inter) - for indicatorName in indicatorNames: - indicator = inter.getIndicator(indicatorName) - if indicator is not None: - saveIndicator(cursor, inter.getNum(), indicator) - except sqlite3.OperationalError as error: - printDBError(error) - connection.commit() - -def loadInteractionsFromSqlite(filename): - '''Loads interaction and their indicators - - TODO choose the interactions to load''' - interactions = [] - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('SELECT INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND WHERE INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type, IND.frame_number') - interactionNum = -1 - indicatorTypeNum = -1 - tmpIndicators = {} - for row in cursor: - if row[0] != interactionNum: - interactionNum = row[0] - interactions.append(events.Interaction(interactionNum, moving.TimeInterval(row[3],row[4]), row[1], row[2])) - interactions[-1].indicators = {} - if indicatorTypeNum != row[5] or row[0] != interactionNum: - indicatorTypeNum = row[5] - indicatorName = events.Interaction.indicatorNames[indicatorTypeNum] - indicatorValues = {row[6]:row[7]} - interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues, mostSevereIsMax = not indicatorName in events.Interaction.timeIndicators) - else: - indicatorValues[row[6]] = row[7] - interactions[-1].indicators[indicatorName].timeInterval.last = row[6] - except sqlite3.OperationalError as error: - printDBError(error) - return [] - return interactions -# load first and last object instants -# CREATE TEMP TABLE IF NOT EXISTS object_instants AS SELECT OF.object_id, min(frame_number) as first_instant, max(frame_number) as last_instant from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id group by OF.object_id order by OF.object_id - -def createBoundingBoxTable(filename, invHomography = None): - '''Create the table to store the object bounding boxes in image space - ''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('CREATE TABLE IF NOT EXISTS bounding_boxes (object_id INTEGER, frame_number INTEGER, x_top_left REAL, y_top_left REAL, x_bottom_right REAL, y_bottom_right REAL, PRIMARY KEY(object_id, frame_number))') - cursor.execute('INSERT INTO bounding_boxes SELECT object_id, frame_number, min(x), min(y), max(x), max(y) from ' - '(SELECT object_id, frame_number, (x*{}+y*{}+{})/w as x, (x*{}+y*{}+{})/w as y from ' - '(SELECT OF.object_id, P.frame_number, P.x_coordinate as x, P.y_coordinate as y, P.x_coordinate*{}+P.y_coordinate*{}+{} as w from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id)) '.format(invHomography[0,0], invHomography[0,1], invHomography[0,2], invHomography[1,0], invHomography[1,1], invHomography[1,2], invHomography[2,0], invHomography[2,1], invHomography[2,2])+ - 'GROUP BY object_id, frame_number') - except sqlite3.OperationalError as error: - printDBError(error) - connection.commit() - -def loadBoundingBoxTableForDisplay(filename): - '''Loads bounding boxes from bounding_boxes table for display over trajectories''' - boundingBoxes = {} # list of bounding boxes for each instant - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('SELECT name FROM sqlite_master WHERE type=\'table\' AND name=\'bounding_boxes\'') - result = cursor.fetchall() - if len(result) > 0: - cursor.execute('SELECT * FROM bounding_boxes') - for row in cursor: - boundingBoxes.setdefault(row[1], []).append([moving.Point(row[2], row[3]), moving.Point(row[4], row[5])]) - except sqlite3.OperationalError as error: - printDBError(error) - return boundingBoxes - return boundingBoxes - -######################### -# saving and loading for scene interpretation: POIs and Prototypes -######################### - -def savePrototypesToSqlite(filename, prototypes): - '''save the prototypes (a prototype is defined by a filename, a number (id) and type''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_filename VARCHAR, prototype_id INTEGER, trajectory_type VARCHAR CHECK (trajectory_type IN (\"feature\", \"object\")), nmatchings INTEGER, PRIMARY KEY (prototype_filename, prototype_id, trajectory_type))') - for p in prototypes: - cursor.execute('INSERT INTO prototypes VALUES(?,?,?,?)', (p.getFilename(), p.getNum(), p.getTrajectoryType(), p.getNMatchings())) - except sqlite3.OperationalError as error: - printDBError(error) - connection.commit() - -def savePrototypeAssignmentsToSqlite(filename, objects, objectType, labels, prototypes): - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - if objectType == 'feature': - tableName = 'features_prototypes' - objectIdColumnName = 'trajectory_id' - elif objectType == 'object': - tableName = 'objects_prototypes' - objectIdColumnName = 'object_id' - cursor.execute('CREATE TABLE IF NOT EXISTS '+tableName+' ('+objectIdColumnName+' INTEGER, prototype_filename VARCHAR, prototype_id INTEGER, trajectory_type VARCHAR CHECK (trajectory_type IN (\"feature\", \"object\")), PRIMARY KEY('+objectIdColumnName+', prototype_filename, prototype_id, trajectory_type))') - for obj, label in zip(objects, labels): - proto = prototypes[label] - cursor.execute('INSERT INTO objects_prototypes VALUES(?,?,?,?)', (obj.getNum(), proto.getFilename(), proto.getNum(), proto.getTrajectoryType())) - except sqlite3.OperationalError as error: - printDBError(error) - connection.commit() - -def loadPrototypesFromSqlite(filename, withTrajectories = True): - 'Loads prototype ids and matchings (if stored)' - prototypes = [] - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - objects = [] - try: - cursor.execute('SELECT * FROM prototypes') - for row in cursor: - prototypes.append(moving.Prototype(row[0], row[1], row[2], row[3])) - if withTrajectories: - for p in prototypes: - p.setMovingObject(loadTrajectoriesFromSqlite(p.getFilename(), p.getTrajectoryType(), [p.getNum()])[0]) - # loadingInformation = {} # complicated slightly optimized - # for p in prototypes: - # dbfn = p.getFilename() - # trajType = p.getTrajectoryType() - # if (dbfn, trajType) in loadingInformation: - # loadingInformation[(dbfn, trajType)].append(p) - # else: - # loadingInformation[(dbfn, trajType)] = [p] - # for k, v in loadingInformation.iteritems(): - # objects += loadTrajectoriesFromSqlite(k[0], k[1], [p.getNum() for p in v]) - except sqlite3.OperationalError as error: - printDBError(error) - if len(set([p.getTrajectoryType() for p in prototypes])) > 1: - print('Different types of prototypes in database ({}).'.format(set([p.getTrajectoryType() for p in prototypes]))) - return prototypes - -def savePOIsToSqlite(filename, gmm, gmmType, gmmId): - '''Saves a Gaussian mixture model (of class sklearn.mixture.GaussianMixture) - gmmType is a type of GaussianMixture, learnt either from beginnings or ends of trajectories''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - if gmmType not in ['beginning', 'end']: - print('Unknown POI type {}. Exiting'.format(gmmType)) - import sys - sys.exit() - try: - cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))') - for i in range(gmm.n_components): - cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist()))) - connection.commit() - except sqlite3.OperationalError as error: - printDBError(error) - -def savePOIAssignmentsToSqlite(filename, objects): - 'save the od fields of objects' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('CREATE TABLE IF NOT EXISTS objects_pois (object_id INTEGER, origin_poi_id INTEGER, destination_poi_id INTEGER, PRIMARY KEY(object_id))') - for o in objects: - cursor.execute('INSERT INTO objects_pois VALUES(?,?,?)', (o.getNum(), o.od[0], o.od[1])) - connection.commit() - except sqlite3.OperationalError as error: - printDBError(error) - -def loadPOIsFromSqlite(filename): - 'Loads all 2D Gaussians in the database' - from sklearn import mixture # todo if not avalaible, load data in duck-typed class with same fields - from ast import literal_eval - pois = [] - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - try: - cursor.execute('SELECT * from gaussians2d') - gmmId = None - gmm = [] - for row in cursor: - if gmmId is None or row[0] != gmmId: - if len(gmm) > 0: - tmp = mixture.GaussianMixture(len(gmm), covarianceType) - tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) - tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) - tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) - tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] - tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) - pois.append(tmp) - gaussian = {'type': row[2], - 'mean': row[3:5], - 'covar': array(literal_eval(row[5])), - 'weight': row[7], - 'precisions': array(literal_eval(row[8]))} - gmm = [gaussian] - covarianceType = row[6] - gmmId = row[0] - else: - gmm.append({'type': row[2], - 'mean': row[3:5], - 'covar': array(literal_eval(row[5])), - 'weight': row[7], - 'precisions': array(literal_eval(row[8]))}) - if len(gmm) > 0: - tmp = mixture.GaussianMixture(len(gmm), covarianceType) - tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) - tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) - tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) - tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] - tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) - pois.append(tmp) - except sqlite3.OperationalError as error: - printDBError(error) - return pois - -######################### -# saving and loading for scene interpretation (Mohamed Gomaa Mohamed's PhD) -######################### - -def writePrototypesToSqlite(prototypes,nMatching, outputFilename): - ''' prototype dataset is a dictionary with keys== routes, values== prototypes Ids ''' - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))') - - for route in prototypes: - if prototypes[route]!=[]: - for i in prototypes[route]: - cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i])) - - connection.commit() - connection.close() - -def readPrototypesFromSqlite(filename): - ''' - This function loads the prototype file in the database - It returns a dictionary for prototypes for each route and nMatching - ''' - prototypes = {} - nMatching={} - - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from prototypes order by prototype_id, routeIDstart,routeIDend, nMatching') - except sqlite3.OperationalError as error: - printDBError(error) - return [] - - for row in cursor: - route=(row[1],row[2]) - if route not in prototypes: - prototypes[route]=[] - prototypes[route].append(row[0]) - nMatching[row[0]]=row[3] - - connection.close() - return prototypes,nMatching - -def writeLabelsToSqlite(labels, outputFilename): - """ labels is a dictionary with keys: routes, values: prototypes Ids - """ - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") - - for route in labels: - if labels[route]!=[]: - for i in labels[route]: - for j in labels[route][i]: - cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) - - connection.commit() - connection.close() - -def loadLabelsFromSqlite(filename): - labels = {} - - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from labels order by object_id, routeIDstart,routeIDend, prototype_id') - except sqlite3.OperationalError as error: - printDBError(error) - return [] - - for row in cursor: - route=(row[1],row[2]) - p=row[3] - if route not in labels: - labels[route]={} - if p not in labels[route]: - labels[route][p]=[] - labels[route][p].append(row[0]) - - connection.close() - return labels - -def writeSpeedPrototypeToSqlite(prototypes,nmatching, outFilename): - """ to match the format of second layer prototypes""" - connection = sqlite3.connect(outFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") - - for route in prototypes: - if prototypes[route]!={}: - for i in prototypes[route]: - if prototypes[route][i]!= []: - for j in prototypes[route][i]: - cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) - - connection.commit() - connection.close() - -def loadSpeedPrototypeFromSqlite(filename): - """ - This function loads the prototypes table in the database of name <filename>. - """ - prototypes = {} - nMatching={} - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from speedprototypes order by spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching') - except sqlite3.OperationalError as error: - printDBError(error) - return [] - - for row in cursor: - route=(row[2],row[3]) - if route not in prototypes: - prototypes[route]={} - if row[1] not in prototypes[route]: - prototypes[route][row[1]]=[] - prototypes[route][row[1]].append(row[0]) - nMatching[row[0]]=row[4] - - connection.close() - return prototypes,nMatching - - -def writeRoutesToSqlite(Routes, outputFilename): - """ This function writes the activity path define by start and end IDs""" - connection = sqlite3.connect(outputFilename) - cursor = connection.cursor() - - cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") - - for route in Routes: - if Routes[route]!=[]: - for i in Routes[route]: - cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) - - connection.commit() - connection.close() - -def loadRoutesFromSqlite(filename): - Routes = {} - - connection = sqlite3.connect(filename) - cursor = connection.cursor() - - try: - cursor.execute('SELECT * from routes order by object_id, routeIDstart,routeIDend') - except sqlite3.OperationalError as error: - printDBError(error) - return [] - - for row in cursor: - route=(row[1],row[2]) - if route not in Routes: - Routes[route]=[] - Routes[route].append(row[0]) - - connection.close() - return Routes - -def setRoutes(filename, objects): - connection = sqlite3.connect(filename) - cursor = connection.cursor() - for obj in objects: - cursor.execute('update objects set startRouteID = {} WHERE object_id = {}'.format(obj.startRouteID, obj.getNum())) - cursor.execute('update objects set endRouteID = {} WHERE object_id = {}'.format(obj.endRouteID, obj.getNum())) - connection.commit() - connection.close() - -######################### -# txt files -######################### - -def openCheck(filename, option = 'r', quitting = False): - '''Open file filename in read mode by default - and checks it is open''' - try: - return open(filename, option) - except IOError: - print('File {} could not be opened.'.format(filename)) - if quitting: - from sys import exit - exit() - return None - -def readline(f, commentCharacters = commentChar): - '''Modified readline function to skip comments - Can take a list of characters or a string (in will work in both)''' - s = f.readline() - while (len(s) > 0) and s[0] in commentCharacters: - s = f.readline() - return s.strip() - -def getLines(f, delimiterChar = delimiterChar, commentCharacters = commentChar): - '''Gets a complete entry (all the lines) in between delimiterChar.''' - dataStrings = [] - s = readline(f, commentCharacters) - while len(s) > 0 and s[0] != delimiterChar: - dataStrings += [s.strip()] - s = readline(f, commentCharacters) - return dataStrings - -def saveList(filename, l): - f = openCheck(filename, 'w') - for x in l: - f.write('{}\n'.format(x)) - f.close() - -def loadListStrings(filename, commentCharacters = commentChar): - f = openCheck(filename, 'r') - result = getLines(f, commentCharacters) - f.close() - return result - -def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar): - values = [] - for l in loadListStrings(filename, commentCharacters): - if l.startswith(option): - values.append(l.split(delimiterChar)[1].strip()) - return values - -def addSectionHeader(propertiesFile, headerName = 'main'): - '''Add fake section header - - from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788 - use read_file in Python 3.2+ - ''' - yield '[{}]\n'.format(headerName) - for line in propertiesFile: - yield line - -def loadPemsTraffic(filename): - '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse - into pandas dataframe''' - f = openCheck(filename) - l = f.readline().strip() - items = l.split(',') - headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] - nLanes = (len(items)-len(headers))/3 - for i in range(nLanes): - headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)] - f.close() - return read_csv(filename, delimiter = ',', names = headers) - -def generatePDLaneColumn(data): - data['LANE'] = data['LANE\\LINK\\NO'].astype(str)+'_'+data['LANE\\INDEX'].astype(str) - -def convertTrajectoriesVissimToSqlite(filename): - '''Relies on a system call to sqlite3 - sqlite3 [file.sqlite] < import_fzp.sql''' - sqlScriptFilename = "import_fzp.sql" - # create sql file - out = openCheck(sqlScriptFilename, "w") - out.write(".separator \";\"\n"+ - "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+ - ".import "+filename+" curvilinear_positions\n"+ - "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") - out.close() - # system call - from subprocess import run - out = openCheck("err.log", "w") - run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) - out.close() - shutil.os.remove(sqlScriptFilename) - -def loadObjectNumbersInLinkFromVissimFile(filename, linkIds): - '''Finds the ids of the objects that go through any of the link in the list linkIds''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - queryStatement = 'SELECT DISTINCT trajectory_id FROM curvilinear_positions where link_id IN ('+','.join([str(id) for id in linkIds])+')' - try: - cursor.execute(queryStatement) - return [row[0] for row in cursor] - except sqlite3.OperationalError as error: - printDBError(error) - -def getNObjectsInLinkFromVissimFile(filename, linkIds): - '''Returns the number of objects that traveled through the link ids''' - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - queryStatement = 'SELECT link_id, COUNT(DISTINCT trajectory_id) FROM curvilinear_positions where link_id IN ('+','.join([str(id) for id in linkIds])+') GROUP BY link_id' - try: - cursor.execute(queryStatement) - return {row[0]:row[1] for row in cursor} - except sqlite3.OperationalError as error: - printDBError(error) - -def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, objectNumbers = None, warmUpLastInstant = None, usePandas = False, nDecimals = 2, lowMemory = True): - '''Reads data from VISSIM .fzp trajectory file - simulationStepsPerTimeUnit is the number of simulation steps per unit of time used by VISSIM (second) - for example, there seems to be 10 simulation steps per simulated second in VISSIM, - so simulationStepsPerTimeUnit should be 10, - so that all times correspond to the number of the simulation step (and can be stored as integers) - - Objects positions will be considered only after warmUpLastInstant - (if the object has no such position, it won't be loaded) - - Assumed to be sorted over time - Warning: if reading from SQLite a limited number of objects, objectNumbers will be the maximum object id''' - objects = {} # dictionary of objects index by their id - - if usePandas: - data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory) - generatePDLaneColumn(data) - data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit - if warmUpLastInstant is not None: - data = data[data['TIME']>=warmUpLastInstant] - grouped = data.loc[:,['NO','TIME']].groupby(['NO'], as_index = False) - instants = grouped['TIME'].agg({'first': npmin, 'last': npmax}) - for row_index, row in instants.iterrows(): - objNum = int(row['NO']) - tmp = data[data['NO'] == objNum] - objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last'])) - # positions should be rounded to nDecimals decimals only - objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) - if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: - return list(objects.values()) - else: - if filename.endswith(".fzp"): - inputfile = openCheck(filename, quitting = True) - line = readline(inputfile, '*$') - while len(line) > 0:#for line in inputfile: - data = line.strip().split(';') - objNum = int(data[1]) - instant = float(data[0])*simulationStepsPerTimeUnit - s = float(data[4]) - y = float(data[5]) - lane = data[2]+'_'+data[3] - if objNum not in objects: - if warmUpLastInstant is None or instant >= warmUpLastInstant: - if objectNumbers is None or len(objects) < objectNumbers: - objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) - objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() - if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: - objects[objNum].timeInterval.last = instant - objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) - line = readline(inputfile, '*$') - elif filename.endswith(".sqlite"): - with sqlite3.connect(filename) as connection: - cursor = connection.cursor() - queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions' - if objectNumbers is not None: - queryStatement += ' WHERE trajectory_id '+getObjectCriteria(objectNumbers) - queryStatement += ' ORDER BY trajectory_id, t' - try: - cursor.execute(queryStatement) - for row in cursor: - objNum = row[1] - instant = row[0]*simulationStepsPerTimeUnit - s = row[4] - y = row[5] - lane = '{}_{}'.format(row[2], row[3]) - if objNum not in objects: - if warmUpLastInstant is None or instant >= warmUpLastInstant: - if objectNumbers is None or len(objects) < objectNumbers: - objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) - objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() - if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: - objects[objNum].timeInterval.last = instant - objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) - except sqlite3.OperationalError as error: - printDBError(error) - else: - print("File type of "+filename+" not supported (only .sqlite and .fzp files)") - return list(objects.values()) - -def selectPDLanes(data, lanes = None): - '''Selects the subset of data for the right lanes - - Lane format is a string 'x_y' where x is link index and y is lane index''' - if lanes is not None: - if 'LANE' not in data.columns: - generatePDLaneColumn(data) - indices = (data['LANE'] == lanes[0]) - for l in lanes[1:]: - indices = indices | (data['LANE'] == l) - return data[indices] - else: - return data - -def countStoppedVehiclesVissim(filename, lanes = None, proportionStationaryTime = 0.7): - '''Counts the number of vehicles stopped for a long time in a VISSIM trajectory file - and the total number of vehicles - - Vehicles are considered finally stationary - if more than proportionStationaryTime of their total time - If lanes is not None, only the data for the selected lanes will be provided - (format as string x_y where x is link index and y is lane index)''' - if filename.endswith(".fzp"): - columns = ['NO', '$VEHICLE:SIMSEC', 'POS'] - if lanes is not None: - columns += ['LANE\\LINK\\NO', 'LANE\\INDEX'] - data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory) - data = selectPDLanes(data, lanes) - data.sort(['$VEHICLE:SIMSEC'], inplace = True) - - nStationary = 0 - nVehicles = 0 - for name, group in data.groupby(['NO'], sort = False): - nVehicles += 1 - positions = array(group['POS']) - diff = positions[1:]-positions[:-1] - if npsum(diff == 0.) >= proportionStationaryTime*(len(positions)-1): - nStationary += 1 - elif filename.endswith(".sqlite"): - # select trajectory_id, t, s_coordinate, speed from curvilinear_positions where trajectory_id between 1860 and 1870 and speed < 0.1 - # pb of the meaning of proportionStationaryTime in arterial network? Why proportion of existence time? - pass - else: - print("File type of "+filename+" not supported (only .sqlite and .fzp files)") - - return nStationary, nVehicles - -def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2, lowMemory = True): - '''Counts the number of collisions per lane in a VISSIM trajectory file - - To distinguish between cars passing and collision, - one checks when the sign of the position difference inverts - (if the time are closer than collisionTimeDifference) - If lanes is not None, only the data for the selected lanes will be provided - (format as string x_y where x is link index and y is lane index)''' - data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory) - data = selectPDLanes(data, lanes) - data = data.convert_objects(convert_numeric=True) - - merged = merge(data, data, how='inner', left_on=['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC'], sort = False) - merged = merged[merged['NO_x']>merged['NO_y']] - - nCollisions = 0 - for name, group in merged.groupby(['LANE\\LINK\\NO', 'LANE\\INDEX', 'NO_x', 'NO_y']): - diff = group['POS_x']-group['POS_y'] - # diff = group['POS_x']-group['POS_y'] # to check the impact of convert_objects and the possibility of using type conversion in read_csv or function to convert strings if any - if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0: - xidx = diff[diff < 0].argmax() - yidx = diff[diff > 0].argmin() - if abs(group.loc[xidx, '$VEHICLE:SIMSEC'] - group.loc[yidx, '$VEHICLE:SIMSEC']) <= collisionTimeDifference: - nCollisions += 1 - - # select TD1.link_id, TD1.lane_id from temp.diff_positions as TD1, temp.diff_positions as TD2 where TD1.link_id = TD2.link_id and TD1.lane_id = TD2.lane_id and TD1.id1 = TD2.id1 and TD1.id2 = TD2.id2 and TD1.t = TD2.t+0.1 and TD1.diff*TD2.diff < 0; # besoin de faire un group by?? - # create temp table diff_positions as select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id; - # SQL select link_id, lane_id, id1, id2, min(diff), max(diff) from (select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id) group by link_id, lane_id, id1, id2 having min(diff)*max(diff) < 0 - return nCollisions - -def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): - '''Reads data from the trajectory data provided by NGSIM project - and returns the list of Feature objects''' - objects = [] - - inputfile = openCheck(filename, quitting = True) - - def createObject(numbers): - firstFrameNum = int(numbers[1]) - # do the geometry and usertype - - firstFrameNum = int(numbers[1]) - lastFrameNum = firstFrameNum+int(numbers[2])-1 - #time = moving.TimeInterval(firstFrameNum, firstFrameNum+int(numbers[2])-1) - obj = moving.MovingObject(num = int(numbers[0]), - timeInterval = moving.TimeInterval(firstFrameNum, lastFrameNum), - positions = moving.Trajectory([[float(numbers[6])],[float(numbers[7])]]), - userType = int(numbers[10])) - obj.userType = int(numbers[10]) - obj.laneNums = [int(numbers[13])] - obj.precedingVehicles = [int(numbers[14])] # lead vehicle (before) - obj.followingVehicles = [int(numbers[15])] # following vehicle (after) - obj.spaceHeadways = [float(numbers[16])] # feet - obj.timeHeadways = [float(numbers[17])] # seconds - obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate - obj.speeds = [float(numbers[11])] - obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object - return obj - - numbers = readline(inputfile).strip().split() - if (len(numbers) > 0): - obj = createObject(numbers) - - for line in inputfile: - numbers = line.strip().split() - if obj.getNum() != int(numbers[0]): - # check and adapt the length to deal with issues in NGSIM data - if (obj.length() != obj.positions.length()): - print('length pb with object {} ({},{})'.format(obj.getNum(),obj.length(),obj.positions.length())) - obj.last = obj.getFirstInstant()+obj.positions.length()-1 - #obj.velocities = utils.computeVelocities(f.positions) # compare norm to speeds ? - objects.append(obj) - if (nObjects>0) and (len(objects)>=nObjects): - break - obj = createObject(numbers) - else: - obj.laneNums.append(int(numbers[13])) - obj.positions.addPositionXY(float(numbers[6]), float(numbers[7])) - obj.curvilinearPositions.addPositionSYL(float(numbers[5]), float(numbers[4]), obj.laneNums[-1]) - obj.speeds.append(float(numbers[11])) - obj.precedingVehicles.append(int(numbers[14])) - obj.followingVehicles.append(int(numbers[15])) - obj.spaceHeadways.append(float(numbers[16])) - obj.timeHeadways.append(float(numbers[17])) - - if (obj.size[0] != float(numbers[8])): - print('changed length obj {}'.format(obj.getNum())) - if (obj.size[1] != float(numbers[9])): - print('changed width obj {}'.format(obj.getNum())) - - inputfile.close() - return objects - -def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): - '''Reads data from the trajectory data provided by NGSIM project - and converts to our current format.''' - if append: - out = openCheck(outputfile,'a') - else: - out = openCheck(outputfile,'w') - nObjectsPerType = [0,0,0] - - features = loadNgsimFile(inputfile, sequenceNum) - for f in features: - nObjectsPerType[f.userType-1] += 1 - f.write(out) - - print(nObjectsPerType) - - out.close() - -def loadPinholeCameraModel(filename, tanalystFormat = True): - '''Loads the data from a file containing the camera parameters - (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) - and returns a dictionary''' - if tanalystFormat: - f = openCheck(filename, quitting = True) - content = getLines(f) - cameraData = {} - for l in content: - tmp = l.split(':') - cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.')) - return cameraData - else: - print('Unknown camera model (not tanalyst format') - return None - -def savePositionsToCsv(f, obj): - timeInterval = obj.getTimeInterval() - positions = obj.getPositions() - curvilinearPositions = obj.getCurvilinearPositions() - for i in range(int(obj.length())): - p1 = positions[i] - s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y) - if curvilinearPositions is not None: - p2 = curvilinearPositions[i] - s += ',{},{}'.format(p2[0],p2[1]) - f.write(s+'\n') - -def saveTrajectoriesToCsv(filename, objects): - f = openCheck(filename, 'w') - for i,obj in enumerate(objects): - savePositionsToCsv(f, obj) - f.close() - - -######################### -# Utils to read .ini type text files for configuration, meta data... -######################### - -class ClassifierParameters(VideoFilenameAddable): - 'Class for the parameters of object classifiers' - def loadConfigFile(self, filename): - from configparser import ConfigParser - - config = ConfigParser() - config.read_file(addSectionHeader(openCheck(filename))) - - parentPath = Path(filename).parent - self.sectionHeader = config.sections()[0] - - self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename')) - self.bikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'bv-svm-filename')) - self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop') - self.minNPixels = config.getint(self.sectionHeader, 'min-npixels-crop') - x = config.getint(self.sectionHeader, 'hog-rescale-size') - self.hogRescaleSize = (x, x) - self.hogNOrientations = config.getint(self.sectionHeader, 'hog-norientations') - x = config.getint(self.sectionHeader, 'hog-npixels-cell') - self.hogNPixelsPerCell = (x, x) - x = config.getint(self.sectionHeader, 'hog-ncells-block') - self.hogNCellsPerBlock = (x, x) - self.hogBlockNorm = config.get(self.sectionHeader, 'hog-block-norm') - - self.speedAggregationMethod = config.get(self.sectionHeader, 'speed-aggregation-method') - self.nFramesIgnoreAtEnds = config.getint(self.sectionHeader, 'nframes-ignore-at-ends') - self.speedAggregationCentile = config.getint(self.sectionHeader, 'speed-aggregation-centile') - self.minSpeedEquiprobable = config.getfloat(self.sectionHeader, 'min-speed-equiprobable') - self.maxPercentUnknown = config.getfloat(self.sectionHeader, 'max-prop-unknown-appearance') - self.maxPedestrianSpeed = config.getfloat(self.sectionHeader, 'max-ped-speed') - self.maxCyclistSpeed = config.getfloat(self.sectionHeader, 'max-cyc-speed') - self.meanPedestrianSpeed = config.getfloat(self.sectionHeader, 'mean-ped-speed') - self.stdPedestrianSpeed = config.getfloat(self.sectionHeader, 'std-ped-speed') - self.locationCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-loc') - self.scaleCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-scale') - self.meanVehicleSpeed = config.getfloat(self.sectionHeader, 'mean-veh-speed') - self.stdVehicleSpeed = config.getfloat(self.sectionHeader, 'std-veh-speed') - - def __init__(self, filename = None): - if filename is not None and Path(filename).exists(): - self.loadConfigFile(filename) - else: - print('Configuration filename {} could not be loaded.'.format(filename)) - - def convertToFrames(self, frameRate, speedRatio = 3.6): - '''Converts parameters with a relationship to time in 'native' frame time - speedRatio is the conversion from the speed unit in the config file - to the distance per second - - ie param(config file) = speedRatio x fps x param(used in program) - eg km/h = 3.6 (m/s to km/h) x frame/s x m/frame''' - denominator = frameRate*speedRatio - #denominator2 = denominator**2 - self.minSpeedEquiprobable = self.minSpeedEquiprobable/denominator - self.maxPedestrianSpeed = self.maxPedestrianSpeed/denominator - self.maxCyclistSpeed = self.maxCyclistSpeed/denominator - self.meanPedestrianSpeed = self.meanPedestrianSpeed/denominator - self.stdPedestrianSpeed = self.stdPedestrianSpeed/denominator - self.meanVehicleSpeed = self.meanVehicleSpeed/denominator - self.stdVehicleSpeed = self.stdVehicleSpeed/denominator - # special case for the lognormal distribution - self.locationCyclistSpeed = self.locationCyclistSpeed-log(denominator) - #self.scaleCyclistSpeed = self.scaleCyclistSpeed # no modification of scale - - -class ProcessParameters(VideoFilenameAddable): - '''Class for all parameters controlling data processing: input, - method parameters, etc. for tracking and safety - - Note: framerate is already taken into account''' - - def loadConfigFile(self, filename): - from configparser import ConfigParser - - config = ConfigParser(strict=False) - config.read_file(addSectionHeader(openCheck(filename))) - - parentPath = Path(filename).parent - self.sectionHeader = config.sections()[0] - # Tracking/display parameters - self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) - self.databaseFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'database-filename')) - self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename')) - if Path(self.homographyFilename).exists(): - self.homography = loadtxt(self.homographyFilename) - else: - self.homography = None - self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename')) - if Path(self.intrinsicCameraFilename).exists(): - self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename) - else: - self.intrinsicCameraMatrix = None - distortionCoefficients = getValuesFromINIFile(filename, 'distortion-coefficients', '=') - self.distortionCoefficients = [float(x) for x in distortionCoefficients] - self.undistortedImageMultiplication = config.getfloat(self.sectionHeader, 'undistorted-size-multiplication') - self.undistort = config.getboolean(self.sectionHeader, 'undistort') - self.firstFrameNum = config.getint(self.sectionHeader, 'frame1') - self.videoFrameRate = config.getfloat(self.sectionHeader, 'video-fps') - - self.minFeatureTime = config.getfloat(self.sectionHeader, 'min-feature-time') - - self.classifierFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'classifier-filename')) - - # Safety parameters - self.maxPredictedSpeed = config.getfloat(self.sectionHeader, 'max-predicted-speed')/3.6/self.videoFrameRate - self.predictionTimeHorizon = config.getfloat(self.sectionHeader, 'prediction-time-horizon')*self.videoFrameRate - self.collisionDistance = config.getfloat(self.sectionHeader, 'collision-distance') - self.crossingZones = config.getboolean(self.sectionHeader, 'crossing-zones') - self.predictionMethod = config.get(self.sectionHeader, 'prediction-method') - self.nPredictedTrajectories = config.getint(self.sectionHeader, 'npredicted-trajectories') - self.maxNormalAcceleration = config.getfloat(self.sectionHeader, 'max-normal-acceleration')/self.videoFrameRate**2 - self.maxNormalSteering = config.getfloat(self.sectionHeader, 'max-normal-steering')/self.videoFrameRate - self.minExtremeAcceleration = config.getfloat(self.sectionHeader, 'min-extreme-acceleration')/self.videoFrameRate**2 - self.maxExtremeAcceleration = config.getfloat(self.sectionHeader, 'max-extreme-acceleration')/self.videoFrameRate**2 - self.maxExtremeSteering = config.getfloat(self.sectionHeader, 'max-extreme-steering')/self.videoFrameRate - self.useFeaturesForPrediction = config.getboolean(self.sectionHeader, 'use-features-prediction') - self.constantSpeedPrototypePrediction = config.getboolean(self.sectionHeader, 'constant-speed') - self.maxLcssDistance = config.getfloat(self.sectionHeader, 'max-lcss-distance') - self.lcssMetric = config.get(self.sectionHeader, 'lcss-metric') - self.minLcssSimilarity = config.getfloat(self.sectionHeader, 'min-lcss-similarity') - - def __init__(self, filename = None): - if filename is not None and Path(filename).exists(): - self.loadConfigFile(filename) - else: - print('Configuration filename {} could not be loaded.'.format(filename)) - -def processVideoArguments(args): - '''Loads information from configuration file - then checks what was passed on the command line - for override (eg video filename and database filename''' - parentPath = Path(args.configFilename).parent - if args.configFilename is not None: # consider there is a configuration file - params = ProcessParameters(args.configFilename) - videoFilename = params.videoFilename - databaseFilename = params.databaseFilename - if params.homography is not None: - invHomography = linalg.inv(params.homography) - else: - invHomography = None - intrinsicCameraMatrix = params.intrinsicCameraMatrix - distortionCoefficients = array(params.distortionCoefficients) - undistortedImageMultiplication = params.undistortedImageMultiplication - undistort = params.undistort - firstFrameNum = params.firstFrameNum - else: - invHomography = None - undistort = False - intrinsicCameraMatrix = None - distortionCoefficients = [] - undistortedImageMultiplication = None - undistort = False - firstFrameNum = 0 - - # override video and database filenames if present on command line - # if not absolute, make all filenames relative to the location of the configuration filename - if args.videoFilename is not None: - videoFilename = args.videoFilename - else: - videoFilename = params.videoFilename - if args.databaseFilename is not None: - databaseFilename = args.databaseFilename - else: - databaseFilename = params.databaseFilename - - return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum - -# deprecated -class SceneParameters(object): - def __init__(self, config, sectionName): - from configparser import NoOptionError - from ast import literal_eval - try: - self.sitename = config.get(sectionName, 'sitename') - self.databaseFilename = config.get(sectionName, 'data-filename') - self.homographyFilename = config.get(sectionName, 'homography-filename') - self.calibrationFilename = config.get(sectionName, 'calibration-filename') - self.videoFilename = config.get(sectionName, 'video-filename') - self.frameRate = config.getfloat(sectionName, 'framerate') - self.date = datetime.strptime(config.get(sectionName, 'date'), datetimeFormat) # 2011-06-22 11:00:39 - self.translation = literal_eval(config.get(sectionName, 'translation')) # = [0.0, 0.0] - self.rotation = config.getfloat(sectionName, 'rotation') - self.duration = config.getint(sectionName, 'duration') - except NoOptionError as e: - print(e) - print('Not a section for scene meta-data') - - @staticmethod - def loadConfigFile(filename): - from configparser import ConfigParser - config = ConfigParser() - config.readfp(openCheck(filename)) - configDict = dict() - for sectionName in config.sections(): - configDict[sectionName] = SceneParameters(config, sectionName) - return configDict - - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/storage.txt') - unittest.TextTestRunner().run(suite) -# #doctest.testmod() -# #doctest.testfile("example.txt")
--- a/python/sumo.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -#! /usr/bin/env python -'''Libraries for the SUMO traffic simulation software -http://sumo.dlr.de -''' - -#import csv - -def loadTazEdges(inFilename): - '''Converts list of OSM edges per OSM edge and groups per TAZ - format is csv with first two columns the OSM id and TAZ id, then the list of SUMO edge id - - Returns the list of SUMO edge per TAZ''' - data = [] - tazs = {} - with open(inFilename,'r') as f: - f.readline() # skip the headers - for r in f: - tmp = r.strip().split(',') - tazID = tmp[1] - for edge in tmp[2:]: - if len(edge) > 0: - if tazID in tazs: - if edge not in tazs[tazID]: - tazs[tazID].append(edge) - else: - tazs[tazID] = [edge] - return tazs - -def edge2Taz(tazs): - '''Returns the associative array of the TAZ of each SUMO edge''' - edge2Tazs = {} - for taz, edges in tazs.items(): - for edge in edges: - if edge in edge2Tazs: - print('error for edge: {} (taz {}/{})'.format(edge, edge2Tazs[edge], taz)) - edge2Tazs[edge] = taz - return edge2Tazs - -def saveTazEdges(outFilename, tazs): - with open(outFilename,'w') as out: - out.write('<tazs>\n') - for tazID in tazs: - out.write('<taz id="{}" edges="'.format(tazID)+' '.join(tazs[tazID])+'"/>\n') - out.write('</tazs>\n') - -# TODO add utils from process-cyber.py? - -# if __name__ == "__main__": -# import doctest -# import unittest -# suite = doctest.DocFileSuite('tests/sumo.txt') -# #suite = doctest.DocTestSuite() -# unittest.TextTestRunner().run(suite)
--- a/python/tests/cvutils.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,46 +0,0 @@ ->>> import cv2, cvutils ->>> from numpy import array, round, ones, dot, linalg, absolute ->>> img = cv2.imread("../samples/val-dor-117-111.png") ->>> width = img.shape[1] ->>> height = img.shape[0] ->>> intrinsicCameraMatrix = array([[ 377.42, 0. , 639.12], [ 0. , 378.43, 490.2 ], [ 0. , 0. , 1. ]]) ->>> distortionCoefficients = array([-0.11759321, 0.0148536, 0.00030756, -0.00020578, -0.00091816])# distortionCoefficients = array([-0.11759321, 0., 0., 0., 0.]) ->>> multiplicationFactor = 1.31 ->>> [map1, map2], tmp = cvutils.computeUndistortMaps(width, height, multiplicationFactor, intrinsicCameraMatrix, distortionCoefficients) ->>> undistorted = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR) ->>> (undistorted.shape == array([int(round(height*multiplicationFactor)), int(round(width*multiplicationFactor)), 3])).all() -True ->>> imgPoints = array([[[150.,170.],[220.,340.],[340.,440.],[401.,521.]]]) ->>> newCameraMatrix = cv2.getDefaultNewCameraMatrix(intrinsicCameraMatrix, (int(round(width*multiplicationFactor)), int(round(height*multiplicationFactor))), True) ->>> undistortedPoints = cv2.undistortPoints(imgPoints, intrinsicCameraMatrix, distortionCoefficients, P = newCameraMatrix).reshape(-1, 2) # undistort and project as if seen by new camera ->>> invNewCameraMatrix = linalg.inv(newCameraMatrix) ->>> tmp = ones((imgPoints[0].shape[0], 3)) ->>> tmp[:,:2] = undistortedPoints ->>> reducedPoints = dot(invNewCameraMatrix, tmp.T).T ->>> origPoints = cv2.projectPoints(reducedPoints, (0.,0.,0.), (0.,0.,0.), intrinsicCameraMatrix, distortionCoefficients)[0].reshape(-1,2) ->>> (round(origPoints[1:,:]) == imgPoints[0][1:,:]).all() -True ->>> (absolute(origPoints[0,:]-imgPoints[0][0,:])).max() < 6. -True ->>> reducedPoints2 = cvutils.newCameraProject(undistortedPoints.T, invNewCameraMatrix) ->>> (reducedPoints == reducedPoints).all() -True - ->>> undistortedPoints2 = cv2.undistortPoints(imgPoints, intrinsicCameraMatrix, distortionCoefficients).reshape(-1, 2) # undistort and project as if seen by new camera ->>> undistortedPoints2 = cvutils.newCameraProject(undistortedPoints2.T, newCameraMatrix) ->>> (undistortedPoints == undistortedPoints2.T).all() -True - ->>> undistortedPoints = cv2.undistortPoints(imgPoints, intrinsicCameraMatrix, distortionCoefficients).reshape(-1, 2) # undistort to ideal points ->>> origPoints = cvutils.worldToImageProject(undistortedPoints.T, intrinsicCameraMatrix, distortionCoefficients).T ->>> (round(origPoints[1:,:]) == imgPoints[0][1:,:]).all() -True ->>> (absolute(origPoints[0,:]-imgPoints[0][0,:])).max() < 6. -True - ->>> undistortedPoints = cvutils.imageToWorldProject(imgPoints[0].T, intrinsicCameraMatrix, distortionCoefficients) ->>> origPoints = cvutils.worldToImageProject(undistortedPoints, intrinsicCameraMatrix, distortionCoefficients).T ->>> (round(origPoints[1:,:]) == imgPoints[0][1:,:]).all() -True ->>> (absolute(origPoints[0,:]-imgPoints[0][0,:])).max() < 6. -True
--- a/python/tests/events.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ ->>> from events import * ->>> from moving import MovingObject, TimeInterval, Point ->>> from prediction import ConstantPredictionParameters - ->>> objects = [MovingObject(num = i, timeInterval = TimeInterval(0,10)) for i in range(10)] ->>> interactions = createInteractions(objects) ->>> len([i for i in interactions if len(i.roadUserNumbers) == 1]) -0 ->>> objects2 = [MovingObject(num = i, timeInterval = TimeInterval(0,10)) for i in range(100, 110)] ->>> interactions = createInteractions(objects, objects2) ->>> len([i for i in interactions if len(i.roadUserNumbers) == 1]) -0 - ->>> o1 = MovingObject.generate(1, Point(-5.,0.), Point(1.,0.), TimeInterval(0,10)) ->>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(0,10)) ->>> inter = Interaction(roadUser1 = o1, roadUser2 = o2) ->>> inter.computeIndicators() ->>> predictionParams = ConstantPredictionParameters(10.) ->>> inter.computeCrossingsCollisions(predictionParams, 0.1, 10) ->>> ttc = inter.getIndicator("Time to Collision") ->>> ttc[0] -5.0 ->>> ttc[1] -4.0 ->>> (inter.collisionPoints[0][0] - Point(0.,0.)).norm2() < 0.0001 -True ->>> (inter.collisionPoints[4][0] - Point(0.,0.)).norm2() < 0.0001 -True ->>> inter.getIndicator(Interaction.indicatorNames[1])[4] < 0.000001 # collision angle -True ->>> inter.getIndicator(Interaction.indicatorNames[1])[5] is None -True ->>> inter.getIndicator(Interaction.indicatorNames[1])[6] # doctest:+ELLIPSIS -3.1415...
--- a/python/tests/indicators.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ ->>> from indicators import * ->>> from moving import TimeInterval,Trajectory - ->>> indic1 = TemporalIndicator('bla', [0,3,-4], TimeInterval(4,6)) ->>> indic1.empty() -False ->>> indic1.getIthValue(1) -3 ->>> indic1.getIthValue(3) ->>> indic1[6] --4 ->>> indic1[7] ->>> [v for v in indic1] -[0, 3, -4] ->>> indic1 = TemporalIndicator('bla', {2:0,4:3,5:-5}) ->>> indic1.getIthValue(1) -3 ->>> indic1.getIthValue(3) ->>> indic1[2] -0 - ->>> t1 = Trajectory([[0.5,1.5,2.5],[0.5,3.5,6.5]]) ->>> m = indicatorMap([1,2,3], t1, 1) ->>> m[(1.0, 3.0)] -2.0 ->>> m[(2.0, 6.0)] -3.0 ->>> m[(0.0, 0.0)] -1.0 ->>> m = indicatorMap([1,2,3], t1, 4) ->>> m[(0.0, 1.0)] -3.0 ->>> m[(0.0, 0.0)] -1.5
--- a/python/tests/ml.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ ->>> from math import fabs ->>> from numpy import ones ->>> from ml import prototypeCluster - ->>> nTrajectories = 7 ->>> similarityFunc = lambda x, y: 1.-fabs(x-y)/(nTrajectories-1) ->>> similarities = -ones((nTrajectories, nTrajectories)) ->>> prototypeIndices = prototypeCluster(range(nTrajectories), similarities, 1., similarityFunc, optimizeCentroid = True) # too large to be similar ->>> len(prototypeIndices) == nTrajectories -True ->>> # could use lists to have a length
--- a/python/tests/moving.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,268 +0,0 @@ ->>> from moving import * ->>> import storage ->>> import numpy as np - ->>> Interval().empty() -True ->>> Interval(0,1).empty() -False ->>> Interval(0,1) -[0, 1] ->>> Interval(0,1).length() -1.0 ->>> Interval(23.2,24.9).length() -1.6999999999999993 ->>> Interval(10,8).length() -0.0 - ->>> TimeInterval(0,1).length() -2.0 ->>> TimeInterval(10,8).length() -0.0 ->>> TimeInterval(10,8) == TimeInterval(10,8) -True ->>> TimeInterval(10,8) == TimeInterval(8,10) -True ->>> TimeInterval(11,8) == TimeInterval(10,8) -False - ->>> [i for i in TimeInterval(9,13)] -[9, 10, 11, 12, 13] - ->>> TimeInterval(2,5).equal(TimeInterval(2,5)) -True ->>> TimeInterval(2,5).equal(TimeInterval(2,4)) -False ->>> TimeInterval(2,5).equal(TimeInterval(5,2)) -False - ->>> TimeInterval(3,6).distance(TimeInterval(4,6)) -0 ->>> TimeInterval(3,6).distance(TimeInterval(6,10)) -0 ->>> TimeInterval(3,6).distance(TimeInterval(8,10)) -2 ->>> TimeInterval(20,30).distance(TimeInterval(3,15)) -5 ->>> TimeInterval.unionIntervals([TimeInterval(3,6), TimeInterval(8,10),TimeInterval(11,15)]) -[3, 15] - ->>> Point(0,3) == Point(0,3) -True ->>> Point(0,3) == Point(0,3.2) -False ->>> Point(3,4)-Point(1,7) -(2.000000,-3.000000) ->>> -Point(1,2) -(-1.000000,-2.000000) ->>> Point(1,2)*0.5 -(0.500000,1.000000) - ->>> Point(3,2).norm2Squared() -13 - ->>> Point.distanceNorm2(Point(3,4),Point(1,7)) -3.605551275463989 - ->>> Point(3,2).inPolygon(np.array([[0,0],[1,0],[1,1],[0,1]])) -False ->>> Point(3,2).inPolygon(np.array([[0,0],[4,0],[4,3],[0,3]])) -True - ->>> predictPositionNoLimit(10, Point(0,0), Point(1,1)) # doctest:+ELLIPSIS -((1.0...,1.0...), (10.0...,10.0...)) - ->>> segmentIntersection(Point(0,0), Point(0,1), Point(1,1), Point(2,3)) ->>> segmentIntersection(Point(0,1), Point(0,3), Point(1,0), Point(3,1)) ->>> segmentIntersection(Point(0.,0.), Point(2.,2.), Point(0.,2.), Point(2.,0.)) -(1.000000,1.000000) ->>> segmentIntersection(Point(0,0), Point(4,4), Point(0,4), Point(4,0)) -(2.000000,2.000000) ->>> segmentIntersection(Point(0,1), Point(1,2), Point(2,0), Point(3,2)) - ->>> t1 = Trajectory.fromPointList([(92.2, 102.9), (56.7, 69.6)]) ->>> t2 = Trajectory.fromPointList([(92.2, 102.9), (56.7, 69.6)]) ->>> t1 == t2 -True ->>> t3 = Trajectory.fromPointList([(92.24, 102.9), (56.7, 69.6)]) ->>> t1 == t3 -False ->>> t3 = Trajectory.fromPointList([(92.2, 102.9), (56.7, 69.6), (56.7, 69.6)]) ->>> t1 == t3 -False - ->>> left = Trajectory.fromPointList([(92.291666666666686, 102.99239033124439), (56.774193548387103, 69.688898836168306)]) ->>> middle = Trajectory.fromPointList([(87.211021505376351, 93.390778871978512), (59.032258064516128, 67.540286481647257)]) ->>> right = Trajectory.fromPointList([(118.82392473118281, 115.68263205013426), (63.172043010752688, 66.600268576544309)]) ->>> alignments = [left, middle, right] ->>> for a in alignments: a.computeCumulativeDistances() ->>> getSYfromXY(Point(73, 82), alignments) -[1, 0, (73.819977,81.106170), 18.172277808821125, 18.172277808821125, 1.2129694042343868] ->>> getSYfromXY(Point(78, 83), alignments, 0.5) -[1, 0, (77.033188,84.053889), 13.811799123113715, 13.811799123113715, -1.4301775140225983] - ->>> Trajectory().length() -0 ->>> t1 = Trajectory([[0.5,1.5,2.5],[0.5,3.5,6.5]]) ->>> t1.length() == 3. -True ->>> t1[1] -(1.500000,3.500000) - ->>> t1.differentiate() -(1.000000,3.000000) (1.000000,3.000000) ->>> t1.differentiate(True) -(1.000000,3.000000) (1.000000,3.000000) (1.000000,3.000000) ->>> t1 = Trajectory([[0.5,1.5,3.5],[0.5,2.5,7.5]]) ->>> t1.differentiate() -(1.000000,2.000000) (2.000000,5.000000) - ->>> t1.computeCumulativeDistances() ->>> t1.getDistance(0) -2.23606797749979 ->>> t1.getDistance(1) -5.385164807134504 ->>> t1.getDistance(2) -Index 2 beyond trajectory length 3-1 ->>> t1.getCumulativeDistance(0) -0.0 ->>> t1.getCumulativeDistance(1) -2.23606797749979 ->>> t1.getCumulativeDistance(2) -7.6212327846342935 ->>> t1.getCumulativeDistance(3) -Index 3 beyond trajectory length 3 - - ->>> from utils import LCSS ->>> lcss = LCSS(lambda x,y: Point.distanceNorm2(x,y) <= 0.1) ->>> Trajectory.lcss(t1, t1, lcss) -3 ->>> lcss = LCSS(lambda p1, p2: (p1-p2).normMax() <= 0.1) ->>> Trajectory.lcss(t1, t1, lcss) -3 - ->>> p1=Point(0,0) ->>> p2=Point(1,0) ->>> v1 = Point(0.1,0.1) ->>> v2 = Point(-0.1, 0.1) ->>> abs(Point.timeToCollision(p1, p2, v1, v2, 0.)-5.0) < 0.00001 -True ->>> abs(Point.timeToCollision(p1, p2, v1, v2, 0.1)-4.5) < 0.00001 -True ->>> p1=Point(0,1) ->>> p2=Point(1,0) ->>> v1 = Point(0,0.1) ->>> v2 = Point(0.1, 0) ->>> Point.timeToCollision(p1, p2, v1, v2, 0.) == None -True ->>> Point.timeToCollision(p2, p1, v2, v1, 0.) == None -True ->>> Point.midPoint(p1, p2) -(0.500000,0.500000) ->>> p1=Point(0.,0.) ->>> p2=Point(5.,0.) ->>> v1 = Point(2.,0.) ->>> v2 = Point(1.,0.) ->>> Point.timeToCollision(p1, p2, v1, v2, 0.) -5.0 ->>> Point.timeToCollision(p1, p2, v1, v2, 1.) -4.0 - ->>> objects = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'object') ->>> len(objects) -5 ->>> objects[0].hasFeatures() -False ->>> features = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'feature') ->>> for o in objects: o.setFeatures(features) ->>> objects[0].hasFeatures() -True - ->>> o1 = MovingObject.generate(1, Point(-5.,0.), Point(1.,0.), TimeInterval(0,10)) ->>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(0,10)) ->>> MovingObject.computePET(o1, o2, 0.1) -(0.0, 5, 5) ->>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(5,15)) ->>> MovingObject.computePET(o1, o2, 0.1) -(5.0, 5, 10) ->>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(15,30)) ->>> MovingObject.computePET(o1, o2, 0.1) -(15.0, 5, 20) - ->>> t = CurvilinearTrajectory(S = [1., 2., 3., 5.], Y = [0.5, 0.5, 0.6, 0.7], lanes = ['1']*4) ->>> t.differentiate() # doctest:+ELLIPSIS -[1.0, 0.0, '1'] [1.0, 0.099..., '1'] [2.0, 0.099..., '1'] ->>> t.differentiate(True) # doctest:+ELLIPSIS -[1.0, 0.0, '1'] [1.0, 0.099..., '1'] [2.0, 0.099..., '1'] [2.0, 0.099..., '1'] ->>> t = CurvilinearTrajectory(S = [1.], Y = [0.5], lanes = ['1']) ->>> t.differentiate().empty() -True - ->>> o1 = MovingObject.generate(1, Point(1., 2.), Point(1., 1.), TimeInterval(0,10)) ->>> o1.features = [o1] ->>> o2 = MovingObject.generate(2, Point(14., 14.), Point(1., 0.), TimeInterval(14,20)) ->>> o2.features = [o2] ->>> o3 = MovingObject.generate(3, Point(2., 2.), Point(1., 1.), TimeInterval(2,12)) ->>> o3.features = [o3] ->>> o13 = MovingObject.concatenate(o1, o3, 4) ->>> o13.getNum() -4 ->>> o13.getTimeInterval() == TimeInterval(0,12) -True ->>> t=5 ->>> o13.getPositionAtInstant(t) == (o1.getPositionAtInstant(t)+o3.getPositionAtInstant(t)).divide(2) -True ->>> len(o13.getFeatures()) -2 ->>> o12 = MovingObject.concatenate(o1, o2, 5) ->>> o12.getTimeInterval() == TimeInterval(o1.getFirstInstant(), o2.getLastInstant()) -True ->>> v = o12.getVelocityAtInstant(12) ->>> v == Point(3./4, 2./4) -True ->>> o12.getPositionAtInstant(11) == o1.getPositionAtInstant(10)+v -True ->>> len(o12.getFeatures()) -3 - ->>> o1 = MovingObject.generate(1, Point(0., 2.), Point(0., 1.), TimeInterval(0,2)) ->>> o1.classifyUserTypeSpeedMotorized(0.5, np.median) ->>> userTypeNames[o1.getUserType()] -'car' ->>> o1.classifyUserTypeSpeedMotorized(1.5, np.median) ->>> userTypeNames[o1.getUserType()] -'pedestrian' - ->>> o1 = MovingObject.generate(1, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) ->>> gt1 = BBMovingObject(1, TimeInterval(0,10), MovingObject.generate(1, Point(0.2,0.6), Point(1.,0.), TimeInterval(0,10)), MovingObject.generate(2, Point(-0.2,-0.4), Point(1.,0.), TimeInterval(0,10))) ->>> gt1.computeCentroidTrajectory() ->>> computeClearMOT([gt1], [], 0.2, 0, 10) -(None, 0.0, 11, 0, 0, 11, None, None) ->>> computeClearMOT([], [o1], 0.2, 0, 10) -(None, None, 0, 0, 11, 0, None, None) ->>> computeClearMOT([gt1], [o1], 0.2, 0, 10) # doctest:+ELLIPSIS -(0.0999..., 1.0, 0, 0, 0, 11, None, None) ->>> computeClearMOT([gt1], [o1], 0.05, 0, 10) -(None, -1.0, 11, 0, 11, 11, None, None) - ->>> o1 = MovingObject(1, TimeInterval(0,3), positions = Trajectory([list(range(4)), [0.1, 0.1, 1.1, 1.1]])) ->>> o2 = MovingObject(2, TimeInterval(0,3), positions = Trajectory([list(range(4)), [0.9, 0.9, -0.1, -0.1]])) ->>> gt1 = BBMovingObject(1, TimeInterval(0,3), MovingObject(positions = Trajectory([list(range(4)), [0.]*4])), MovingObject(positions = Trajectory([list(range(4)), [0.]*4]))) ->>> gt1.computeCentroidTrajectory() ->>> gt2 = BBMovingObject(2, TimeInterval(0,3), MovingObject(positions = Trajectory([list(range(4)), [1.]*4])), MovingObject(positions = Trajectory([list(range(4)), [1.]*4]))) ->>> gt2.computeCentroidTrajectory() ->>> computeClearMOT([gt1, gt2], [o1, o2], 0.2, 0, 3) # doctest:+ELLIPSIS -(0.1..., 0.75, 0, 2, 0, 8, None, None) ->>> computeClearMOT([gt2, gt1], [o2, o1], 0.2, 0, 3) # doctest:+ELLIPSIS -(0.1..., 0.75, 0, 2, 0, 8, None, None) ->>> computeClearMOT([gt1], [o1, o2], 0.2, 0, 3) -(0.1, -0.25, 0, 1, 4, 4, None, None) ->>> computeClearMOT([gt1], [o2, o1], 0.2, 0, 3) # symmetry -(0.1, -0.25, 0, 1, 4, 4, None, None) ->>> computeClearMOT([gt1, gt2], [o1], 0.2, 0, 3) # doctest:+ELLIPSIS -(0.100..., 0.375, 4, 1, 0, 8, None, None) ->>> computeClearMOT([gt2, gt1], [o1], 0.2, 0, 3) # doctest:+ELLIPSIS -(0.100..., 0.375, 4, 1, 0, 8, None, None) ->>> computeClearMOT([gt1, gt2], [o1, o2], 0.08, 0, 3) -(None, -1.0, 8, 0, 8, 8, None, None)
--- a/python/tests/moving_shapely.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ ->>> from moving import * ->>> from shapely.geometry import Polygon ->>> from shapely.prepared import prep - ->>> t1 = Trajectory([[0.5,1.5,2.5],[0.5,3.5,6.5]]) ->>> poly = Polygon([[0,0],[4,0],[4,3],[0,3]]) ->>> sub1, sub2 = t1.getTrajectoryInPolygon(poly) ->>> sub1 -(0.500000,0.500000) ->>> sub1, sub2 = t1.getTrajectoryInPolygon(Polygon([[10,10],[14,10],[14,13],[10,13]])) ->>> sub1.length() -0 ->>> sub1, sub2 = t1.getTrajectoryInPolygon(prep(poly)) ->>> sub1 -(0.500000,0.500000) ->>> t2 = t1.differentiate(True) ->>> sub1, sub2 = t1.getTrajectoryInPolygon(prep(poly), t2) ->>> sub1.length() == sub2.length() -True ->>> sub1 -(0.500000,0.500000) ->>> sub2 -(1.000000,3.000000) - ->>> t1.proportionInPolygon(poly, 0.5) -False ->>> t1.proportionInPolygon(poly, 0.3) -True
--- a/python/tests/prediction.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,71 +0,0 @@ ->>> from prediction import * ->>> import moving, storage, utils ->>> from numpy import absolute, array - ->>> et = PredictedTrajectoryConstant(moving.Point(0,0), moving.Point(1,0)) ->>> et.predictPosition(4) # doctest:+ELLIPSIS -(4.0...,0.0...) ->>> et.predictPosition(1) # doctest:+ELLIPSIS -(1.0...,0.0...) - ->>> et = PredictedTrajectoryConstant(moving.Point(0,0), moving.Point(1,0), moving.NormAngle(0.1,0), maxSpeed = 2) ->>> et.predictPosition(10) # doctest:+ELLIPSIS -(15.5...,0.0...) ->>> et.predictPosition(11) # doctest:+ELLIPSIS -(17.5...,0.0...) ->>> et.predictPosition(12) # doctest:+ELLIPSIS -(19.5...,0.0...) - ->>> import random ->>> acceleration = lambda: random.uniform(-0.5,0.5) ->>> steering = lambda: random.uniform(-0.1,0.1) ->>> et = PredictedTrajectoryRandomControl(moving.Point(0,0),moving.Point(1,1), acceleration, steering, maxSpeed = 2) ->>> p = et.predictPosition(500) ->>> from numpy import max ->>> max(et.getPredictedSpeeds()) <= 2. -True - ->>> p = moving.Point(3,4) ->>> sp = SafetyPoint(p, 0.1, 0) ->>> print(sp) -3 4 0.1 0 - ->>> et1 = PredictedTrajectoryConstant(moving.Point(-5.,0.), moving.Point(1.,0.)) ->>> et2 = PredictedTrajectoryConstant(moving.Point(0.,-5.), moving.Point(0.,1.)) ->>> collision, t, cp1, cp2 = computeCollisionTime(et1, et2, 0.1, 10) ->>> collision -True ->>> t -5 ->>> collision, t, cp1, cp2 = computeCollisionTime(et1, et2, 0.1, 5) ->>> collision -True ->>> t -5 ->>> collision, t, cp1, cp2 = computeCollisionTime(et1, et2, 0.1, 4) ->>> collision -False - ->>> proto = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'feature', [1204])[0] ->>> proto.getPositions().computeCumulativeDistances() ->>> et = PredictedTrajectoryPrototype(proto.getPositionAt(10)+moving.Point(0.5, 0.5), proto.getVelocityAt(10)*0.9, proto, True) ->>> absolute(et.initialSpeed - proto.getVelocityAt(10).norm2()*0.9) < 1e-5 -True ->>> for t in range(int(proto.length())): x=et.predictPosition(t) ->>> traj = et.getPredictedTrajectory() ->>> traj.computeCumulativeDistances() ->>> absolute(array(traj.distances).mean() - et.initialSpeed < 1e-3) -True - ->>> et = PredictedTrajectoryPrototype(proto.getPositionAt(10)+moving.Point(0.6, 0.6), proto.getVelocityAt(10)*0.7, proto, False) ->>> absolute(et.initialSpeed - proto.getVelocityAt(10).norm2()*0.7) < 1e-5 -True ->>> proto = moving.MovingObject.generate(1, moving.Point(-5.,0.), moving.Point(1.,0.), moving.TimeInterval(0,10)) ->>> et = PredictedTrajectoryPrototype(proto.getPositionAt(0)+moving.Point(0., 1.), proto.getVelocityAt(0)*0.5, proto, False) ->>> for t in range(int(proto.length()/0.5)): x=et.predictPosition(t) ->>> et.predictPosition(10) # doctest:+ELLIPSIS -(0.0...,1.0...) ->>> et.predictPosition(12) # doctest:+ELLIPSIS -(1.0...,1.0...) - -
--- a/python/tests/storage.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,128 +0,0 @@ ->>> from storage import * ->>> from io import StringIO ->>> from moving import MovingObject, Point, TimeInterval, Trajectory, prepareSplines - ->>> f = openCheck('non_existant_file.txt') -File non_existant_file.txt could not be opened. - ->>> nonexistentFilename = "nonexistent" ->>> loadTrajectoriesFromSqlite(nonexistentFilename, 'feature') -DB Error: no such table: positions -DB Error: no such table: velocities -[] ->>> from os import remove ->>> remove(nonexistentFilename) - ->>> o1 = MovingObject.generate(2, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) ->>> o2 = MovingObject.generate(3, Point(1.,1.), Point(-0.5,-0.2), TimeInterval(0,9)) ->>> saveTrajectoriesToSqlite('test.sqlite', [o1, o2], 'feature') ->>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'feature') ->>> objects[0].getNum() == o1.num -True ->>> objects[1].getNum() == o2.num -True ->>> o1.getTimeInterval() == objects[0].getTimeInterval() -True ->>> o2.getTimeInterval() == objects[1].getTimeInterval() -True ->>> o1.getVelocities().length() == objects[0].getVelocities().length() -True ->>> o2.getVelocities().length() == objects[1].getVelocities().length() -True ->>> o1.getVelocities() == objects[0].getVelocities() -True ->>> o2.getVelocities() == objects[1].getVelocities() -True ->>> o1.getPositions() == objects[0].getPositions() -True ->>> o2.getPositions() == objects[1].getPositions() -True ->>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'feature', timeStep = 2) ->>> objects[0].positions.length() -6 ->>> objects[1].positions.length() -5 ->>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'feature', timeStep = 3) ->>> objects[0].positions.length() -4 ->>> objects[1].positions.length() -4 ->>> align1 = Trajectory.fromPointList([Point(-1, 0), Point(20, 0)]) ->>> align2 = Trajectory.fromPointList([Point(-9, -3), Point(6, 3)]) ->>> align1.computeCumulativeDistances() ->>> align2.computeCumulativeDistances() ->>> prepareSplines([align1, align2]) ->>> o1.projectCurvilinear([align1, align2]) ->>> o2.projectCurvilinear([align1, align2]) ->>> saveTrajectoriesToSqlite('test.sqlite', [o1, o2], 'curvilinear') ->>> addCurvilinearTrajectoriesFromSqlite('test.sqlite', {o.num: o for o in objects}) ->>> o1.curvilinearPositions[3][:2] == objects[0].curvilinearPositions[3][:2] -True ->>> o1.curvilinearPositions[7][:2] == objects[0].curvilinearPositions[7][:2] -True ->>> [str(l) for l in o1.curvilinearPositions.getLanes()] == objects[0].curvilinearPositions.getLanes() -True ->>> o2.curvilinearPositions[2][:2] == objects[1].curvilinearPositions[2][:2] -True ->>> o2.curvilinearPositions[6][:2] == objects[1].curvilinearPositions[6][:2] -True ->>> [str(l) for l in o2.curvilinearPositions.getLanes()] == objects[1].curvilinearPositions.getLanes() -True ->>> remove('test.sqlite') - ->>> f1 = MovingObject.generate(3, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) ->>> f2 = MovingObject.generate(4, Point(1.,1.), Point(-0.5,-0.2), TimeInterval(0,9)) ->>> o1 = MovingObject(num = 1, userType = 1) ->>> o1.features = [f1, f2] ->>> saveTrajectoriesToSqlite('test.sqlite', [o1], 'object') ->>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'object', withFeatures = True) ->>> len(objects) -1 ->>> reloaded1 = objects[0] ->>> reloaded1.getNum() == o1.getNum() -True ->>> reloaded1.getUserType() == o1.getUserType() -True ->>> len(reloaded1.featureNumbers) -2 ->>> len(reloaded1.features) -2 ->>> reloaded1.getPositionAt(0) == Point.midPoint(f1.getPositionAt(0), f2.getPositionAt(0)) -True ->>> reloaded1.getPositionAt(5) == Point.midPoint(f1.getPositionAt(5), f2.getPositionAt(5)) -True ->>> reloaded1.getPositionAt(10) == f1.getPositionAt(10) -True ->>> set(reloaded1.featureNumbers) == set([f1.num, f2.num]) -True ->>> remove('test.sqlite') - ->>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') ->>> readline(strio) -'sadlkfjsdlakjf' ->>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') ->>> readline(strio, ['#']) -'sadlkfjsdlakjf' ->>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') ->>> readline(strio, ['%']) -'# asdlfjasdlkj0' ->>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') ->>> readline(strio, '%*$') -'# asdlfjasdlkj0' ->>> readline(strio, '%#') -'sadlkfjsdlakjf' - ->>> from sklearn.mixture import GaussianMixture ->>> from numpy.random import random_sample ->>> nPoints = 50 ->>> points = random_sample(nPoints*2).reshape(nPoints,2) ->>> gmm = GaussianMixture(4, covariance_type = 'full') ->>> tmp = gmm.fit(points) ->>> gmmId = 0 ->>> savePOIsToSqlite('pois-tmp.sqlite', gmm, 'end', gmmId) ->>> reloadedGmm = loadPOIsFromSqlite('pois-tmp.sqlite') ->>> sum(gmm.predict(points) == reloadedGmm[gmmId].predict(points)) == nPoints -True ->>> reloadedGmm[gmmId].gmmTypes[0] == 'end' -True ->>> remove('pois-tmp.sqlite')
--- a/python/tests/tutorials.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ -import unittest - -class TestNGSIM(unittest.TestCase): - 'Tutorial example for NGSIM data' - - def test_ex1(self): - import storage - objects = storage.loadTrajectoriesFromNgsimFile('../samples/trajectories-0400-0415.txt',100) - for o in objects: o.plot() - -class TestTrajectoryLoading(unittest.TestCase): - 'Tutorial example for NGSIM data' - - def test_ex1(self): - import storage - objects = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'object') - - speed = objects[0].getVelocityAtInstant(10).norm2() - timeInterval = objects[0].getTimeInterval() - speeds = [objects[0].getVelocityAtInstant(t).norm2() for t in range(timeInterval.first, timeInterval.last)] - speeds = [v.norm2() for v in objects[0].getVelocities()] - - from matplotlib.pyplot import plot, close, axis - plot(range(timeInterval.first, timeInterval.last+1), speeds) - - close('all') - objects[0].plot() - axis('equal') - - features = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'feature') - objects[0].setFeatures(features) - - for f in objects[0].features: - f.plot() - axis('equal') - - -if __name__ == '__main__': - unittest.main()
--- a/python/tests/utils.txt Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,144 +0,0 @@ ->>> from utils import * ->>> from moving import Point - ->>> upperCaseFirstLetter('mmmm... donuts') -'Mmmm... Donuts' ->>> s = upperCaseFirstLetter('much ado about nothing') ->>> s == 'Much Ado About Nothing' -True ->>> upperCaseFirstLetter(s) == s -True - ->>> computeChi2([],[]) -0 ->>> computeChi2(list(range(1,10)),list(range(1,10))) -0.0 ->>> computeChi2(list(range(1,9)),list(range(1,10))) -0.0 - ->>> ceilDecimals(1.23, 0) -2.0 ->>> ceilDecimals(1.23, 1) -1.3 - ->>> inBetween(1,2,1.5) -True ->>> inBetween(2.1,1,1.5) -True ->>> inBetween(1,2,0) -False - ->>> removeExtension('test-adfasdf.asdfa.txt') -'test-adfasdf.asdfa' ->>> removeExtension('test-adfasdf') -'test-adfasdf' - ->>> values = line2Ints('1 2 3 5 6') ->>> values[0] -1 ->>> values[-1] -6 ->>> values = line2Floats('1.3 2.45 7.158e+01 5 6') ->>> values[0] -1.3 ->>> values[2] #doctest: +ELLIPSIS -71.5... ->>> values[-1] -6.0 - ->>> stepPlot([3, 5, 7, 8], 1, 10, 0) -([1, 3, 3, 5, 5, 7, 7, 8, 8, 10], [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]) - ->>> mostCommon(['a','b','c','b']) -'b' ->>> mostCommon(['a','b','c','b', 'c']) -'b' ->>> mostCommon(list(range(10))+[1]) -1 ->>> mostCommon([list(range(2)), list(range(4)), list(range(2))]) -[0, 1] - ->>> res = sortByLength([list(range(3)), list(range(4)), list(range(1))]) ->>> [len(r) for r in res] -[1, 3, 4] ->>> res = sortByLength([list(range(3)), list(range(4)), list(range(1)), list(range(5))], reverse = True) ->>> [len(r) for r in res] -[5, 4, 3, 1] - ->>> lcss = LCSS(similarityFunc = lambda x,y: abs(x-y) <= 0.1) ->>> lcss.compute(list(range(5)), list(range(5))) -5 ->>> lcss.compute(list(range(1,5)), list(range(5))) -4 ->>> lcss.compute(list(range(5,10)), list(range(5))) -0 ->>> lcss.compute(list(range(5)), list(range(10))) -5 ->>> lcss.similarityFunc = lambda x,y: x == y ->>> lcss.compute(['a','b','c'], ['a','b','c', 'd']) -3 ->>> lcss.computeNormalized(['a','b','c'], ['a','b','c', 'd']) #doctest: +ELLIPSIS -1.0 ->>> lcss.computeNormalized(['a','b','c','x'], ['a','b','c', 'd']) #doctest: +ELLIPSIS -0.75 ->>> lcss.compute(['a','b','c'], ['a','b','c', 'd']) -3 ->>> lcss.compute(['a','x','b','c'], ['a','b','c','d','x']) -3 ->>> lcss.compute(['a','b','c','x','d'], ['a','b','c','d','x']) -4 ->>> lcss.delta = 1 ->>> lcss.compute(['a','b','c'], ['a','b','x','x','c']) -2 - ->>> lcss.delta = float('inf') ->>> lcss.compute(['a','b','c'], ['a','b','c', 'd'], computeSubSequence = True) -3 ->>> lcss.subSequenceIndices -[(0, 0), (1, 1), (2, 2)] ->>> lcss.compute(['a','b','c'], ['x','a','b','c'], computeSubSequence = True) -3 ->>> lcss.subSequenceIndices -[(0, 1), (1, 2), (2, 3)] ->>> lcss.compute(['a','g','b','c'], ['a','b','c', 'd'], computeSubSequence = True) -3 ->>> lcss.subSequenceIndices -[(0, 0), (2, 1), (3, 2)] - ->>> alignedLcss = LCSS(lambda x,y:(abs(x-y) <= 0.1), delta = 2, aligned = True) ->>> alignedLcss.compute(list(range(5)), list(range(5))) -5 ->>> alignedLcss.compute(list(range(1,5)), list(range(5))) -4 - ->>> alignedLcss.compute(list(range(5,10)), list(range(10))) -5 - ->>> lcss.delta = 2 ->>> lcss.compute(list(range(5,10)), list(range(10))) -0 ->>> alignedLcss.delta = 6 ->>> alignedLcss.compute(list(range(5)), list(range(5))) -5 ->>> alignedLcss.compute(list(range(5)), list(range(6))) -5 ->>> lcss.delta = 10 ->>> alignedLcss.compute(list(range(1,7)), list(range(6))) -5 ->>> lcss = LCSS(lambda x,y: x == y, delta = 2, aligned = True) ->>> lcss.compute(list(range(20)), [2,4,6,7,8,9,11,13], True) -8 ->>> lcss.subSequenceIndices -[(2, 0), (4, 1), (6, 2), (7, 3), (8, 4), (9, 5), (11, 6), (13, 7)] - ->>> lcss = LCSS(metric = 'cityblock', epsilon = 0.1) ->>> lcss.compute([[i] for i in range(5)], [[i] for i in range(5)]) -5 ->>> lcss.compute([[i] for i in range(1,5)], [[i] for i in range(5)]) -4 ->>> lcss.compute([[i] for i in range(5,10)], [[i] for i in range(5)]) -0 ->>> lcss.compute([[i] for i in range(5)], [[i] for i in range(10)]) -5 - -
--- a/python/traffic_engineering.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,337 +0,0 @@ -#! /usr/bin/env python -''' Traffic Engineering Tools and Examples''' - -import prediction - -from math import ceil - - -######################### -# Simulation -######################### - -def generateTimeHeadways(meanTimeHeadway, simulationTime): - '''Generates the time headways between arrivals - given the meanTimeHeadway and the negative exponential distribution - over a time interval of length simulationTime (assumed to be in same time unit as headway''' - from random import expovariate - headways = [] - totalTime = 0 - flow = 1/meanTimeHeadway - while totalTime < simulationTime: - h = expovariate(flow) - headways.append(h) - totalTime += h - return headways - -class RoadUser(object): - '''Simple example of inheritance to plot different road users ''' - def __init__(self, position, velocity): - 'Both fields are 2D numpy arrays' - self.position = position.astype(float) - self.velocity = velocity.astype(float) - - def move(self, deltaT): - self.position += deltaT*self.velocity - - def draw(self, init = False): - from matplotlib.pyplot import plot - if init: - self.plotLine = plot(self.position[0], self.position[1], self.getDescriptor())[0] - else: - self.plotLine.set_data(self.position[0], self.position[1]) - - -class PassengerVehicle(RoadUser): - def getDescriptor(self): - return 'dr' - -class Pedestrian(RoadUser): - def getDescriptor(self): - return 'xb' - -class Cyclist(RoadUser): - def getDescriptor(self): - return 'og' - -######################### -# queueing models -######################### - -class CapacityReduction(object): - def __init__(self, beta, reductionDuration, demandCapacityRatio = None, demand = None, capacity = None): - '''reduction duration should be positive - demandCapacityRatio is demand/capacity (q/s)''' - if demandCapacityRatio is None and demand is None and capacity is None: - print('Missing too much information (demand, capacity and ratio)') - import sys - sys.exit() - if 0 <= beta < 1: - self.beta = beta - self.reductionDuration = reductionDuration - - if demandCapacityRatio is not None: - self.demandCapacityRatio = demandCapacityRatio - if demand is not None: - self.demand = demand - if capacity is not None: - self.capacity = capacity - if capacity is not None and demand is not None: - self.demandCapacityRatio = float(self.demand)/self.capacity - if demand <= beta*capacity: - print('There is no queueing as the demand {} is inferior to the reduced capacity {}'.format(demand, beta*capacity)) - else: - print('reduction coefficient (beta={}) is not in [0, 1['.format(beta)) - - def queueingDuration(self): - return self.reductionDuration*(1-self.beta)/(1-self.demandCapacityRatio) - - def nArrived(self, t): - if self.demand is None: - print('Missing demand field') - return None - return self.demand*t - - def nServed(self, t): - if self.capacity is None: - print('Missing capacity field') - return None - if 0<=t<=self.reductionDuration: - return self.beta*self.capacity*t - elif self.reductionDuration < t <= self.queueingDuration(): - return self.beta*self.capacity*self.reductionDuration+self.capacity*(t-self.reductionDuration) - - def nQueued(self, t): - return self.nArrived(t)-self.nServed(t) - - def maxNQueued(self): - return self.nQueued(self.reductionDuration) - - def totalDelay(self): - if self.capacity is None: - print('Missing capacity field') - return None - return self.capacity*self.reductionDuration**2*(1-self.beta)*(self.demandCapacityRatio-self.beta)/(2*(1-self.demandCapacityRatio)) - - def averageDelay(self): - return self.reductionDuration*(self.demandCapacityRatio-self.beta)/(2*self.demandCapacityRatio) - - def averageNQueued(self): - return self.totalDelay()/self.queueingDuration() - - -######################### -# fundamental diagram -######################### - -class FundamentalDiagram(object): - ''' ''' - def __init__(self, name): - self.name = name - - def q(self, k): - return k*self.v(k) - - @staticmethod - def meanHeadway(k): - return 1/k - - @staticmethod - def meanSpacing(q): - return 1/q - - def plotVK(self, language='fr', units={}): - from numpy import arange - from matplotlib.pyplot import figure,plot,xlabel,ylabel - densities = [k for k in arange(1, self.kj+1)] - figure() - plot(densities, [self.v(k) for k in densities]) - xlabel('Densite (veh/km)') # todo other languages and adapt to units - ylabel('Vitesse (km/h)') - - def plotQK(self, language='fr', units={}): - from numpy import arange - from matplotlib.pyplot import figure,plot,xlabel,ylabel - densities = [k for k in arange(1, self.kj+1)] - figure() - plot(densities, [self.q(k) for k in densities]) - xlabel('Densite (veh/km)') # todo other languages and adapt to units - ylabel('Debit (km/h)') - -class GreenbergFD(FundamentalDiagram): - '''Speed is the logarithm of density''' - def __init__(self, vc, kj): - FundamentalDiagram.__init__(self,'Greenberg') - self.vc=vc - self.kj=kj - - def v(self,k): - from numpy import log - return self.vc*log(self.kj/k) - - def criticalDensity(self): - from numpy import e - self.kc = self.kj/e - return self.kc - - def capacity(self): - self.qmax = self.kc*self.vc - return self.qmax - -######################### -# intersection -######################### - -class FourWayIntersection(object): - '''Simple class for simple intersection outline''' - def __init__(self, dimension, coordX, coordY): - self.dimension = dimension - self.coordX = coordX - self.coordY = coordY - - def plot(self, options = 'k'): - from matplotlib.pyplot import plot, axis - - minX = min(self.dimension[0]) - maxX = max(self.dimension[0]) - minY = min(self.dimension[1]) - maxY = max(self.dimension[1]) - - plot([minX, self.coordX[0], self.coordX[0]], [self.coordY[0], self.coordY[0], minY],options) - plot([self.coordX[1], self.coordX[1], maxX], [minY, self.coordY[0], self.coordY[0]],options) - plot([minX, self.coordX[0], self.coordX[0]], [self.coordY[1], self.coordY[1], maxY],options) - plot([self.coordX[1], self.coordX[1], maxX], [maxY, self.coordY[1], self.coordY[1]],options) - axis('equal') - -######################### -# traffic signals -######################### - -class Volume(object): - '''Class to represent volumes with varied vehicule types ''' - def __init__(self, volume, types = ['pc'], proportions = [1], equivalents = [1], nLanes = 1): - '''mvtEquivalent is the equivalent if the movement is right of left turn''' - - # check the sizes of the lists - if sum(proportions) == 1: - self.volume = volume - self.types = types - self.proportions = proportions - self.equivalents = equivalents - self.nLanes = nLanes - else: - print('Proportions do not sum to 1') - pass - - def checkProtected(self, opposedThroughMvt): - '''Checks if this left movement should be protected, - ie if one of the main two conditions on left turn is verified''' - return self.volume >= 200 or self.volume*opposedThroughMvt.volume/opposedThroughMvt.nLanes > 50000 - - def getPCUVolume(self): - '''Returns the passenger-car equivalent for the input volume''' - v = 0 - for p, e in zip(self.proportions, self.equivalents): - v += p*e - return v*self.volume - -class IntersectionMovement(object): - '''Represents an intersection movement - with a volume, a type (through, left or right) - and an equivalent for movement type''' - def __init__(self, volume, mvtEquivalent = 1): - self.volume = volume - self.mvtEquivalent = mvtEquivalent - - def getTVUVolume(self): - return self.mvtEquivalent*self.volume.getPCUVolume() - -class LaneGroup(object): - '''Class that represents a group of mouvements''' - - def __init__(self, movements, nLanes): - self.movements = movements - self.nLanes = nLanes - - def getTVUVolume(self): - return sum([mvt.getTVUVolume() for mvt in self.movements]) - - def getCharge(self, saturationVolume): - return self.getTVUVolume()/(self.nLanes*saturationVolume) - -def optimalCycle(lostTime, criticalCharge): - return (1.5*lostTime+5)/(1-criticalCharge) - -def minimumCycle(lostTime, criticalCharge, degreeSaturation=1.): - 'degree of saturation can be used as the peak hour factor too' - return lostTime/(1-criticalCharge/degreeSaturation) - -class Cycle(object): - '''Class to compute optimal cycle and the split of effective green times''' - def __init__(self, phases, lostTime, saturationVolume): - '''phases is a list of phases - a phase is a list of lanegroups''' - self.phases = phases - self.lostTime = lostTime - self.saturationVolume = saturationVolume - - def computeCriticalCharges(self): - self.criticalCharges = [max([lg.getCharge(self.saturationVolume) for lg in phase]) for phase in self.phases] - self.criticalCharge = sum(self.criticalCharges) - - def computeOptimalCycle(self): - self.computeCriticalCharges() - self.C = optimalCycle(self.lostTime, self.criticalCharge) - return self.C - - def computeMinimumCycle(self, degreeSaturation=1.): - self.computeCriticalCharges() - self.C = minimumCycle(self.lostTime, self.criticalCharge, degreeSaturation) - return self.C - - def computeEffectiveGreen(self): - #from numpy import round - #self.computeCycle() # in case it was not done before - effectiveGreenTime = self.C-self.lostTime - self.effectiveGreens = [round(c*effectiveGreenTime/self.criticalCharge,1) for c in self.criticalCharges] - return self.effectiveGreens - - -def computeInterGreen(perceptionReactionTime, initialSpeed, intersectionLength, vehicleAverageLength = 6, deceleration = 3): - '''Computes the intergreen time (yellow/amber plus all red time) - Deceleration is positive - All variables should be in the same units''' - if deceleration > 0: - return [perceptionReactionTime+float(initialSpeed)/(2*deceleration), float(intersectionLength+vehicleAverageLength)/initialSpeed] - else: - print('Issue deceleration should be strictly positive') - return None - -def uniformDelay(cycleLength, effectiveGreen, saturationDegree): - '''Computes the uniform delay''' - return 0.5*cycleLength*(1-float(effectiveGreen)/cycleLength)**2/(1-float(effectiveGreen*saturationDegree)/cycleLength) - -def randomDelay(volume, saturationDegree): - '''Computes the random delay = queueing time for M/D/1''' - return saturationDegree**2/(2*volume*(1-saturationDegree)) - -def incrementalDelay(T, X, c, k=0.5, I=1): - '''Computes the incremental delay (HCM) - T in hours - c capacity of the lane group - k default for fixed time signal - I=1 for isolated intersection (Poisson arrival)''' - from math import sqrt - return 900*T*(X - 1 + sqrt((X - 1)**2 + 8*k*I*X/(c*T))) - -######################### -# misc -######################### - -def timeChangingSpeed(v0, vf, a, TPR): - 'for decelerations, a < 0' - return TPR-(vf-v0)/a - -def distanceChangingSpeed(v0, vf, a, TPR): - 'for decelerations, a < 0' - return TPR*v0-(vf**2-v0**2)/(2*a)
--- a/python/ubc_utils.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,226 +0,0 @@ -#! /usr/bin/env python -'''Various utilities to load data saved by the UBC tool(s)''' - -import utils, events, storage -from moving import MovingObject, TimeInterval, Trajectory - - -fileTypeNames = ['feature', - 'object', - 'prototype', - 'contoursequence'] - -severityIndicatorNames = ['Distance', - 'Collision Course Cosine', - 'Velocity Cosine', - 'Speed Differential', - 'Collision Probability', - 'Severity Index', - 'Time to Collision'] - -userTypeNames = ['car', - 'pedestrian', - 'twowheels', - 'bus', - 'truck'] - -# severityIndicator = {'Distance': 0, -# 'Cosine': 1, -# 'Velocity Cosine': 2, -# 'Speed Differential': 3, -# 'Collision Probability': 4, -# 'Severity Index': 5, -# 'TTC': 6} - -mostSevereIsMax = [False, - False, - True, - True, - True, - True, - False] - -ignoredValue = [None, None, None, None, None, None, -1] - -def getFileType(s): - 'Finds the type in fileTypeNames' - for fileType in fileTypeNames: - if s.find(fileType)>0: - return fileType - return '' - -def isFileType(s, fileType): - return (s.find(fileType)>0) - -def saveTrajectoryUserTypes(inFilename, outFilename, objects): - '''The program saves the objects, - by just copying the corresponding trajectory and velocity data - from the inFilename, and saving the characteristics in objects (first line) - into outFilename''' - infile = storage.openCheck(inFilename) - outfile = storage.openCheck(outFilename,'w') - - if (inFilename.find('features') >= 0) or (not infile) or (not outfile): - return - - lines = storage.getLines(infile) - objNum = 0 # in inFilename - while lines != []: - # find object in objects (index i) - i = 0 - while (i<len(objects)) and (objects[i].num != objNum): - i+=1 - - if i<len(objects): - l = lines[0].split(' ') - l[3] = str(objects[i].userType) - outfile.write(' '.join(l)+'\n') - for l in lines[1:]: - outfile.write(l+'\n') - outfile.write(utils.delimiterChar+'\n') - # next object - objNum += 1 - lines = storage.getLines(infile) - - print('read {0} objects'.format(objNum)) - -def modifyTrajectoryFile(modifyLines, filenameIn, filenameOut): - '''Reads filenameIn, replaces the lines with the result of modifyLines and writes the result in filenameOut''' - fileIn = storage.openCheck(filenameIn, 'r', True) - fileOut = storage.openCheck(filenameOut, "w", True) - - lines = storage.getLines(fileIn) - trajNum = 0 - while (lines != []): - modifiedLines = modifyLines(trajNum, lines) - if modifiedLines: - for l in modifiedLines: - fileOut.write(l+"\n") - fileOut.write(utils.delimiterChar+"\n") - lines = storage.getLines(fileIn) - trajNum += 1 - - fileIn.close() - fileOut.close() - -def copyTrajectoryFile(keepTrajectory, filenameIn, filenameOut): - '''Reads filenameIn, keeps the trajectories for which the function keepTrajectory(trajNum, lines) is True - and writes the result in filenameOut''' - fileIn = storage.openCheck(filenameIn, 'r', True) - fileOut = storage.openCheck(filenameOut, "w", True) - - lines = storage.getLines(fileIn) - trajNum = 0 - while (lines != []): - if keepTrajectory(trajNum, lines): - for l in lines: - fileOut.write(l+"\n") - fileOut.write(utils.delimiterChar+"\n") - lines = storage.getLines(fileIn) - trajNum += 1 - - fileIn.close() - fileOut.close() - -def loadTrajectories(filename, nObjects = -1): - '''Loads trajectories''' - - file = storage.openCheck(filename) - if (not file): - return [] - - objects = [] - objNum = 0 - objectType = getFileType(filename) - lines = storage.getLines(file) - while (lines != []) and ((nObjects<0) or (objNum<nObjects)): - l = lines[0].split(' ') - parsedLine = [int(n) for n in l[:4]] - obj = MovingObject(num = objNum, timeInterval = TimeInterval(parsedLine[1],parsedLine[2])) - #add = True - if len(lines) >= 3: - obj.positions = Trajectory.load(lines[1], lines[2]) - if len(lines) >= 5: - obj.velocities = Trajectory.load(lines[3], lines[4]) - if objectType == 'object': - obj.userType = parsedLine[3] - obj.nObjects = float(l[4]) - obj.featureNumbers = [int(n) for n in l[5:]] - - # load contour data if available - if len(lines) >= 6: - obj.contourType = utils.line2Floats(lines[6]) - obj.contourOrigins = Trajectory.load(lines[7], lines[8]) - obj.contourSizes = Trajectory.load(lines[9], lines[10]) - elif objectType == 'prototype': - obj.userType = parsedLine[3] - obj.nMatchings = int(l[4]) - - if len(lines) != 2: - objects.append(obj) - objNum+=1 - else: - print("Error two lines of data for feature {}".format(f.num)) - - lines = storage.getLines(file) - - file.close() - return objects - -def getFeatureNumbers(objects): - featureNumbers=[] - for o in objects: - featureNumbers += o.featureNumbers - return featureNumbers - -def loadInteractions(filename, nInteractions = -1): - 'Loads interactions from the old UBC traffic event format' - from events import Interaction - from indicators import SeverityIndicator - file = storage.openCheck(filename) - if (not file): - return [] - - interactions = [] - interactionNum = 0 - lines = storage.getLines(file) - while (lines != []) and ((nInteractions<0) or (interactionNum<nInteractions)): - parsedLine = [int(n) for n in lines[0].split(' ')] - inter = Interaction(interactionNum, TimeInterval(parsedLine[1],parsedLine[2]), parsedLine[3], parsedLine[4], categoryNum = parsedLine[5]) - - indicatorFrameNums = [int(n) for n in lines[1].split(' ')] - for indicatorNum,line in enumerate(lines[2:]): - values = {} - for i,v in enumerate([float(n) for n in line.split(' ')]): - if not ignoredValue[indicatorNum] or v != ignoredValue[indicatorNum]: - values[indicatorFrameNums[i]] = v - inter.addIndicator(SeverityIndicator(severityIndicatorNames[indicatorNum], values, None, mostSevereIsMax[indicatorNum])) - - interactions.append(inter) - interactionNum+=1 - lines = storage.getLines(file) - - file.close() - return interactions - -def loadCollisionPoints(filename, nPoints = -1): - '''Loads collision points and returns a dict - with keys as a pair of the numbers of the two interacting objects''' - file = storage.openCheck(filename) - if (not file): - return [] - - points = {} - num = 0 - lines = storage.getLines(file) - while (lines != []) and ((nPoints<0) or (num<nPoints)): - parsedLine = [int(n) for n in lines[0].split(' ')] - protagonistNums = (parsedLine[0], parsedLine[1]) - points[protagonistNums] = [[float(n) for n in lines[1].split(' ')], - [float(n) for n in lines[2].split(' ')]] - - num+=1 - lines = storage.getLines(file) - - file.close() - return points
--- a/python/utils.py Fri Jun 15 11:18:43 2018 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1070 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -''' Generic utilities.''' - -import matplotlib.pyplot as plt -from datetime import time, datetime -from argparse import ArgumentTypeError -from pathlib import Path -from math import sqrt, ceil, floor -from scipy.stats import kruskal, shapiro, lognorm -from scipy.spatial import distance -from scipy.sparse import dok_matrix -from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, mean, median, percentile, isnan, ones, convolve, dtype, isnan, NaN, ma, isinf, savez, load as npload, log - - -datetimeFormat = "%Y-%m-%d %H:%M:%S" - -sjcamDatetimeFormat = "%Y_%m%d_%H%M%S"#2017_0626_143720 - -######################### -# Strings -######################### - -def upperCaseFirstLetter(s): - words = s.split(' ') - lowerWords = [w[0].upper()+w[1:].lower() for w in words] - return ' '.join(lowerWords) - -class TimeConverter: - def __init__(self, datetimeFormat = datetimeFormat): - self.datetimeFormat = datetimeFormat - - def convert(self, s): - try: - return datetime.strptime(s, self.datetimeFormat) - except ValueError: - msg = "Not a valid date: '{0}'.".format(s) - raise ArgumentTypeError(msg) - -######################### -# Enumerations -######################### - -def inverseEnumeration(l): - 'Returns the dictionary that provides for each element in the input list its index in the input list' - result = {} - for i,x in enumerate(l): - result[x] = i - return result - -######################### -# Simple statistics -######################### - -def logNormalMeanVar(loc, scale): - '''location and scale are respectively the mean and standard deviation of the normal in the log-normal distribution - https://en.wikipedia.org/wiki/Log-normal_distribution - - same as lognorm.stats(scale, 0, exp(loc))''' - mean = exp(loc+(scale**2)/2) - var = (exp(scale**2)-1)*exp(2*loc+scale**2) - return mean, var - -def fitLogNormal(x): - 'returns the fitted location and scale of the lognormal (general definition)' - shape, loc, scale = lognorm.fit(x, floc=0.) - return log(scale), shape - -def sampleSize(stdev, tolerance, percentConfidence, nRoundingDigits = None, printLatex = False): - from scipy.stats.distributions import norm - if nRoundingDigits is None: - k = round(norm.ppf(0.5+percentConfidence/200., 0, 1), 2) # 1.-(100-percentConfidence)/200. - else: - k = round(norm.ppf(0.5+percentConfidence/200., 0, 1), nRoundingDigits) - stdev = round(stdev, nRoundingDigits) - tolerance = round(tolerance, nRoundingDigits) - if printLatex: - print('$z_{{{}}}^2\\frac{{s^2}}{{e^2}}={}^2\\frac{{{}^2}}{{{}^2}}$'.format(0.5+percentConfidence/200.,k, stdev, tolerance)) - return (k*stdev/tolerance)**2 - -def confidenceInterval(mean, stdev, nSamples, percentConfidence, trueStd = True, printLatex = False): - '''if trueStd, use normal distribution, otherwise, Student - - Use otherwise t.interval or norm.interval for the boundaries - ex: norm.interval(0.95) - t.interval(0.95, nSamples-1)''' - from scipy.stats.distributions import norm, t - if trueStd: - k = round(norm.ppf(0.5+percentConfidence/200., 0, 1), 2) - else: # use Student - k = round(t.ppf(0.5+percentConfidence/200., nSamples-1), 2) - e = k*stdev/sqrt(nSamples) - if printLatex: - print('${0} \pm {1}\\frac{{{2}}}{{\sqrt{{{3}}}}}$'.format(mean, k, stdev, nSamples)) - return mean-e, mean+e - -def computeChi2(expected, observed): - '''Returns the Chi2 statistics''' - return sum([((e-o)*(e-o))/float(e) for e, o in zip(expected, observed)]) - -class DistributionSample(object): - def nSamples(self): - return sum(self.counts) - -def cumulativeDensityFunction(sample, normalized = False): - '''Returns the cumulative density function of the sample of a random variable''' - xaxis = sorted(sample) - counts = arange(1,len(sample)+1) # dtype = float - if normalized: - counts /= float(len(sample)) - return xaxis, counts - -class DiscreteDistributionSample(DistributionSample): - '''Class to represent a sample of a distribution for a discrete random variable''' - def __init__(self, categories, counts): - self.categories = categories - self.counts = counts - - def mean(self): - result = [float(x*y) for x,y in zip(self.categories, self.counts)] - return npsum(result)/self.nSamples() - - def var(self, mean = None): - if not mean: - m = self.mean() - else: - m = mean - result = 0. - squares = [float((x-m)*(x-m)*y) for x,y in zip(self.categories, self.counts)] - return npsum(squares)/(self.nSamples()-1) - - def referenceCounts(self, probability): - '''probability is a function that returns the probability of the random variable for the category values''' - refProba = [probability(c) for c in self.categories] - refProba[-1] = 1-npsum(refProba[:-1]) - refCounts = [r*self.nSamples() for r in refProba] - return refCounts, refProba - -class ContinuousDistributionSample(DistributionSample): - '''Class to represent a sample of a distribution for a continuous random variable - with the number of observations for each interval - intervals (categories variable) are defined by their left limits, the last one being the right limit - categories contain therefore one more element than the counts''' - def __init__(self, categories, counts): - # todo add samples for initialization and everything to None? (or setSamples?) - self.categories = categories - self.counts = counts - - @staticmethod - def generate(sample, categories): - if min(sample) < min(categories): - print('Sample has lower min than proposed categories ({}, {})'.format(min(sample), min(categories))) - if max(sample) > max(categories): - print('Sample has higher max than proposed categories ({}, {})'.format(max(sample), max(categories))) - dist = ContinuousDistributionSample(sorted(categories), [0]*(len(categories)-1)) - for s in sample: - i = 0 - while i<len(dist.categories) and dist.categories[i] <= s: - i += 1 - if i <= len(dist.counts): - dist.counts[i-1] += 1 - #print('{} in {} {}'.format(s, dist.categories[i-1], dist.categories[i])) - else: - print('Element {} is not in the categories'.format(s)) - return dist - - def mean(self): - result = 0. - for i in range(len(self.counts)-1): - result += self.counts[i]*(self.categories[i]+self.categories[i+1])/2 - return result/self.nSamples() - - def var(self, mean = None): - if not mean: - m = self.mean() - else: - m = mean - result = 0. - for i in range(len(self.counts)-1): - mid = (self.categories[i]+self.categories[i+1])/2 - result += self.counts[i]*(mid - m)*(mid - m) - return result/(self.nSamples()-1) - - def referenceCounts(self, cdf): - '''cdf is a cumulative distribution function - returning the probability of the variable being less that x''' - # refCumulativeCounts = [0]#[cdf(self.categories[0][0])] -# for inter in self.categories: -# refCumulativeCounts.append(cdf(inter[1])) - refCumulativeCounts = [cdf(x) for x in self.categories[1:-1]] - - refProba = [refCumulativeCounts[0]] - for i in xrange(1,len(refCumulativeCounts)): - refProba.append(refCumulativeCounts[i]-refCumulativeCounts[i-1]) - refProba.append(1-refCumulativeCounts[-1]) - refCounts = [p*self.nSamples() for p in refProba] - - return refCounts, refProba - - def printReferenceCounts(self, refCounts=None): - if refCounts: - ref = refCounts - else: - ref = self.referenceCounts - for i in xrange(len(ref[0])): - print('{0}-{1} & {2:0.3} & {3:0.3} \\\\'.format(self.categories[i],self.categories[i+1],ref[1][i], ref[0][i])) - - -######################### -# maths section -######################### - -# def kernelSmoothing(sampleX, X, Y, weightFunc, halfwidth): -# '''Returns a smoothed weighted version of Y at the predefined values of sampleX -# Sum_x weight(sample_x,x) * y(x)''' -# from numpy import zeros, array -# smoothed = zeros(len(sampleX)) -# for i,x in enumerate(sampleX): -# weights = array([weightFunc(x,xx, halfwidth) for xx in X]) -# if sum(weights)>0: -# smoothed[i] = sum(weights*Y)/sum(weights) -# else: -# smoothed[i] = 0 -# return smoothed - -def kernelSmoothing(x, X, Y, weightFunc, halfwidth): - '''Returns the smoothed estimate of (X,Y) at x - Sum_x weight(sample_x,x) * y(x)''' - weights = array([weightFunc(x,observedx, halfwidth) for observedx in X]) - if sum(weights)>0: - return sum(weights*Y)/sum(weights) - else: - return 0 - -def uniform(center, x, halfwidth): - if abs(center-x)<halfwidth: - return 1. - else: - return 0. - -def gaussian(center, x, halfwidth): - return exp(-((center-x)/halfwidth)**2/2) - -def epanechnikov(center, x, halfwidth): - diff = abs(center-x) - if diff<halfwidth: - return 1.-(diff/halfwidth)**2 - else: - return 0. - -def triangular(center, x, halfwidth): - diff = abs(center-x) - if diff<halfwidth: - return 1.-abs(diff/halfwidth) - else: - return 0. - -def medianSmoothing(x, X, Y, halfwidth): - '''Returns the media of Y's corresponding to X's in the interval [x-halfwidth, x+halfwidth]''' - return median([y for observedx, y in zip(X,Y) if abs(x-observedx)<halfwidth]) - -def argmaxDict(d): - return max(d, key=d.get) - -def deltaFrames(t1, t2, frameRate): - '''Returns the number of frames between t1 and t2 - positive if t1<=t2, negative otherwise''' - if t1 > t2: - return -(t1-t2).seconds*frameRate - else: - return (t2-t1).seconds*frameRate - -def framesToTime(nFrames, frameRate, initialTime = time()): - '''returns a datetime.time for the time in hour, minutes and seconds - initialTime is a datetime.time''' - seconds = int(floor(float(nFrames)/float(frameRate))+initialTime.hour*3600+initialTime.minute*60+initialTime.second) - h = int(floor(seconds/3600.)) - seconds = seconds - h*3600 - m = int(floor(seconds/60)) - seconds = seconds - m*60 - return time(h, m, seconds) - -def timeToFrames(t, frameRate): - return frameRate*(t.hour*3600+t.minute*60+t.second) - -def sortXY(X,Y): - 'returns the sorted (x, Y(x)) sorted on X' - D = {} - for x, y in zip(X,Y): - D[x]=y - xsorted = sorted(D.keys()) - return xsorted, [D[x] for x in xsorted] - -def compareLengthForSort(i, j): - if len(i) < len(j): - return -1 - elif len(i) == len(j): - return 0 - else: - return 1 - -def sortByLength(instances, reverse = False): - '''Returns a new list with the instances sorted by length (method __len__) - reverse is passed to sorted''' - return sorted(instances, key = len, reverse = reverse) - -def ceilDecimals(v, nDecimals): - '''Rounds the number at the nth decimal - eg 1.23 at 0 decimal is 2, at 1 decimal is 1.3''' - tens = 10**nDecimals - return ceil(v*tens)/tens - -def inBetween(bound1, bound2, x): - 'useful if one does not know the order of bound1/bound2' - return bound1 <= x <= bound2 or bound2 <= x <= bound1 - -def pointDistanceL2(x1,y1,x2,y2): - ''' Compute point-to-point distance (L2 norm, ie Euclidean distance)''' - return sqrt((x2-x1)**2+(y2-y1)**2) - -def crossProduct(l1, l2): - return l1[0]*l2[1]-l1[1]*l2[0] - -def cat_mvgavg(cat_list, halfWidth): - ''' Return a list of categories/values smoothed according to a window. - halfWidth is the search radius on either side''' - from copy import deepcopy - smoothed = deepcopy(cat_list) - for point in range(len(cat_list)): - lower_bound_check = max(0,point-halfWidth) - upper_bound_check = min(len(cat_list)-1,point+halfWidth+1) - window_values = cat_list[lower_bound_check:upper_bound_check] - smoothed[point] = max(set(window_values), key=window_values.count) - return smoothed - -def filterMovingWindow(inputSignal, halfWidth): - '''Returns an array obtained after the smoothing of the input by a moving average - The first and last points are copied from the original.''' - width = float(halfWidth*2+1) - win = ones(width,'d') - result = convolve(win/width,array(inputSignal),'same') - result[:halfWidth] = inputSignal[:halfWidth] - result[-halfWidth:] = inputSignal[-halfWidth:] - return result - -def linearRegression(x, y, deg = 1, plotData = False): - '''returns the least square estimation of the linear regression of y = ax+b - as well as the plot''' - from numpy.lib.polynomial import polyfit - from numpy.core.multiarray import arange - coef = polyfit(x, y, deg) - if plotData: - def poly(x): - result = 0 - for i in range(len(coef)): - result += coef[i]*x**(len(coef)-i-1) - return result - plt.plot(x, y, 'x') - xx = arange(min(x), max(x),(max(x)-min(x))/1000) - plt.plot(xx, [poly(z) for z in xx]) - return coef - -def correlation(data, correlationMethod = 'pearson', plotFigure = False, displayNames = None, figureFilename = None): - '''Computes (and displays) the correlation matrix for a pandas DataFrame''' - columns = data.columns.tolist() - for var in data.columns: - uniqueValues = data[var].unique() - if len(uniqueValues) == 1 or data.dtypes[var] == dtype('O') or (len(uniqueValues) == 2 and len(data.loc[~isnan(data[var]), var].unique()) == 1): # last condition: only one other value than nan - columns.remove(var) - c=data[columns].corr(correlationMethod) - if plotFigure: - fig = plt.figure(figsize=(4+0.4*c.shape[0], 0.4*c.shape[0])) - fig.add_subplot(1,1,1) - #plt.imshow(np.fabs(c), interpolation='none') - plt.imshow(c, vmin=-1., vmax = 1., interpolation='none', cmap = 'RdYlBu_r') # coolwarm - if displayNames is not None: - colnames = [displayNames.get(s.strip(), s.strip()) for s in columns] - else: - colnames = columns - #correlation.plot_corr(c, xnames = colnames, normcolor=True, title = filename) - plt.xticks(range(len(colnames)), colnames, rotation=90) - plt.yticks(range(len(colnames)), colnames) - plt.tick_params('both', length=0) - plt.subplots_adjust(bottom = 0.29) - plt.colorbar() - plt.title('Correlation ({})'.format(correlationMethod)) - plt.tight_layout() - if len(colnames) > 50: - plt.subplots_adjust(left=.06) - if figureFilename is not None: - plt.savefig(figureFilename, dpi = 150, transparent = True) - return c - -def addDummies(data, variables, allVariables = True): - '''Add binary dummy variables for each value of a nominal variable - in a pandas DataFrame''' - newVariables = [] - for var in variables: - if var in data.columns and data.dtypes[var] == dtype('O') and len(data[var].unique()) > 2: - values = data[var].unique() - if not allVariables: - values = values[:-1] - for val in values: - if val is not NaN: - newVariable = (var+'_{}'.format(val)).replace('.','').replace(' ','').replace('-','') - data[newVariable] = (data[var] == val) - newVariables.append(newVariable) - return newVariables - -def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, filenamePrefix = None, figureFileType = 'pdf', saveLatex = False, renameVariables = lambda s: s, kwCaption = ''): - '''Studies the influence of (nominal) independent variable over the dependent variable - - Makes tests if the conditional distributions are normal - using the Shapiro-Wilk test (in which case ANOVA could be used) - Implements uses the non-parametric Kruskal Wallis test''' - tmp = data[data[independentVariable].notnull()] - independentVariableValues = sorted(tmp[independentVariable].unique().tolist()) - if len(independentVariableValues) >= 2: - if saveLatex: - from storage import openCheck - out = openCheck(filenamePrefix+'-{}-{}.tex'.format(dependentVariable, independentVariable), 'w') - for x in independentVariableValues: - print('Shapiro-Wilk normality test for {} when {}={}: {} obs'.format(dependentVariable,independentVariable, x, len(tmp.loc[tmp[independentVariable] == x, dependentVariable]))) - if len(tmp.loc[tmp[independentVariable] == x, dependentVariable]) >= 3: - print(shapiro(tmp.loc[tmp[independentVariable] == x, dependentVariable])) - if plotFigure: - plt.figure() - plt.boxplot([tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues]) - plt.xticks(range(1,len(independentVariableValues)+1), independentVariableValues) - plt.title('{} vs {}'.format(dependentVariable, independentVariable)) - if filenamePrefix is not None: - plt.savefig(filenamePrefix+'-{}-{}.{}'.format(dependentVariable, independentVariable, figureFileType)) - table = tmp.groupby([independentVariable])[dependentVariable].describe().unstack().sort(['50%'], ascending = False) - table['count'] = table['count'].astype(int) - testResult = kruskal(*[tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues]) - if saveLatex: - out.write('\\begin{minipage}{\\linewidth}\n' - +'\\centering\n' - +'\\captionof{table}{'+(kwCaption.format(dependentVariable, independentVariable, *testResult))+'}\n' - +table.to_latex(float_format = lambda x: '{:.3f}'.format(x)).encode('ascii')+'\n' - +'\\end{minipage}\n' - +'\\ \\vspace{0.5cm}\n') - else: - print(table) - return testResult - else: - return None - -def prepareRegression(data, dependentVariable, independentVariables, maxCorrelationThreshold, correlations, maxCorrelationP, correlationFunc, stdoutText = ['Removing {} (constant: {})', 'Removing {} (correlation {} with {})', 'Removing {} (no correlation: {}, p={})'], saveFiles = False, filenamePrefix = None, latexHeader = '', latexTable = None, latexFooter=''): - '''Removes variables from candidate independent variables if - - if two independent variables are correlated (> maxCorrelationThreshold), one is removed - - if an independent variable is not correlated with the dependent variable (p>maxCorrelationP) - Returns the remaining non-correlated variables, correlated with the dependent variable - - correlationFunc is spearmanr or pearsonr from scipy.stats - text is the template to display for the two types of printout (see default): 3 elements if no saving to latex file, 8 otherwise - - TODO: pass the dummies for nominal variables and remove if all dummies are correlated, or none is correlated with the dependentvariable''' - from copy import copy - from pandas import DataFrame - result = copy(independentVariables) - table1 = '' - table2 = {} - # constant variables - for var in independentVariables: - uniqueValues = data[var].unique() - if (len(uniqueValues) == 1) or (len(uniqueValues) == 2 and uniqueValues.dtype != dtype('O') and len(data.loc[~isnan(data[var]), var].unique()) == 1): - print(stdoutText[0].format(var, uniqueValues)) - if saveFiles: - table1 += latexTable[0].format(var, *uniqueValues) - result.remove(var) - # correlated variables - for v1 in copy(result): - if v1 in correlations.index: - for v2 in copy(result): - if v2 != v1 and v2 in correlations.index: - if abs(correlations.loc[v1, v2]) > maxCorrelationThreshold: - if v1 in result and v2 in result: - if saveFiles: - table1 += latexTable[1].format(v2, v1, correlations.loc[v1, v2]) - print(stdoutText[1].format(v2, v1, correlations.loc[v1, v2])) - result.remove(v2) - # not correlated with dependent variable - table2['Correlations'] = [] - table2['Valeurs p'] = [] - for var in copy(result): - if data.dtypes[var] != dtype('O'): - cor, p = correlationFunc(data[dependentVariable], data[var]) - if p > maxCorrelationP: - if saveFiles: - table1 += latexTable[2].format(var, cor, p) - print(stdoutText[2].format(var, cor, p)) - result.remove(var) - else: - table2['Correlations'].append(cor) - table2['Valeurs p'].append(p) - - if saveFiles: - from storage import openCheck - out = openCheck(filenamePrefix+'-removed-variables.tex', 'w') - out.write(latexHeader) - out.write(table1) - out.write(latexFooter) - out.close() - out = openCheck(filenamePrefix+'-correlations.html', 'w') - table2['Variables'] = [var for var in result if data.dtypes[var] != dtype('O')] - out.write(DataFrame(table2)[['Variables', 'Correlations', 'Valeurs p']].to_html(formatters = {'Correlations': lambda x: '{:.2f}'.format(x), 'Valeurs p': lambda x: '{:.3f}'.format(x)}, index = False)) - out.close() - return result - -def saveDokMatrix(filename, m, lowerTriangle = False): - 'Saves a dok_matrix using savez' - if lowerTriangle: - keys = [k for k in m if k[0] > k[1]] - savez(filename, shape = m.shape, keys = keys, values = [m[k[0],k[1]] for k in keys]) - else: - savez(filename, shape = m.shape, keys = list(m.keys()), values = list(m.values())) - -def loadDokMatrix(filename): - 'Loads a dok_matrix saved using the above saveDokMatrix' - data = npload(filename) - m = dok_matrix(tuple(data['shape'])) - for k, v in zip(data['keys'], data['values']): - m[tuple(k)] = v - return m - -def aggregationFunction(funcStr, centile = 50): - '''return the numpy function corresponding to funcStr - centile can be a list of centiles to compute at once, eg [25, 50, 75] for the 3 quartiles''' - if funcStr == 'median': - return median - elif funcStr == 'mean': - return mean - elif funcStr == 'centile': - return lambda x: percentile(x, centile) - elif funcStr == '85centile': - return lambda x: percentile(x, 85) - else: - print('Unknown aggregation method: {}'.format(funcStr)) - return None - -######################### -# regression analysis using statsmodels (and pandas) -######################### - -# TODO make class for experiments? -# TODO add tests with public dataset downloaded from Internet (IRIS et al) -def modelString(experiment, dependentVariable, independentVariables): - return dependentVariable+' ~ '+' + '.join([independentVariable for independentVariable in independentVariables if experiment[independentVariable]]) - -def runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols'): - import statsmodels.formula.api as smf - modelStr = modelString(experiment, dependentVariable, independentVariables) - if regressionType == 'ols': - model = smf.ols(modelStr, data = data) - elif regressionType == 'gls': - model = smf.gls(modelStr, data = data) - elif regressionType == 'rlm': - model = smf.rlm(modelStr, data = data) - else: - print('Unknown regression type {}. Exiting'.format(regressionType)) - import sys - sys.exit() - return model.fit() - -def runModels(experiments, data, dependentVariable, independentVariables, regressionType = 'ols'): - '''Runs several models and stores 3 statistics - adjusted R2, condition number (should be small, eg < 1000) - and p-value for Shapiro-Wilk test of residual normality''' - for i,experiment in experiments.iterrows(): - if experiment[independentVariables].any(): - results = runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols') - experiments.loc[i,'r2adj'] = results.rsquared_adj - experiments.loc[i,'condNum'] = results.condition_number - experiments.loc[i, 'shapiroP'] = shapiro(results.resid)[1] - experiments.loc[i,'nobs'] = int(results.nobs) - return experiments - -def generateExperiments(independentVariables): - '''Generates all possible models for including or not each independent variable''' - from pandas import DataFrame - experiments = {} - nIndependentVariables = len(independentVariables) - if nIndependentVariables != len(set(independentVariables)): - print("Duplicate variables. Exiting") - import sys - sys.exit() - nModels = 2**nIndependentVariables - for i,var in enumerate(independentVariables): - pattern = [False]*(2**i)+[True]*(2**i) - experiments[var] = pattern*(2**(nIndependentVariables-i-1)) - experiments = DataFrame(experiments) - experiments['r2adj'] = 0. - experiments['condNum'] = NaN - experiments['shapiroP'] = -1 - experiments['nobs'] = -1 - return experiments - -def findBestModel(data, dependentVariable, independentVariables, regressionType = 'ols', nProcesses = 1): - '''Generates all possible model with the independentVariables - and runs them, saving the results in experiments - with multiprocess option''' - from pandas import concat - from multiprocessing import Pool - experiments = generateExperiments(independentVariables) - nModels = len(experiments) - print("Running {} models with {} processes".format(nModels, nProcesses)) - print("IndependentVariables: {}".format(independentVariables)) - if nProcesses == 1: - return runModels(experiments, data, dependentVariable, independentVariables, regressionType) - else: - pool = Pool(processes = nProcesses) - chunkSize = int(ceil(nModels/nProcesses)) - jobs = [pool.apply_async(runModels, args = (experiments[i*chunkSize:(i+1)*chunkSize], data, dependentVariable, independentVariables, regressionType)) for i in range(nProcesses)] - return concat([job.get() for job in jobs]) - -def findBestModelFwd(data, dependentVariable, independentVariables, modelFunc, experiments = None): - '''Forward search for best model (based on adjusted R2) - Randomly starting with one variable and adding randomly variables - if they improve the model - - The results are added to experiments if provided as argument - Storing in experiment relies on the index being the number equal - to the binary code derived from the independent variables''' - from numpy.random import permutation as nppermutation - if experiments is None: - experiments = generateExperiments(independentVariables) - nIndependentVariables = len(independentVariables) - permutation = nppermutation(list(range(nIndependentVariables))) - variableMapping = {j: independentVariables[i] for i,j in enumerate(permutation)} - print('Tested variables '+', '.join([variableMapping[i] for i in range(nIndependentVariables)])) - bestModel = [False]*nIndependentVariables - currentVarNum = 0 - currentR2Adj = 0. - for currentVarNum in range(nIndependentVariables): - currentModel = [i for i in bestModel] - currentModel[currentVarNum] = True - rowIdx = sum([0]+[2**i for i in range(nIndependentVariables) if currentModel[permutation[i]]]) - #print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in range(nIndependentVariables) if currentModel[permutation[i]]]) - if experiments.loc[rowIdx, 'shapiroP'] < 0: - modelStr = modelString(experiments.loc[rowIdx], dependentVariable, independentVariables) - model = modelFunc(modelStr, data = data) - results = model.fit() - experiments.loc[rowIdx, 'r2adj'] = results.rsquared_adj - experiments.loc[rowIdx, 'condNum'] = results.condition_number - experiments.loc[rowIdx, 'shapiroP'] = shapiro(results.resid)[1] - experiments.loc[rowIdx, 'nobs'] = int(results.nobs) - if currentR2Adj < experiments.loc[rowIdx, 'r2adj']: - currentR2Adj = experiments.loc[rowIdx, 'r2adj'] - bestModel[currentVarNum] = True - return experiments - -def displayModelResults(results, model = None, plotFigures = True, filenamePrefix = None, figureFileType = 'pdf', text = {'title-shapiro': 'Shapiro-Wilk normality test for residuals: {:.2f} (p={:.3f})', 'true-predicted.xlabel': 'Predicted values', 'true-predicted.ylabel': 'True values', 'residuals-predicted.xlabel': 'Predicted values', 'residuals-predicted.ylabel': 'Residuals'}): - import statsmodels.api as sm - '''Displays some model results - - 3 graphics, true-predicted, residuals-predicted, ''' - print(results.summary()) - shapiroResult = shapiro(results.resid) - print(shapiroResult) - if plotFigures: - fig = plt.figure(figsize=(7,6.3*(2+int(model is not None)))) - if model is not None: - ax = fig.add_subplot(3,1,1) - plt.plot(results.predict(), model.endog, 'x') - x=plt.xlim() - y=plt.ylim() - plt.plot([max(x[0], y[0]), min(x[1], y[1])], [max(x[0], y[0]), min(x[1], y[1])], 'r') - #plt.axis('equal') - if text is not None: - plt.title(text['title-shapiro'].format(*shapiroResult)) - #plt.title(text['true-predicted.title']) - plt.xlabel(text['true-predicted.xlabel']) - plt.ylabel(text['true-predicted.ylabel']) - fig.add_subplot(3,1,2, sharex = ax) - plt.plot(results.predict(), results.resid, 'x') - nextSubplotNum = 3 - else: - fig.add_subplot(2,1,1) - plt.plot(results.predict(), results.resid, 'x') - nextSubplotNum = 2 - if text is not None: - if model is None: - plt.title(text['title-shapiro'].format(*shapiroResult)) - plt.xlabel(text['residuals-predicted.xlabel']) - plt.ylabel(text['residuals-predicted.ylabel']) - qqAx = fig.add_subplot(nextSubplotNum,1,nextSubplotNum) - sm.qqplot(results.resid, fit = True, line = '45', ax = qqAx) - plt.axis('equal') - if text is not None and 'qqplot.xlabel' in text: - plt.xlabel(text['qqplot.xlabel']) - plt.ylabel(text['qqplot.ylabel']) - plt.tight_layout() - if filenamePrefix is not None: - from storage import openCheck - out = openCheck(filenamePrefix+'-coefficients.html', 'w') - out.write(results.summary().as_html()) - plt.savefig(filenamePrefix+'-model-results.'+figureFileType) - -######################### -# iterable section -######################### - -def mostCommon(L): - '''Returns the most frequent element in a iterable - - taken from http://stackoverflow.com/questions/1518522/python-most-common-element-in-a-list''' - from itertools import groupby - from operator import itemgetter - # get an iterable of (item, iterable) pairs - SL = sorted((x, i) for i, x in enumerate(L)) - # print 'SL:', SL - groups = groupby(SL, key=itemgetter(0)) - # auxiliary function to get "quality" for an item - def _auxfun(g): - item, iterable = g - count = 0 - min_index = len(L) - for _, where in iterable: - count += 1 - min_index = min(min_index, where) - # print 'item %r, count %r, minind %r' % (item, count, min_index) - return count, -min_index - # pick the highest-count/earliest item - return max(groups, key=_auxfun)[0] - -######################### -# sequence section -######################### - -class LCSS(object): - '''Class that keeps the LCSS parameters - and puts together the various computations - - the methods with names starting with _ are not to be shadowed - in child classes, who will shadow the other methods, - ie compute and computeXX methods''' - def __init__(self, similarityFunc = None, metric = None, epsilon = None, delta = float('inf'), aligned = False, lengthFunc = min): - '''One should provide either a similarity function - that indicates (return bool) whether elements in the compares lists are similar - - eg distance(p1, p2) < epsilon - - or a type of metric usable in scipy.spatial.distance.cdist with an epsilon''' - if similarityFunc is None and metric is None: - print("No way to compute LCSS, similarityFunc and metric are None. Exiting") - import sys - sys.exit() - elif metric is not None and epsilon is None: - print("Please provide a value for epsilon if using a cdist metric. Exiting") - import sys - sys.exit() - else: - if similarityFunc is None and metric is not None and not isinf(delta): - print('Warning: you are using a cdist metric and a finite delta, which will make probably computation slower than using the equivalent similarityFunc (since all pairwise distances will be computed by cdist).') - self.similarityFunc = similarityFunc - self.metric = metric - self.epsilon = epsilon - self.aligned = aligned - self.delta = delta - self.lengthFunc = lengthFunc - self.subSequenceIndices = [(0,0)] - - def similarities(self, l1, l2, jshift=0): - n1 = len(l1) - n2 = len(l2) - self.similarityTable = zeros((n1+1,n2+1), dtype = npint) - if self.similarityFunc is not None: - for i in range(1,n1+1): - for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): - if self.similarityFunc(l1[i-1], l2[j-1]): - self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 - else: - self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1]) - elif self.metric is not None: - similarElements = distance.cdist(l1, l2, self.metric) <= self.epsilon - for i in range(1,n1+1): - for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): - if similarElements[i-1, j-1]: - self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 - else: - self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1]) - - - def subSequence(self, i, j): - '''Returns the subsequence of two sequences - http://en.wikipedia.org/wiki/Longest_common_subsequence_problem''' - if i == 0 or j == 0: - return [] - elif self.similarityTable[i][j] == self.similarityTable[i][j-1]: - return self.subSequence(i, j-1) - elif self.similarityTable[i][j] == self.similarityTable[i-1][j]: - return self.subSequence(i-1, j) - else: - return self.subSequence(i-1, j-1) + [(i-1,j-1)] - - def _compute(self, _l1, _l2, computeSubSequence = False): - '''returns the longest common subsequence similarity - l1 and l2 should be the right format - eg list of tuple points for cdist - or elements that can be compare using similarityFunc - - if aligned, returns the best matching if using a finite delta by shifting the series alignments - ''' - if len(_l2) < len(_l1): # l1 is the shortest - l1 = _l2 - l2 = _l1 - revertIndices = True - else: - l1 = _l1 - l2 = _l2 - revertIndices = False - n1 = len(l1) - n2 = len(l2) - - if self.aligned: - lcssValues = {} - similarityTables = {} - for i in range(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2 - self.similarities(l1, l2, i) - lcssValues[i] = self.similarityTable.max() - similarityTables[i] = self.similarityTable - #print self.similarityTable - alignmentShift = argmaxDict(lcssValues) # ideally get the medium alignment shift, the one that minimizes distance - self.similarityTable = similarityTables[alignmentShift] - else: - alignmentShift = 0 - self.similarities(l1, l2) - - # threshold values for the useful part of the similarity table are n2-n1-delta and n1-n2-delta - self.similarityTable = self.similarityTable[:min(n1, n2+alignmentShift+self.delta)+1, :min(n2, n1-alignmentShift+self.delta)+1] - - if computeSubSequence: - self.subSequenceIndices = self.subSequence(self.similarityTable.shape[0]-1, self.similarityTable.shape[1]-1) - if revertIndices: - self.subSequenceIndices = [(j,i) for i,j in self.subSequenceIndices] - return self.similarityTable[-1,-1] - - def compute(self, l1, l2, computeSubSequence = False): - '''get methods are to be shadowed in child classes ''' - return self._compute(l1, l2, computeSubSequence) - - def computeAlignment(self): - return mean([j-i for i,j in self.subSequenceIndices]) - - def _computeNormalized(self, l1, l2, computeSubSequence = False): - ''' compute the normalized LCSS - ie, the LCSS divided by the min or mean of the indicator lengths (using lengthFunc) - lengthFunc = lambda x,y:float(x,y)/2''' - return float(self._compute(l1, l2, computeSubSequence))/self.lengthFunc(len(l1), len(l2)) - - def computeNormalized(self, l1, l2, computeSubSequence = False): - return self._computeNormalized(l1, l2, computeSubSequence) - - def _computeDistance(self, l1, l2, computeSubSequence = False): - ''' compute the LCSS distance''' - return 1-self._computeNormalized(l1, l2, computeSubSequence) - - def computeDistance(self, l1, l2, computeSubSequence = False): - return self._computeDistance(l1, l2, computeSubSequence) - -######################### -# plotting section -######################### - -def plotPolygon(poly, options = '', **kwargs): - 'Plots shapely polygon poly' - from matplotlib.pyplot import plot - x,y = poly.exterior.xy - plot(x, y, options, **kwargs) - -def stepPlot(X, firstX, lastX, initialCount = 0, increment = 1): - '''for each value in X, increment by increment the initial count - returns the lists that can be plotted - to obtain a step plot increasing by one for each value in x, from first to last value - firstX and lastX should be respectively smaller and larger than all elements in X''' - - sortedX = [] - counts = [initialCount] - for x in sorted(X): - sortedX += [x,x] - counts.append(counts[-1]) - counts.append(counts[-1]+increment) - counts.append(counts[-1]) - return [firstX]+sortedX+[lastX], counts - -class PlottingPropertyValues(object): - def __init__(self, values): - self.values = values - - def __getitem__(self, i): - return self.values[i%len(self.values)] - -markers = PlottingPropertyValues(['+', '*', ',', '.', 'x', 'D', 's', 'o']) -scatterMarkers = PlottingPropertyValues(['s','o','^','>','v','<','d','p','h','8','+','x']) - -linestyles = PlottingPropertyValues(['-', '--', '-.', ':']) - -colors = PlottingPropertyValues('brgmyck') # 'w' - -def monochromeCycler(withMarker = False): - from cycler import cycler - if withMarker: - monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', ':', '-.']) * cycler('marker', ['^',',', '.'])) - else: - monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', ':', '-.'])) - plt.rc('axes', prop_cycle=monochrome) - -def plotIndicatorMap(indicatorMap, squareSize, masked = True, defaultValue=-1): - from matplotlib.pyplot import pcolor - coords = array(list(indicatorMap.keys())) - minX = min(coords[:,0]) - minY = min(coords[:,1]) - X = arange(minX, max(coords[:,0])+1.1)*squareSize - Y = arange(minY, max(coords[:,1])+1.1)*squareSize - C = defaultValue*ones((len(Y), len(X))) - for k,v in indicatorMap.items(): - C[k[1]-minY,k[0]-minX] = v - if masked: - pcolor(X, Y, ma.masked_where(C==defaultValue,C)) - else: - pcolor(X, Y, C) - -######################### -# Data download -######################### - -def downloadECWeather(stationID, years, months = [], outputDirectoryname = '.', english = True): - '''Downloads monthly weather data from Environment Canada - If month is provided (number 1 to 12), it means hourly data for the whole month - Otherwise, means the data for each day, for the whole year - - Example: MONTREAL MCTAVISH 10761 - MONTREALPIERRE ELLIOTT TRUDEAU INTL A 5415 - see ftp://client_climate@ftp.tor.ec.gc.ca/Pub/Get_More_Data_Plus_de_donnees/Station%20Inventory%20EN.csv - - To get daily data for 2010 and 2011, downloadECWeather(10761, [2010,2011], [], '/tmp') - To get hourly data for 2009 and 2012, January, March and October, downloadECWeather(10761, [2009,2012], [1,3,10], '/tmp') - - for annee in `seq 2016 2017`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&timeframe=2&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done - for annee in `seq 2016 2017`;do for mois in `seq 1 12`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&Month=${mois}&timeframe=1&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done;done - ''' - import urllib.request - if english: - language = 'e' - else: - language = 'f' - if len(months) == 0: - timeFrame = 2 - months = [1] - else: - timeFrame = 1 - - for year in years: - for month in months: - outFilename = '{}/{}-{}'.format(outputDirectoryname, stationID, year) - if timeFrame == 1: - outFilename += '-{}-hourly'.format(month) - else: - outFilename += '-daily' - outFilename += '.csv' - url = urllib.request.urlretrieve('http://climate.weather.gc.ca/climate_data/bulk_data_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=Download+Data'.format(language, stationID, year, month, timeFrame), outFilename) - -######################### -# File I/O -######################### - -def removeExtension(filename, delimiter = '.'): - '''Returns the filename minus the extension (all characters after last .)''' - i = filename.rfind(delimiter) - if i>0: - return filename[:i] - else: - return filename - -def getExtension(filename, delimiter = '.'): - '''Returns the filename minus the extension (all characters after last .)''' - i = filename.rfind(delimiter) - if i>0: - return filename[i+1:] - else: - return '' - -def cleanFilename(s): - 'cleans filenames obtained when contatenating figure characteristics' - return s.replace(' ','-').replace('.','').replace('/','-').replace(',','') - -def getRelativeFilename(parentPath, filename): - 'Returns filename if absolute, otherwise parentPath/filename as string' - filePath = Path(filename) - if filePath.is_absolute(): - return filename - else: - return str(parentPath/filePath) - -def listfiles(dirname, extension, remove = False): - '''Returns the list of files with the extension in the directory dirname - If remove is True, the filenames are stripped from the extension''' - d = Path(dirname) - if d.is_dir(): - tmp = [str(f) for f in d.glob('*.extension')] - if remove: - return [removeExtension(f, extension) for f in tmp] - else: - return tmp - else: - print(dirname+' is not a directory') - return [] - -def mkdir(dirname): - 'Creates a directory if it does not exist' - p = Path(dirname) - if not p.exists(): - p.mkdir() - else: - print(dirname+' already exists') - -def removeFile(filename): - '''Deletes the file while avoiding raising an error - if the file does not exist''' - f = Path(filename) - if (f.exists()): - f.unlink() - else: - print(filename+' does not exist') - -def line2Floats(l, separator=' '): - '''Returns the list of floats corresponding to the string''' - return [float(x) for x in l.split(separator)] - -def line2Ints(l, separator=' '): - '''Returns the list of ints corresponding to the string''' - return [int(x) for x in l.split(separator)] - -######################### -# Profiling -######################### - -def analyzeProfile(profileFilename, stripDirs = True): - '''Analyze the file produced by cProfile - - obtained by for example: - - call in script (for main() function in script) - import cProfile, os - cProfile.run('main()', os.path.join(os.getcwd(),'main.profile')) - - - or on the command line: - python -m cProfile [-o profile.bin] [-s sort] scriptfile [arg]''' - import pstats, os - p = pstats.Stats(os.path.join(os.pardir, profileFilename)) - if stripDirs: - p.strip_dirs() - p.sort_stats('time') - p.print_stats(.2) - #p.sort_stats('time') - # p.print_callees(.1, 'int_prediction.py:') - return p - -######################### -# running tests -######################### - -if __name__ == "__main__": - import doctest - import unittest - suite = doctest.DocFileSuite('tests/utils.txt') - #suite = doctest.DocTestSuite() - unittest.TextTestRunner().run(suite) - #doctest.testmod() - #doctest.testfile("example.txt")
--- a/run-tests.sh Fri Jun 15 11:18:43 2018 -0400 +++ b/run-tests.sh Fri Jun 15 11:19:10 2018 -0400 @@ -1,7 +1,7 @@ #!/bin/sh echo "------------" echo "Python tests" -cd python +cd trafficintelligence ./run-tests.sh cd .. echo "------------"
--- a/scripts/classify-objects.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/classify-objects.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,13 +1,13 @@ #! /usr/bin/env python3 -import cvutils, moving, ml, storage, utils +import sys, argparse import numpy as np -import sys, argparse -#from cv2.ml import SVM_RBF, SVM_C_SVC import cv2 from scipy.stats import norm, lognorm +from trafficintelligence import cvutils, moving, ml, storage, utils + # TODO add mode detection live, add choice of kernel and svm type (to be saved in future classifier format) parser = argparse.ArgumentParser(description='The program processes indicators for all pairs of road users in the scene')
--- a/scripts/compute-clearmot.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/compute-clearmot.py Fri Jun 15 11:19:10 2018 -0400 @@ -3,7 +3,8 @@ import sys, argparse from numpy import loadtxt from numpy.linalg import inv -import moving, storage, cvutils + +from trafficintelligence import moving, storage, cvutils # TODO: need to trim objects to same mask ?
--- a/scripts/compute-homography.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/compute-homography.py Fri Jun 15 11:19:10 2018 -0400 @@ -6,7 +6,7 @@ import numpy as np import cv2 -import cvutils, utils, storage +from trafficintelligence import cvutils, utils, storage # TODO add option to use RANSAC or other robust homography estimation method?
--- a/scripts/create-bounding-boxes.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/create-bounding-boxes.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,11 +2,11 @@ import argparse -import storage - from numpy.linalg.linalg import inv from numpy import loadtxt +from trafficintelligence import storage + parser = argparse.ArgumentParser(description='The program creates bounding boxes in image space around all features (for display and for comparison to ground truth in the form of bouding boxes.') parser.add_argument('-d', dest = 'databaseFilename', help = 'name of the Sqlite database file', required = True) parser.add_argument('-o', dest = 'homography', help = 'name of the image to world homography')
--- a/scripts/create-metadata.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/create-metadata.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,7 +2,8 @@ import argparse from datetime import datetime -import metadata, utils + +from trafficintelligence import metadata, utils timeConverter = utils.TimeConverter()
--- a/scripts/delete-tables.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/delete-tables.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,8 +2,7 @@ import sys, argparse -import utils -import storage +from trafficintelligence import utils, storage parser = argparse.ArgumentParser(description='The program deletes (drops) the tables in the database before saving new results (for objects, tables object_features and objects are dropped; for interactions, the tables interactions and indicators are dropped') #parser.add_argument('configFilename', help = 'name of the configuration file')
--- a/scripts/display-synced-trajectories.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/display-synced-trajectories.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,10 +2,12 @@ import sys, argparse, os.path from datetime import datetime, timedelta + import numpy as np import cv2 -import cvutils, utils, storage -from metadata import connectDatabase, Site, CameraView, VideoSequence + +from trafficintelligence import cvutils, utils, storage +from trafficintelligence.metadata import connectDatabase, Site, CameraView, VideoSequence parser = argparse.ArgumentParser(description='The program displays several views of the same site synchronously.') parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
--- a/scripts/display-trajectories.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/display-trajectories.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,11 +2,11 @@ import sys, argparse -import storage, cvutils, utils - from numpy.linalg import inv from numpy import loadtxt +from trafficintelligence import storage, cvutils, utils + parser = argparse.ArgumentParser(description='The program displays feature or object trajectories overlaid over the video frames.', epilog = 'Either the configuration filename or the other parameters (at least video and database filenames) need to be provided.') parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') parser.add_argument('-d', dest = 'databaseFilename', help = 'name of the Sqlite database file (overrides the configuration file)')
--- a/scripts/extract-appearance-images.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/extract-appearance-images.py Fri Jun 15 11:19:10 2018 -0400 @@ -5,7 +5,7 @@ from pandas import read_csv from matplotlib.pyplot import imshow, figure -import cvutils, moving, ml, storage +from trafficintelligence import cvutils, moving, ml, storage parser = argparse.ArgumentParser(description='The program extracts labeled image patches to train the HoG-SVM classifier, and optionnally speed information') parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file', required = True)
--- a/scripts/extract-camera-parameters.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/extract-camera-parameters.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,7 +2,7 @@ import argparse -import storage, cvutils +from trafficintelligence import storage, cvutils parser = argparse.ArgumentParser(description='The program extracts the intrinsic camera from the tacal camera calibration file used by T-Analyst (http://www.tft.lth.se/en/research/video-analysis/co-operation/software/t-analyst/).') parser.add_argument('-i', dest = 'filename', help = 'filename of the camera calibration (.tacal)', required = True)
--- a/scripts/info-video.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/info-video.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,8 +1,8 @@ #! /usr/bin/env python3 import sys, argparse -import cvutils +from trafficintelligence import cvutils parser = argparse.ArgumentParser(description='The program displays the video.') parser.add_argument('-i', dest = 'videoFilename', help = 'name of the video file', required = True)
--- a/scripts/init-tracking.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/init-tracking.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,10 +1,13 @@ #! /usr/bin/env python3 -import sys, argparse, os.path, storage, utils +import sys, argparse, os.path from shutil import copy + from cvutils import getImagesFromVideo from matplotlib.pyplot import imsave +from trafficintelligence import storage, utils + # could try to guess the video # check if there is already a tracking.cfg file
--- a/scripts/learn-motion-patterns.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/learn-motion-patterns.py Fri Jun 15 11:19:10 2018 -0400 @@ -2,10 +2,9 @@ import sys, argparse -#import matplotlib.pyplot as plt import numpy as np -import ml, utils, storage, moving +from trafficintelligence import ml, utils, storage, moving parser = argparse.ArgumentParser(description='The program learns prototypes for the motion patterns') #, epilog = '' #parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
--- a/scripts/learn-poi.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/learn-poi.py Fri Jun 15 11:19:10 2018 -0400 @@ -6,7 +6,7 @@ from sklearn import mixture import matplotlib.pyplot as plt -import storage, ml +from trafficintelligence import storage, ml parser = argparse.ArgumentParser(description='The program learns and displays Gaussians fit to beginnings and ends of object trajectories (based on Mohamed Gomaa Mohamed 2015 PhD).') parser.add_argument('-d', dest = 'databaseFilename', help = 'name of the Sqlite database file', required = True)
--- a/scripts/merge-features.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/merge-features.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,10 +1,11 @@ #! /usr/bin/env python3 import sys, argparse, os.path, sqlite3 -import cvutils, utils, moving, storage -from metadata import connectDatabase, Site, VideoSequence, CameraView, getSite from datetime import datetime, timedelta +from trafficintelligence import cvutils, utils, moving, storage +from trafficintelligence.metadata import connectDatabase, Site, VideoSequence, CameraView, getSite + timeConverter = utils.TimeConverter() parser = argparse.ArgumentParser(description='The program merges feature trajectories recorded from the same site synchronously between start and end time.')
--- a/scripts/performance-db.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/performance-db.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,8 +1,8 @@ #! /usr/bin/env python3 -import sys, shutil, os, sqlite3, timeit#, argparse +import sys, shutil, os, sqlite3, timeit -import storage +from trafficintelligence import storage if len(sys.argv) >= 2: dbFilename = sys.argv[1]
--- a/scripts/play-synced-videos.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/play-synced-videos.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,10 +1,11 @@ #! /usr/bin/env python3 import sys, argparse, os.path -import cvutils, utils -from metadata import connectDatabase, Site, CameraView, VideoSequence, getSite from datetime import datetime, timedelta +from trafficintelligence import cvutils, utils +from trafficintelligence.metadata import connectDatabase, Site, CameraView, VideoSequence, getSite + timeConverter = utils.TimeConverter() parser = argparse.ArgumentParser(description='The program displays several views of the same site synchronously.')
--- a/scripts/play-video.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/play-video.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,8 +1,8 @@ #! /usr/bin/env python3 import sys, argparse -import cvutils +from trafficintelligence import cvutils parser = argparse.ArgumentParser(description='The program displays the video.') parser.add_argument('-i', dest = 'videoFilename', help = 'name of the video file', required = True)
--- a/scripts/polytracktopdtv.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/polytracktopdtv.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,14 +1,12 @@ #! /usr/bin/env python3 +import sys, os, datetime, argparse +import shutil, sqlite3, zipfile + +import cv2 from pdtv import TsaiCamera, ZipVideo, SyncedVideos, TrackSet, Track, State -import sys, os, datetime, argparse -import shutil -import sqlite3 -import zipfile -import utils -import cvutils -import cv2 +from trafficintelligence import utils, cvutils def zipFolder(inputFolder, outputFile): '''Method to compress the content of the inputFolder in the outputFile'''
--- a/scripts/process.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/process.py Fri Jun 15 11:19:10 2018 -0400 @@ -10,8 +10,8 @@ from numpy import percentile from pandas import DataFrame -import storage, events, prediction, cvutils, utils -from metadata import * +from trafficintelligence import storage, events, prediction, cvutils, utils +from trafficintelligence.metadata import * parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') # input
--- a/scripts/rescale-homography.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/rescale-homography.py Fri Jun 15 11:19:10 2018 -0400 @@ -6,8 +6,7 @@ import numpy as np import cv2 -import cvutils -import utils +from trafficintelligence import cvutils, utils if len(sys.argv) < 4: print('Usage: {} homography_filename original_size new_size (size can be width or height)'.format(sys.argv[0]))
--- a/scripts/safety-analysis.py Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/safety-analysis.py Fri Jun 15 11:19:10 2018 -0400 @@ -1,13 +1,13 @@ #! /usr/bin/env python3 -import storage, prediction, events, moving - import sys, argparse, random from multiprocessing import Pool import matplotlib.pyplot as plt import numpy as np +from trafficintelligence import storage, prediction, events, moving + # todo: very slow if too many predicted trajectories # add computation of probality of unsucessful evasive action
--- a/scripts/setup-tracking.sh Fri Jun 15 11:18:43 2018 -0400 +++ b/scripts/setup-tracking.sh Fri Jun 15 11:19:10 2018 -0400 @@ -6,7 +6,6 @@ sudo apt -qq install build-essential checkinstall cmake pkg-config yasm libtiff5-dev libjpeg-dev libjasper-dev libavcodec-dev libavformat-dev libswscale-dev libgstreamer0.10-dev libgstreamer-plugins-base0.10-dev libv4l-dev python-dev libtbb-dev libgtk2.0-dev libfaac-dev libmp3lame-dev libtheora-dev libvorbis-dev libxvidcore-dev x264 # libdc1394-22-dev libxine-dev python-numpy libqt4-dev libopencore-amrnb-dev libopencore-amrwb-dev v4l-utils ffmpeg libboost-all-dev sudo apt -qq install libavfilter-dev libboost-dev libboost-program-options-dev libboost-graph-dev python-pip sqlite3 libsqlite3-dev cmake-qt-gui libgeos-dev -sudo -H pip install --upgrade mercurial numpy matplotlib scikit-image scikit-learn echo "Installing OpenCV" $version cd mkdir OpenCV @@ -36,6 +35,9 @@ cmake . make TrajectoryManagementAndAnalysis cd +wget https://bootstrap.pypa.io/get-pip.py +sudo -H python3 get-pip.py +sudo -H pip3 install -r trafficintelligence/python-requirements.txt --upgrade cd trafficintelligence/c/ make feature-based-tracking cd
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name='trafficintelligence', + version=0.2, + description='Python modules of the Traffic Intelligence project', + author='Nicolas Saunier', + author_email='nicolas.saunier@polymtl.ca', + url='https://bitbucket.org/Nicolas/trafficintelligence', + packages=['trafficintelligence'] + #py_modules = ['moving', 'utils'] + #scripts=['helloworld'] +)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/base.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,7 @@ +'''Module for few base classes to avoid issues of circular import''' + +class VideoFilenameAddable(object): + 'Base class with the capability to attach a video filename' + + def setVideoFilename(self, videoFilename): + self.videoFilename = videoFilename
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/cvutils.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,663 @@ +#! /usr/bin/env python +'''Image/Video utilities''' + +from trafficintelligence import utils, moving + +try: + import cv2 + opencvAvailable = True +except ImportError: + print('OpenCV library could not be loaded (video replay functions will not be available)') # TODO change to logging module + opencvAvailable = False +try: + import skimage + skimageAvailable = True +except ImportError: + print('Scikit-image library could not be loaded (HoG-based classification methods will not be available)') + skimageAvailable = False + +from sys import stdout +from os import listdir +from subprocess import run +from math import floor, log10, ceil + +from numpy import dot, array, append, float32, loadtxt, savetxt, append, zeros, ones, identity, abs as npabs, logical_and, unravel_index, sum as npsum, isnan, mgrid, median, floor as npfloor, ceil as npceil +from numpy.linalg import inv +from matplotlib.mlab import find +from matplotlib.pyplot import imread, imsave + +videoFilenameExtensions = ['mov', 'avi', 'mp4', 'MOV', 'AVI', 'MP4'] +trackerExe = 'feature-based-tracking' +#importaggdraw # agg on top of PIL (antialiased drawing) + +cvRed = {'default': (0,0,255), + 'colorblind': (0,114,178)} +cvGreen = {'default': (0,255,0), + 'colorblind': (0,158,115)} +cvBlue = {'default': (255,0,0), + 'colorblind': (213,94,0)} +cvCyan = {'default': (255, 255, 0), + 'colorblind': (240,228,66)} +cvYellow = {'default': (0, 255, 255), + 'colorblind': (86,180,233)} +cvMagenta = {'default': (255, 0, 255), + 'colorblind': (204,121,167)} +cvWhite = {k: (255, 255, 255) for k in ['default', 'colorblind']} +cvBlack = {k: (0,0,0) for k in ['default', 'colorblind']} + +cvColors3 = {k: utils.PlottingPropertyValues([cvRed[k], cvGreen[k], cvBlue[k]]) for k in ['default', 'colorblind']} +cvColors = {k: utils.PlottingPropertyValues([cvRed[k], cvGreen[k], cvBlue[k], cvCyan[k], cvYellow[k], cvMagenta[k], cvWhite[k], cvBlack[k]]) for k in ['default', 'colorblind']} + +def quitKey(key): + return chr(key&255)== 'q' or chr(key&255) == 'Q' + +def saveKey(key): + return chr(key&255) == 's' + +def int2FOURCC(x): + fourcc = '' + for i in range(4): + fourcc += chr((x >> 8*i)&255) + return fourcc + +def rgb2gray(rgb): + return dot(rgb[...,:3], [0.299, 0.587, 0.144]) + +def matlab2PointCorrespondences(filename): + '''Loads and converts the point correspondences saved + by the matlab camera calibration tool''' + points = loadtxt(filename, delimiter=',') + savetxt(utils.removeExtension(filename)+'-point-correspondences.txt',append(points[:,:2].T, points[:,3:].T, axis=0)) + +def loadPointCorrespondences(filename): + '''Loads and returns the corresponding points in world (first 2 lines) and image spaces (last 2 lines)''' + points = loadtxt(filename, dtype=float32) + return (points[:2,:].T, points[2:,:].T) # (world points, image points) + +def cvMatToArray(cvmat): + '''Converts an OpenCV CvMat to numpy array.''' + print('Deprecated, use new interface') + a = zeros((cvmat.rows, cvmat.cols))#array([[0.0]*cvmat.width]*cvmat.height) + for i in range(cvmat.rows): + for j in range(cvmat.cols): + a[i,j] = cvmat[i,j] + return a + +def createWhiteImage(height, width, filename): + img = ones((height, width, 3), uint8)*255 + imsave(filename, img) + +if opencvAvailable: + def computeHomography(srcPoints, dstPoints, method=0, ransacReprojThreshold=3.0): + '''Returns the homography matrix mapping from srcPoints to dstPoints (dimension Nx2)''' + H, mask = cv2.findHomography(srcPoints, dstPoints, method, ransacReprojThreshold) + return H + + def cvPlot(img, positions, color, lastCoordinate = None, **kwargs): + if lastCoordinate is None: + last = positions.length()-1 + elif lastCoordinate >=0: + last = min(positions.length()-1, lastCoordinate) + for i in range(0, last): + cv2.line(img, positions[i].asint().astuple(), positions[i+1].asint().astuple(), color, **kwargs) + + def cvImshow(windowName, img, rescale = 1.0): + 'Rescales the image (in particular if too large)' + from cv2 import resize + if rescale != 1.: + size = (int(round(img.shape[1]*rescale)), int(round(img.shape[0]*rescale))) + resizedImg = resize(img, size) + cv2.imshow(windowName, resizedImg) + else: + cv2.imshow(windowName, img) + + def computeUndistortMaps(width, height, undistortedImageMultiplication, intrinsicCameraMatrix, distortionCoefficients): + newImgSize = (int(round(width*undistortedImageMultiplication)), int(round(height*undistortedImageMultiplication))) + newCameraMatrix = cv2.getDefaultNewCameraMatrix(intrinsicCameraMatrix, newImgSize, True) + return cv2.initUndistortRectifyMap(intrinsicCameraMatrix, array(distortionCoefficients), None, newCameraMatrix, newImgSize, cv2.CV_32FC1), newCameraMatrix + + def playVideo(filenames, windowNames = None, firstFrameNums = None, frameRate = -1, interactive = False, printFrames = True, text = None, rescale = 1., step = 1, colorBlind = False): + '''Plays the video(s)''' + if colorBlind: + colorType = 'colorblind' + else: + colorType = 'default' + if len(filenames) == 0: + print('Empty filename list') + return + if windowNames is None: + windowNames = ['frame{}'.format(i) for i in range(len(filenames))] + wait = 5 + if rescale == 1.: + for windowName in windowNames: + cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) + if frameRate > 0: + wait = int(round(1000./frameRate)) + if interactive: + wait = 0 + captures = [cv2.VideoCapture(fn) for fn in filenames] + if array([cap.isOpened() for cap in captures]).all(): + key = -1 + ret = True + nFramesShown = 0 + if firstFrameNums is not None: + for i in range(len(captures)): + captures[i].set(cv2.CAP_PROP_POS_FRAMES, firstFrameNums[i]) + while ret and not quitKey(key): + rets = [] + images = [] + for cap in captures: + ret, img = cap.read() + rets.append(ret) + images.append(img) + ret = array(rets).all() + if ret: + if printFrames: + print('frame shown {0}'.format(nFramesShown)) + for i in range(len(filenames)): + if text is not None: + cv2.putText(images[i], text, (10,50), cv2.FONT_HERSHEY_PLAIN, 1, cvRed[colorType]) + cvImshow(windowNames[i], images[i], rescale) # cv2.imshow('frame', img) + key = cv2.waitKey(wait) + if saveKey(key): + cv2.imwrite('image-{}.png'.format(frameNum), img) + nFramesShown += step + if step > 1: + for i in range(len(captures)): + captures[i].set(cv2.CAP_PROP_POS_FRAMES, firstFrameNums[i]+nFramesShown) + cv2.destroyAllWindows() + else: + print('Video captures for {} failed'.format(filenames)) + + def infoVideo(filename): + '''Provides all available info on video ''' + cvPropertyNames = {cv2.CAP_PROP_FORMAT: "format", + cv2.CAP_PROP_FOURCC: "codec (fourcc)", + cv2.CAP_PROP_FPS: "fps", + cv2.CAP_PROP_FRAME_COUNT: "number of frames", + cv2.CAP_PROP_FRAME_HEIGHT: "heigh", + cv2.CAP_PROP_FRAME_WIDTH: "width", + cv2.CAP_PROP_RECTIFICATION: "rectification", + cv2.CAP_PROP_SATURATION: "saturation"} + capture = cv2.VideoCapture(filename) + videoProperties = {} + if capture.isOpened(): + for cvprop in [#cv2.CAP_PROP_BRIGHTNESS + #cv2.CAP_PROP_CONTRAST + #cv2.CAP_PROP_CONVERT_RGB + #cv2.CAP_PROP_EXPOSURE + cv2.CAP_PROP_FORMAT, + cv2.CAP_PROP_FOURCC, + cv2.CAP_PROP_FPS, + cv2.CAP_PROP_FRAME_COUNT, + cv2.CAP_PROP_FRAME_HEIGHT, + cv2.CAP_PROP_FRAME_WIDTH, + #cv2.CAP_PROP_GAIN, + #cv2.CAP_PROP_HUE + #cv2.CAP_PROP_MODE + #cv2.CAP_PROP_POS_AVI_RATIO + #cv2.CAP_PROP_POS_FRAMES + #cv2.CAP_PROP_POS_MSEC + #cv2.CAP_PROP_RECTIFICATION, + #cv2.CAP_PROP_SATURATION + ]: + prop = capture.get(cvprop) + if cvprop == cv2.CAP_PROP_FOURCC and prop > 0: + prop = int2FOURCC(int(prop)) + videoProperties[cvPropertyNames[cvprop]] = prop + else: + print('Video capture for {} failed'.format(filename)) + return videoProperties + + def getImagesFromVideo(videoFilename, firstFrameNum = 0, lastFrameNum = 1, step = 1, saveImage = False, outputPrefix = 'image'): + '''Returns nFrames images from the video sequence''' + images = [] + capture = cv2.VideoCapture(videoFilename) + if capture.isOpened(): + rawCount = capture.get(cv2.CAP_PROP_FRAME_COUNT) + if rawCount < 0: + rawCount = lastFrameNum+1 + nDigits = int(floor(log10(rawCount)))+1 + ret = False + capture.set(cv2.CAP_PROP_POS_FRAMES, firstFrameNum) + frameNum = firstFrameNum + while frameNum<lastFrameNum and frameNum<rawCount: + ret, img = capture.read() + i = 0 + while not ret and i<10: + ret, img = capture.read() + i += 1 + if img is not None and img.size>0: + if saveImage: + frameNumStr = format(frameNum, '0{}d'.format(nDigits)) + cv2.imwrite(outputPrefix+frameNumStr+'.png', img) + else: + images.append(img) + frameNum +=step + if step > 1: + capture.set(cv2.CAP_PROP_POS_FRAMES, frameNum) + capture.release() + else: + print('Video capture for {} failed'.format(videoFilename)) + return images + + def getFPS(videoFilename): + capture = cv2.VideoCapture(videoFilename) + if capture.isOpened(): + fps = capture.get(cv2.CAP_PROP_FPS) + capture.release() + return fps + else: + print('Video capture for {} failed'.format(videoFilename)) + return None + + def imageBoxSize(obj, frameNum, width, height, px = 0.2, py = 0.2): + 'Computes the bounding box size of object at frameNum' + x = [] + y = [] + if obj.hasFeatures(): + for f in obj.getFeatures(): + if f.existsAtInstant(frameNum): + p = f.getPositionAtInstant(frameNum) + x.append(p.x) + y.append(p.y) + xmin = min(x) + xmax = max(x) + ymin = min(y) + ymax = max(y) + xMm = px * (xmax - xmin) + yMm = py * (ymax - ymin) + a = max(ymax - ymin + (2 * yMm), xmax - (xmin + 2 * xMm)) + yCropMin = int(max(0, .5 * (ymin + ymax - a))) + yCropMax = int(min(height - 1, .5 * (ymin + ymax + a))) + xCropMin = int(max(0, .5 * (xmin + xmax - a))) + xCropMax = int(min(width - 1, .5 * (xmin + xmax + a))) + return yCropMin, yCropMax, xCropMin, xCropMax + + def imageBox(img, obj, frameNum, width, height, px = 0.2, py = 0.2, minNPixels = 800): + 'Computes the bounding box of object at frameNum' + yCropMin, yCropMax, xCropMin, xCropMax = imageBoxSize(obj, frameNum, width, height, px, py) + if yCropMax != yCropMin and xCropMax != xCropMin and (yCropMax - yCropMin) * (xCropMax - xCropMin) > minNPixels: + return img[yCropMin : yCropMax, xCropMin : xCropMax] + else: + return None + + def tracking(configFilename, grouping, videoFilename = None, dbFilename = None, homographyFilename = None, maskFilename = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, dryRun = False): + '''Runs the tracker in a subprocess + if grouping is True, it is feature grouping + otherwise it is feature tracking''' + if grouping: + trackingMode = '--gf' + else: + trackingMode = '--tf' + cmd = [trackerExe, configFilename, trackingMode, '--quiet'] + + if videoFilename is not None: + cmd += ['--video-filename', videoFilename] + if dbFilename is not None: + cmd += ['--database-filename', dbFilename] + if homographyFilename is not None: + cmd += ['--homography-filename', homographyFilename] + if maskFilename is not None: + cmd += ['--mask-filename', maskFilename] + if undistort: + cmd += ['--undistort', 'true'] + if intrinsicCameraMatrix is not None: # we currently have to save a file + from time import time + intrinsicCameraFilename = '/tmp/intrinsic-{}.txt'.format(time()) + savetxt(intrinsicCameraFilename, intrinsicCameraMatrix) + cmd += ['--intrinsic-camera-filename', intrinsicCameraFilename] + if distortionCoefficients is not None: + cmd += ['--distortion-coefficients '+' '.join([str(x) for x in distortionCoefficients])] + if dryRun: + print(cmd) + else: + run(cmd) + + def displayTrajectories(videoFilename, objects, boundingBoxes = {}, homography = None, firstFrameNum = 0, lastFrameNumArg = None, printFrames = True, rescale = 1., nFramesStep = 1, saveAllImages = False, nZerosFilenameArg = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., annotations = [], gtMatches = {}, toMatches = {}, colorBlind = False): + '''Displays the objects overlaid frame by frame over the video ''' + if colorBlind: + colorType = 'colorblind' + else: + colorType = 'default' + + capture = cv2.VideoCapture(videoFilename) + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + windowName = 'frame' + if rescale == 1.: + cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) + + if undistort: # setup undistortion + [map1, map2], newCameraMatrix = computeUndistortMaps(width, height, undistortedImageMultiplication, intrinsicCameraMatrix, distortionCoefficients) + if capture.isOpened(): + key = -1 + ret = True + frameNum = firstFrameNum + capture.set(cv2.CAP_PROP_POS_FRAMES, firstFrameNum) + if lastFrameNumArg is None: + lastFrameNum = float("inf") + else: + lastFrameNum = lastFrameNumArg + if nZerosFilenameArg is None: + if lastFrameNumArg is None: + nZerosFilename = int(ceil(log10(objects[-1].getLastInstant()))) + else: + nZerosFilename = int(ceil(log10(lastFrameNum))) + else: + nZerosFilename = nZerosFilenameArg + while ret and not quitKey(key) and frameNum <= lastFrameNum: + ret, img = capture.read() + if ret: + if undistort: + img = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR) + if printFrames: + print('frame {0}'.format(frameNum)) + # plot objects + for obj in objects[:]: + if obj.existsAtInstant(frameNum): + if not hasattr(obj, 'projectedPositions'): + obj.projectedPositions = obj.getPositions().homographyProject(homography) + if undistort: + obj.projectedPositions = obj.projectedPositions.newCameraProject(newCameraMatrix) + cvPlot(img, obj.projectedPositions, cvColors[colorType][obj.getNum()], frameNum-obj.getFirstInstant()) + if frameNum not in boundingBoxes and obj.hasFeatures(): + yCropMin, yCropMax, xCropMin, xCropMax = imageBoxSize(obj, frameNum, homography, width, height) + cv2.rectangle(img, (xCropMin, yCropMin), (xCropMax, yCropMax), cvBlue[colorType], 1) + objDescription = '{} '.format(obj.num) + if moving.userTypeNames[obj.userType] != 'unknown': + objDescription += moving.userTypeNames[obj.userType][0].upper() + if len(annotations) > 0: # if we loaded annotations, but there is no match + if frameNum not in toMatches[obj.getNum()]: + objDescription += " FA" + cv2.putText(img, objDescription, obj.projectedPositions[frameNum-obj.getFirstInstant()].asint().astuple(), cv2.FONT_HERSHEY_PLAIN, 1, cvColors[colorType][obj.getNum()]) + if obj.getLastInstant() == frameNum: + objects.remove(obj) + # plot object bounding boxes + if frameNum in boundingBoxes: + for rect in boundingBoxes[frameNum]: + cv2.rectangle(img, rect[0].asint().astuple(), rect[1].asint().astuple(), cvColors[colorType][obj.getNum()]) + # plot ground truth + if len(annotations) > 0: + for gt in annotations: + if gt.existsAtInstant(frameNum): + if frameNum in gtMatches[gt.getNum()]: + color = cvColors[colorType][gtMatches[gt.getNum()][frameNum]] # same color as object + else: + color = cvRed[colorType] + cv2.putText(img, 'Miss', gt.topLeftPositions[frameNum-gt.getFirstInstant()].asint().astuple(), cv2.FONT_HERSHEY_PLAIN, 1, color) + cv2.rectangle(img, gt.topLeftPositions[frameNum-gt.getFirstInstant()].asint().astuple(), gt.bottomRightPositions[frameNum-gt.getFirstInstant()].asint().astuple(), color) + # saving images and going to next + if not saveAllImages: + cvImshow(windowName, img, rescale) + key = cv2.waitKey() + if saveAllImages or saveKey(key): + cv2.imwrite('image-{{:0{}}}.png'.format(nZerosFilename).format(frameNum), img) + frameNum += nFramesStep + if nFramesStep > 1: + capture.set(cv2.CAP_PROP_POS_FRAMES, frameNum) + cv2.destroyAllWindows() + else: + print('Cannot load file ' + videoFilename) + + def computeHomographyFromPDTV(camera): + '''Returns the homography matrix at ground level from PDTV camera + https://bitbucket.org/hakanardo/pdtv''' + # camera = pdtv.load(cameraFilename) + srcPoints = [[x,y] for x, y in zip([1.,2.,2.,1.],[1.,1.,2.,2.])] # need floats!! + dstPoints = [] + for srcPoint in srcPoints: + projected = camera.image_to_world(tuple(srcPoint)) + dstPoints.append([projected[0], projected[1]]) + H, mask = cv2.findHomography(array(srcPoints), array(dstPoints), method = 0) # No need for different methods for finding homography + return H + + def getIntrinsicCameraMatrix(cameraData): + return array([[cameraData['f']*cameraData['Sx']/cameraData['dx'], 0, cameraData['Cx']], + [0, cameraData['f']/cameraData['dy'], cameraData['Cy']], + [0, 0, 1.]]) + + def getDistortionCoefficients(cameraData): + return array([cameraData['k']]+4*[0]) + + def undistortedCoordinates(map1, map2, x, y, maxDistance = 1.): + '''Returns the coordinates of a point in undistorted image + map1 and map2 are the mapping functions from undistorted image + to distorted (original image) + map1(x,y) = originalx, originaly''' + distx = npabs(map1-x) + disty = npabs(map2-y) + indices = logical_and(distx<maxDistance, disty<maxDistance) + closeCoordinates = unravel_index(find(indices), distx.shape) # returns i,j, ie y,x + xWeights = 1-distx[indices] + yWeights = 1-disty[indices] + return dot(xWeights, closeCoordinates[1])/npsum(xWeights), dot(yWeights, closeCoordinates[0])/npsum(yWeights) + + def undistortTrajectoryFromCVMapping(map1, map2, t): + '''test 'perfect' inversion''' + undistortedTrajectory = moving.Trajectory() + for i,p in enumerate(t): + res = undistortedCoordinates(map1, map2, p.x,p.y) + if not isnan(res).any(): + undistortedTrajectory.addPositionXY(res[0], res[1]) + else: + print('{} {} {}'.format(i,p,res)) + return undistortedTrajectory + + def computeInverseMapping(originalImageSize, map1, map2): + 'Computes inverse mapping from maps provided by cv2.initUndistortRectifyMap' + invMap1 = -ones(originalImageSize) + invMap2 = -ones(originalImageSize) + for x in range(0,originalImageSize[1]): + for y in range(0,originalImageSize[0]): + res = undistortedCoordinates(x,y, map1, map2) + if not isnan(res).any(): + invMap1[y,x] = res[0] + invMap2[y,x] = res[1] + return invMap1, invMap2 + + def intrinsicCameraCalibration(path, checkerBoardSize=[6,7], secondPassSearch=False, display=False, fixK2 = True, fixK3 = True, zeroTangent = True): + ''' Camera calibration searches through all the images (jpg or png) located + in _path_ for matches to a checkerboard pattern of size checkboardSize. + These images should all be of the same camera with the same resolution. + + For best results, use an asymetric board and ensure that the image has + very high contrast, including the background. + + cherckerBoardSize is the number of internal corners (7x10 squares have 6x9 internal corners) + + The code below is based off of: + https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_calib3d/py_calibration/py_calibration.html + Modified by Paul St-Aubin + ''' + import glob, os + + # termination criteria + criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) + + # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0) + objp = zeros((checkerBoardSize[0]*checkerBoardSize[1],3), float32) + objp[:,:2] = mgrid[0:checkerBoardSize[1],0:checkerBoardSize[0]].T.reshape(-1,2) + + # Arrays to store object points and image points from all the images. + objpoints = [] # 3d point in real world space + imgpoints = [] # 2d points in image plane. + + ## Loop throuhg all images in _path_ + images = glob.glob(os.path.join(path,'*.[jJ][pP][gG]'))+glob.glob(os.path.join(path,'*.[jJ][pP][eE][gG]'))+glob.glob(os.path.join(path,'*.[pP][nN][gG]')) + for fname in images: + img = cv2.imread(fname) + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + # Find the chess board corners + ret, corners = cv2.findChessboardCorners(gray, (checkerBoardSize[1],checkerBoardSize[0]), None) + + # If found, add object points, image points (after refining them) + if ret: + print('Found pattern in '+fname) + + if secondPassSearch: + corners = cv2.cornerSubPix(gray, corners, (11,11), (-1,-1), criteria) + + objpoints.append(objp) + imgpoints.append(corners) + + # Draw and display the corners + if display: + cv2.drawChessboardCorners(img, (checkerBoardSize[1],checkerBoardSize[0]), corners, ret) + if img is not None: + cv2.imshow('img',img) + cv2.waitKey(0) + else: + print('Pattern not found in '+fname) + ## Close up image loading and calibrate + cv2.destroyAllWindows() + if len(objpoints) == 0 or len(imgpoints) == 0: + return None + try: + flags = 0 + if fixK2: + flags += cv2.CALIB_FIX_K2 + if fixK3: + flags += cv2.CALIB_FIX_K3 + if zeroTangent: + flags += cv2.CALIB_ZERO_TANGENT_DIST + ret, camera_matrix, dist_coeffs, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None, flags = flags) + except NameError: + return None + savetxt('intrinsic-camera.txt', camera_matrix) + print('error: {}'.format(ret)) + return camera_matrix, dist_coeffs + + def undistortImage(img, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., interpolation=cv2.INTER_LINEAR): + '''Undistorts the image passed in argument''' + width = img.shape[1] + height = img.shape[0] + [map1, map2] = computeUndistortMaps(width, height, undistortedImageMultiplication, intrinsicCameraMatrix, distortionCoefficients) + return cv2.remap(img, map1, map2, interpolation=interpolation) + +def homographyProject(points, homography, output3D = False): + '''Returns the coordinates of the points (2xN array) projected through homography''' + if points.shape[0] != 2: + raise Exception('points of dimension {}'.format(points.shape)) + + if homography is not None and homography.size>0: + if output3D: + outputDim = 3 + else: + outputDim = 2 + augmentedPoints = append(points,[[1]*points.shape[1]], 0) # 3xN + prod = dot(homography, augmentedPoints) + return prod[:outputDim,:]/prod[2] + elif output3D: + return append(points,[[1]*points.shape[1]], 0) # 3xN + else: + return points + +def imageToWorldProject(points, intrinsicCameraMatrix = None, distortionCoefficients = None, homography = None): + '''Projects points (2xN array) from image (video) space to world space + 1. through undistorting if provided by intrinsic camera matrix and distortion coefficients + 2. through homograph projection (from ideal point (no camera) to world)''' + if points.shape[0] != 2: + raise Exception('points of dimension {}'.format(points.shape)) + + if intrinsicCameraMatrix is not None and distortionCoefficients is not None: + undistortedPoints = cv2.undistortPoints(points.T.reshape(1,points.shape[1], 2), intrinsicCameraMatrix, distortionCoefficients).reshape(-1,2) + return homographyProject(undistortedPoints.T, homography) + else: + return homographyProject(points, homography) + +def worldToImageProject(points, intrinsicCameraMatrix = None, distortionCoefficients = None, homography = None): + '''Projects points (2xN array) from image (video) space to world space + 1. through undistorting if provided by intrinsic camera matrix and distortion coefficients + 2. through homograph projection (from ideal point (no camera) to world)''' + if points.shape[0] != 2: + raise Exception('points of dimension {}'.format(points.shape)) + + if intrinsicCameraMatrix is not None and distortionCoefficients is not None: + projected3D = homographyProject(points, homography, True) + projected, jacobian = cv2.projectPoints(projected3D.T, (0.,0.,0.), (0.,0.,0.), intrinsicCameraMatrix, distortionCoefficients) # in: 3xN, out: 2x1xN + return projected.reshape(-1,2).T + else: + return homographyProject(points, homography) + +def newCameraProject(points, newCameraMatrix): + '''Projects points (2xN array) as if seen by camera + (or reverse by inverting the camera matrix)''' + if points.shape[0] != 2: + raise Exception('points of dimension {}'.format(points.shape)) + + if newCameraMatrix is not None: + augmentedPoints = append(points,[[1]*points.shape[1]], 0) # 3xN + projected = dot(newCameraMatrix, augmentedPoints) + return projected[:2,:] + else: + return points + +if opencvAvailable: + def computeTranslation(img1, img2, img1Points, maxTranslation2, minNMatches, windowSize = (5,5), level = 5, criteria = (cv2.TERM_CRITERIA_EPS, 0, 0.01)): + '''Computes the translation of img2 with respect to img1 + (loaded using OpenCV as numpy arrays) + img1Points are used to compute the translation + + TODO add diagnostic if data is all over the place, and it most likely is not a translation (eg zoom, other non linear distortion)''' + + nextPoints = array([]) + (img2Points, status, track_error) = cv2.calcOpticalFlowPyrLK(img1, img2, img1Points, nextPoints, winSize=windowSize, maxLevel=level, criteria=criteria) + # calcOpticalFlowPyrLK(prevImg, nextImg, prevPts[, nextPts[, status[, err[, winSize[, maxLevel[, criteria[, derivLambda[, flags]]]]]]]]) -> nextPts, status, err + delta = [] + for (k, (p1,p2)) in enumerate(zip(img1Points, img2Points)): + if status[k] == 1: + dp = p2-p1 + d = npsum(dp**2) + if d < maxTranslation2: + delta.append(dp) + if len(delta) >= minNMatches: + return median(delta, axis=0) + else: + print(dp) + return None + +if skimageAvailable: + from skimage.feature import hog + from skimage import color, transform + + def HOG(image, rescaleSize = (64, 64), orientations = 9, pixelsPerCell = (8,8), cellsPerBlock = (2,2), blockNorm = 'L1', visualize = False, transformSqrt = False): + bwImg = color.rgb2gray(image) + inputImg = transform.resize(bwImg, rescaleSize) + features = hog(inputImg, orientations, pixelsPerCell, cellsPerBlock, blockNorm, visualize, transformSqrt, True) + if visualize: + from matplotlib.pyplot import imshow, figure, subplot + hogViz = features[1] + features = features[0] + figure() + subplot(1,2,1) + imshow(inputImg) + subplot(1,2,2) + imshow(hogViz) + return float32(features) + + def createHOGTrainingSet(imageDirectory, classLabel, rescaleSize = (64,64), orientations = 9, pixelsPerCell = (8,8), blockNorm = 'L1', cellsPerBlock = (2, 2), visualize = False, transformSqrt = False): + inputData = [] + for filename in listdir(imageDirectory): + img = imread(imageDirectory+filename) + features = HOG(img, rescaleSize, orientations, pixelsPerCell, cellsPerBlock, blockNorm, visualize, transformSqrt) + inputData.append(features) + + nImages = len(inputData) + return array(inputData, dtype = float32), array([classLabel]*nImages) + + +######################### +# running tests +######################### + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/cvutils.txt') + #suite = doctest.DocTestSuite() + unittest.TextTestRunner().run(suite) + #doctest.testmod() + #doctest.testfile("example.txt")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/events.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,341 @@ +#! /usr/bin/env python +'''Libraries for events +Interactions, pedestrian crossing...''' + +from trafficintelligence import moving, prediction, indicators, utils, cvutils, ml +from trafficintelligence.base import VideoFilenameAddable + +import numpy as np + +import multiprocessing +import itertools, logging + + +def findRoute(prototypes,objects,i,j,noiseEntryNums,noiseExitNums,minSimilarity= 0.3, spatialThreshold=1.0, delta=180): + if i[0] not in noiseEntryNums: + prototypesRoutes= [ x for x in sorted(prototypes.keys()) if i[0]==x[0]] + elif i[1] not in noiseExitNums: + prototypesRoutes=[ x for x in sorted(prototypes.keys()) if i[1]==x[1]] + else: + prototypesRoutes=[x for x in sorted(prototypes.keys())] + routeSim={} + lcss = utils.LCSS(similarityFunc=lambda x,y: (distanceForLCSS(x,y) <= spatialThreshold),delta=delta) + for y in prototypesRoutes: + if y in prototypes: + prototypesIDs=prototypes[y] + similarity=[] + for x in prototypesIDs: + s=lcss.computeNormalized(objects[j].positions, objects[x].positions) + similarity.append(s) + routeSim[y]=max(similarity) + route=max(routeSim, key=routeSim.get) + if routeSim[route]>=minSimilarity: + return route + else: + return i + +def getRoute(obj,prototypes,objects,noiseEntryNums,noiseExitNums,useDestination=True): + route=(obj.startRouteID,obj.endRouteID) + if useDestination: + if route not in prototypes: + route= findRoute(prototypes,objects,route,obj.getNum(),noiseEntryNums,noiseExitNums) + return route + +class Interaction(moving.STObject, VideoFilenameAddable): + '''Class for an interaction between two road users + or a road user and an obstacle + + link to the moving objects + contains the indicators in a dictionary with the names as keys + ''' + + categories = {'Head On': 0, + 'rearend': 1, + 'side': 2, + 'parallel': 3} + + indicatorNames = ['Collision Course Dot Product', + 'Collision Course Angle', + 'Distance', + 'Minimum Distance', + 'Velocity Angle', + 'Speed Differential', + 'Collision Probability', + 'Time to Collision', # 7 + 'Probability of Successful Evasive Action', + 'predicted Post Encroachment Time', + 'Post Encroachment Time'] + + indicatorNameToIndices = utils.inverseEnumeration(indicatorNames) + + indicatorShortNames = ['CCDP', + 'CCA', + 'Dist', + 'MinDist', + 'VA', + 'SD', + 'PoC', + 'TTC', + 'P(SEA)', + 'pPET', + 'PET'] + + indicatorUnits = ['', + 'rad', + 'm', + 'm', + 'rad', + 'm/s', + '', + 's', + '', + 's', + 's'] + + timeIndicators = ['Time to Collision', 'predicted Post Encroachment Time'] + + def __init__(self, num = None, timeInterval = None, roaduserNum1 = None, roaduserNum2 = None, roadUser1 = None, roadUser2 = None, categoryNum = None): + moving.STObject.__init__(self, num, timeInterval) + if timeInterval is None and roadUser1 is not None and roadUser2 is not None: + self.timeInterval = roadUser1.commonTimeInterval(roadUser2) + self.roadUser1 = roadUser1 + self.roadUser2 = roadUser2 + if roaduserNum1 is not None and roaduserNum2 is not None: + self.roadUserNumbers = set([roaduserNum1, roaduserNum2]) + elif roadUser1 is not None and roadUser2 is not None: + self.roadUserNumbers = set([roadUser1.getNum(), roadUser2.getNum()]) + else: + self.roadUserNumbers = None + self.categoryNum = categoryNum + self.indicators = {} + self.interactionInterval = None + # list for collison points and crossing zones + self.collisionPoints = None + self.crossingZones = None + + def getRoadUserNumbers(self): + return self.roadUserNumbers + + def setRoadUsers(self, objects): + nums = sorted(list(self.getRoadUserNumbers())) + if nums[0]<len(objects) and objects[nums[0]].getNum() == nums[0]: + self.roadUser1 = objects[nums[0]] + if nums[1]<len(objects) and objects[nums[1]].getNum() == nums[1]: + self.roadUser2 = objects[nums[1]] + + if self.roadUser1 is None or self.roadUser2 is None: + self.roadUser1 = None + self.roadUser2 = None + i = 0 + while i < len(objects) and self.roadUser2 is None: + if objects[i].getNum() in nums: + if self.roadUser1 is None: + self.roadUser1 = objects[i] + else: + self.roadUser2 = objects[i] + i += 1 + + def getIndicator(self, indicatorName): + return self.indicators.get(indicatorName, None) + + def addIndicator(self, indicator): + if indicator is not None: + self.indicators[indicator.name] = indicator + + def getIndicatorValueAtInstant(self, indicatorName, instant): + indicator = self.getIndicator(indicatorName) + if indicator is not None: + return indicator[instant] + else: + return None + + def getIndicatorValuesAtInstant(self, instant): + '''Returns list of indicator values at instant + as dict (with keys from indicators dict)''' + values = {} + for k, indicator in self.indicators.items(): + values[k] = indicator[instant] + return values + + def plot(self, options = '', withOrigin = False, timeStep = 1, withFeatures = False, **kwargs): + self.roadUser1.plot(options, withOrigin, timeStep, withFeatures, **kwargs) + self.roadUser2.plot(options, withOrigin, timeStep, withFeatures, **kwargs) + + def plotOnWorldImage(self, nPixelsPerUnitDistance, options = '', withOrigin = False, timeStep = 1, **kwargs): + self.roadUser1.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, **kwargs) + self.roadUser2.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, **kwargs) + + def play(self, videoFilename, homography = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1., allUserInstants = False): + if self.roadUser1 is not None and self.roadUser2 is not None: + if allUserInstants: + firstFrameNum = min(self.roadUser1.getFirstInstant(), self.roadUser2.getFirstInstant()) + lastFrameNum = max(self.roadUser1.getLastInstant(), self.roadUser2.getLastInstant()) + else: + firstFrameNum = self.getFirstInstant() + lastFrameNum = self.getLastInstant() + cvutils.displayTrajectories(videoFilename, [self.roadUser1, self.roadUser2], homography = homography, firstFrameNum = firstFrameNum, lastFrameNumArg = lastFrameNum, undistort = undistort, intrinsicCameraMatrix = intrinsicCameraMatrix, distortionCoefficients = distortionCoefficients, undistortedImageMultiplication = undistortedImageMultiplication) + else: + print('Please set the interaction road user attributes roadUser1 and roadUser1 through the method setRoadUsers') + + def computeIndicators(self): + '''Computes the collision course cosine only if the cosine is positive''' + collisionCourseDotProducts = {}#[0]*int(self.timeInterval.length()) + collisionCourseAngles = {} + velocityAngles = {} + distances = {}#[0]*int(self.timeInterval.length()) + speedDifferentials = {} + interactionInstants = [] + for instant in self.timeInterval: + deltap = self.roadUser1.getPositionAtInstant(instant)-self.roadUser2.getPositionAtInstant(instant) + v1 = self.roadUser1.getVelocityAtInstant(instant) + v2 = self.roadUser2.getVelocityAtInstant(instant) + deltav = v2-v1 + velocityAngles[instant] = np.arccos(moving.Point.dot(v1, v2)/(v1.norm2()*v2.norm2())) + collisionCourseDotProducts[instant] = moving.Point.dot(deltap, deltav) + distances[instant] = deltap.norm2() + speedDifferentials[instant] = deltav.norm2() + if collisionCourseDotProducts[instant] > 0: + interactionInstants.append(instant) + if distances[instant] != 0 and speedDifferentials[instant] != 0: + collisionCourseAngles[instant] = np.arccos(collisionCourseDotProducts[instant]/(distances[instant]*speedDifferentials[instant])) + + if len(interactionInstants) >= 2: + self.interactionInterval = moving.TimeInterval(interactionInstants[0], interactionInstants[-1]) + else: + self.interactionInterval = moving.TimeInterval() + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[0], collisionCourseDotProducts)) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[1], collisionCourseAngles)) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[2], distances, mostSevereIsMax = False)) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[4], velocityAngles)) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[5], speedDifferentials)) + + # if we have features, compute other indicators + if self.roadUser1.hasFeatures() and self.roadUser2.hasFeatures(): + minDistances={} + for instant in self.timeInterval: + minDistances[instant] = moving.MovingObject.minDistance(self.roadUser1, self.roadUser2, instant) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[3], minDistances, mostSevereIsMax = False)) + + def computeCrossingsCollisions(self, predictionParameters, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, timeInterval = None): + '''Computes all crossing and collision points at each common instant for two road users. ''' + TTCs = {} + collisionProbabilities = {} + if timeInterval is not None: + commonTimeInterval = timeInterval + else: + commonTimeInterval = self.timeInterval + self.collisionPoints, crossingZones = predictionParameters.computeCrossingsCollisions(self.roadUser1, self.roadUser2, collisionDistanceThreshold, timeHorizon, computeCZ, debug, commonTimeInterval) + for i, cps in self.collisionPoints.items(): + TTCs[i] = prediction.SafetyPoint.computeExpectedIndicator(cps) + collisionProbabilities[i] = sum([p.probability for p in cps]) + if len(TTCs) > 0: + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[7], TTCs, mostSevereIsMax=False)) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[6], collisionProbabilities)) + + # crossing zones and pPET + if computeCZ: + self.crossingZones = crossingZones + pPETs = {} + for i, cz in self.crossingZones.items(): + pPETs[i] = prediction.SafetyPoint.computeExpectedIndicator(cz) + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[9], pPETs, mostSevereIsMax=False)) + # TODO add probability of collision, and probability of successful evasive action + + def computePET(self, collisionDistanceThreshold): + pet, t1, t2= moving.MovingObject.computePET(self.roadUser1, self.roadUser2, collisionDistanceThreshold) + if pet is not None: + self.addIndicator(indicators.SeverityIndicator(Interaction.indicatorNames[10], {min(t1, t2): pet}, mostSevereIsMax = False)) + + def setCollision(self, collision): + '''indicates if it is a collision: argument should be boolean''' + self.collision = collision + + def isCollision(self): + if hasattr(self, 'collision'): + return self.collision + else: + return None + + def getCollisionPoints(self): + return self.collisionPoints + + def getCrossingZones(self): + return self.crossingZones + +def createInteractions(objects, _others = None): + '''Create all interactions of two co-existing road users''' + if _others is not None: + others = _others + + interactions = [] + num = 0 + for i in range(len(objects)): + if _others is None: + others = objects[:i] + for j in range(len(others)): + commonTimeInterval = objects[i].commonTimeInterval(others[j]) + if not commonTimeInterval.empty(): + interactions.append(Interaction(num, commonTimeInterval, objects[i].num, others[j].num, objects[i], others[j])) + num += 1 + return interactions + +def findInteraction(interactions, roadUserNum1, roadUserNum2): + 'Returns the right interaction in the set' + i=0 + while i<len(interactions) and set([roadUserNum1, roadUserNum2]) != interactions[i].getRoadUserNumbers(): + i+=1 + if i<len(interactions): + return interactions[i] + else: + return None + +def computeIndicators(interactions, computeMotionPrediction, computePET, predictionParameters, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, timeInterval = None): + for inter in interactions: + print('processing interaction {}'.format(inter.getNum())) # logging.debug('processing interaction {}'.format(inter.getNum())) + inter.computeIndicators() + if computeMotionPrediction: + inter.computeCrossingsCollisions(predictionParameters, collisionDistanceThreshold, timeHorizon, computeCZ, debug, timeInterval) + if computePET: + inter.computePET(collisionDistanceThreshold) + return interactions + +def aggregateSafetyPoints(interactions, pointType = 'collision'): + '''Put all collision points or crossing zones in a list for display''' + allPoints = [] + if pointType == 'collision': + for i in interactions: + for points in i.collisionPoints.values(): + allPoints += points + elif pointType == 'crossing': + for i in interactions: + for points in i.crossingZones.values(): + allPoints += points + else: + print('unknown type of point: '+pointType) + return allPoints + +def prototypeCluster(interactions, similarities, indicatorName, minSimilarity, similarityFunc = None, minClusterSize = None, randomInitialization = False): + return ml.prototypeCluster([inter.getIndicator(indicatorName) for inter in interactions], similarities, minSimilarity, similarityFunc, minClusterSize, randomInitialization) + +class Crossing(moving.STObject): + '''Class for the event of a street crossing + + TODO: detecter passage sur la chaussee + identifier origines et destination (ou uniquement chaussee dans FOV) + carac traversee + detecter proximite veh (retirer si trop similaire simultanement + carac interaction''' + + def __init__(self, roaduserNum = None, num = None, timeInterval = None): + moving.STObject.__init__(self, num, timeInterval) + self.roaduserNum = roaduserNum + + + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/events.txt') + #suite = doctest.DocTestSuite() + unittest.TextTestRunner().run(suite) +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/indicators.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,258 @@ +#! /usr/bin/env python +'''Class for indicators, temporal indicators, and safety indicators''' + +from trafficintelligence import moving +#import matplotlib.nxutils as nx +from matplotlib.pyplot import plot, ylim +from matplotlib.pylab import find +from numpy import array, arange, mean, floor, mean +from scipy import percentile + +def multivariateName(indicatorNames): + return '_'.join(indicatorNames) + +# need for a class representing the indicators, their units, how to print them in graphs... +class TemporalIndicator(object): + '''Class for temporal indicators + i.e. indicators that take a value at specific instants + + values should be + * a dict, for the values at specific time instants + * or a list with a time interval object if continuous measurements + + it should have more information like name, unit''' + + def __init__(self, name, values, timeInterval = None, maxValue = None): + self.name = name + if timeInterval is None: + self.values = values + instants = sorted(self.values.keys()) + if len(instants) > 0: + self.timeInterval = moving.TimeInterval(instants[0], instants[-1]) + else: + self.timeInterval = moving.TimeInterval() + else: + assert len(values) == timeInterval.length() + self.timeInterval = timeInterval + self.values = {} + for i in range(int(round(self.timeInterval.length()))): + self.values[self.timeInterval[i]] = values[i] + self.maxValue = maxValue + + def __len__(self): + return len(self.values) + + def empty(self): + return len(self.values) == 0 + + def __getitem__(self, t): + 'Returns the value at time t' + return self.values.get(t) + + def getIthValue(self, i): + sortedKeys = sorted(self.values.keys()) + if 0<=i<len(sortedKeys): + return self.values[sortedKeys[i]] + else: + return None + + def __iter__(self): + self.iterInstantNum = 0 # index in the interval or keys of the dict + return self + + def __next__(self): + if self.iterInstantNum >= len(self.values):#(self.timeInterval and self.iterInstantNum>=self.timeInterval.length())\ + # or (self.iterInstantNum >= self.values) + raise StopIteration + else: + self.iterInstantNum += 1 + return self.getIthValue(self.iterInstantNum-1) + + def getTimeInterval(self): + return self.timeInterval + + def getName(self): + return self.name + + def getValues(self): + return [self.__getitem__(t) for t in self.timeInterval] + + def plot(self, options = '', xfactor = 1., yfactor = 1., timeShift = 0, **kwargs): + if self.getTimeInterval().length() == 1: + marker = 'o' + else: + marker = '' + time = sorted(self.values.keys()) + plot([(x+timeShift)/xfactor for x in time], [self.values[i]/yfactor for i in time], options+marker, **kwargs) + if self.maxValue: + ylim(ymax = self.maxValue) + + @classmethod + def createMultivariate(cls, indicators): + '''Creates a new temporal indicator where the value at each instant is a list + of the indicator values at the instant, in the same order + the time interval will be the union of the time intervals of the indicators + name is concatenation of the indicator names''' + if len(indicators) < 2: + print('Error creating multivariate indicator with only {} indicator'.format(len(indicators))) + return None + + timeInterval = moving.TimeInterval.unionIntervals([indic.getTimeInterval() for indic in indicators]) + values = {} + for t in timeInterval: + tmpValues = [indic[t] for indic in indicators] + uniqueValues = set(tmpValues) + if len(uniqueValues) >= 2 or uniqueValues.pop() is not None: + values[t] = tmpValues + return cls(multivariateName([indic.name for indic in indicators]), values) + +# TODO static method avec class en parametre pour faire des indicateurs agrege, list par instant + +def l1Distance(x, y): # lambda x,y:abs(x-y) + if x is None or y is None: + return float('inf') + else: + return abs(x-y) + +def multiL1Matching(x, y, thresholds, proportionMatching=1.): + n = 0 + nDimensions = len(x) + for i in range(nDimensions): + if l1Distance(x[i], y[i]) <= thresholds[i]: + n += 1 + return n >= nDimensions*proportionMatching + +from utils import LCSS as utilsLCSS + +class LCSS(utilsLCSS): + '''Adapted LCSS class for indicators, same pattern''' + def __init__(self, similarityFunc, delta = float('inf'), minLength = 0, aligned = False, lengthFunc = min): + utilsLCSS.__init__(self, similarityFunc = similarityFunc, delta = delta, aligned = aligned, lengthFunc = lengthFunc) + self.minLength = minLength + + def checkIndicator(self, indicator): + return indicator is not None and len(indicator) >= self.minLength + + def compute(self, indicator1, indicator2, computeSubSequence = False): + if self.checkIndicator(indicator1) and self.checkIndicator(indicator2): + return self._compute(indicator1.getValues(), indicator2.getValues(), computeSubSequence) + else: + return 0 + + def computeNormalized(self, indicator1, indicator2, computeSubSequence = False): + if self.checkIndicator(indicator1) and self.checkIndicator(indicator2): + return self._computeNormalized(indicator1.getValues(), indicator2.getValues(), computeSubSequence) + else: + return 0. + + def computeDistance(self, indicator1, indicator2, computeSubSequence = False): + if self.checkIndicator(indicator1) and self.checkIndicator(indicator2): + return self._computeDistance(indicator1.getValues(), indicator2.getValues(), computeSubSequence) + else: + return 1. + +class SeverityIndicator(TemporalIndicator): + '''Class for severity indicators + field mostSevereIsMax is True + if the most severe value taken by the indicator is the maximum''' + + def __init__(self, name, values, timeInterval=None, mostSevereIsMax=True, maxValue = None): + TemporalIndicator.__init__(self, name, values, timeInterval, maxValue) + self.mostSevereIsMax = mostSevereIsMax + + def getMostSevereValue(self, minNInstants=1, centile=15.): + '''if there are more than minNInstants observations, + returns either the average of these maximum values + or if centile is not None the n% centile from the most severe value + + eg for TTC, 15 returns the 15th centile (value such that 15% of observations are lower)''' + if self.__len__() < minNInstants: + return None + else: + values = list(self.values.values()) + if centile is not None: + if self.mostSevereIsMax: + c = 100-centile + else: + c = centile + return percentile(values, c) + else: + values = sorted(values, reverse = self.mostSevereIsMax) # inverted if most severe is max -> take the first values + return mean(values[:minNInstants]) + + def getInstantOfMostSevereValue(self): + '''Returns the instant at which the indicator reaches its most severe value''' + if self.mostSevereIsMax: + return max(self.values, key=self.values.get) + else: + return min(self.values, key=self.values.get) + +# functions to aggregate discretized maps of indicators +# TODO add values in the cells between the positions (similar to discretizing vector graphics to bitmap) + +def indicatorMap(indicatorValues, trajectory, squareSize): + '''Returns a dictionary + with keys for the indices of the cells (squares) + in which the trajectory positions are located + at which the indicator values are attached + + ex: speeds and trajectory''' + + assert len(indicatorValues) == trajectory.length() + indicatorMap = {} + for k in range(trajectory.length()): + p = trajectory[k] + i = floor(p.x/squareSize) + j = floor(p.y/squareSize) + if (i,j) in indicatorMap: + indicatorMap[(i,j)].append(indicatorValues[k]) + else: + indicatorMap[(i,j)] = [indicatorValues[k]] + for k in indicatorMap: + indicatorMap[k] = mean(indicatorMap[k]) + return indicatorMap + +# def indicatorMapFromPolygon(value, polygon, squareSize): +# '''Fills an indicator map with the value within the polygon +# (array of Nx2 coordinates of the polygon vertices)''' +# points = [] +# for x in arange(min(polygon[:,0])+squareSize/2, max(polygon[:,0]), squareSize): +# for y in arange(min(polygon[:,1])+squareSize/2, max(polygon[:,1]), squareSize): +# points.append([x,y]) +# inside = nx.points_inside_poly(array(points), polygon) +# indicatorMap = {} +# for i in range(len(inside)): +# if inside[i]: +# indicatorMap[(floor(points[i][0]/squareSize), floor(points[i][1]/squareSize))] = 0 +# return indicatorMap + +def indicatorMapFromAxis(value, limits, squareSize): + '''axis = [xmin, xmax, ymin, ymax] ''' + indicatorMap = {} + for x in arange(limits[0], limits[1], squareSize): + for y in arange(limits[2], limits[3], squareSize): + indicatorMap[(floor(x/squareSize), floor(y/squareSize))] = value + return indicatorMap + +def combineIndicatorMaps(maps, squareSize, combinationFunction): + '''Puts many indicator maps together + (averaging the values in each cell + if more than one maps has a value)''' + indicatorMap = {} + for m in maps: + for k,v in m.items(): + if k in indicatorMap: + indicatorMap[k].append(v) + else: + indicatorMap[k] = [v] + for k in indicatorMap: + indicatorMap[k] = combinationFunction(indicatorMap[k]) + return indicatorMap + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/indicators.txt') + unittest.TextTestRunner().run(suite) +# #doctest.testmod() +# #doctest.testfile("example.txt")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/metadata.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,415 @@ +from datetime import datetime, timedelta +from os import path, listdir, sep +from math import floor + +from numpy import zeros, loadtxt, array + +from sqlalchemy import orm, create_engine, Column, Integer, Float, DateTime, String, ForeignKey, Boolean, Interval +from sqlalchemy.orm import relationship, backref, sessionmaker +from sqlalchemy.ext.declarative import declarative_base + +from trafficintelligence.utils import datetimeFormat, removeExtension, getExtension, TimeConverter +from trafficintelligence.cvutils import computeUndistortMaps, videoFilenameExtensions, infoVideo +from trafficintelligence.moving import TimeInterval + +""" +Metadata to describe how video data and configuration files for video analysis are stored + +Typical example is + +site1/view1/2012-06-01/video.avi + /2012-06-02/video.avi + ... + /view2/2012-06-01/video.avi + /2012-06-02/video.avi + ... + +- where site1 is the path to the directory containing all information pertaining to the site, +relative to directory of the SQLite file storing the metadata +represented by Site class +(can contain for example the aerial or map image of the site, used for projection) + +- view1 is the directory for the first camera field of view (camera fixed position) at site site1 +represented by CameraView class +(can contain for example the homography file, mask file and tracking configuration file) + +- YYYY-MM-DD is the directory containing all the video files for that day +with camera view view1 at site site1 + + +""" + +Base = declarative_base() + +class Site(Base): + __tablename__ = 'sites' + idx = Column(Integer, primary_key=True) + name = Column(String) # path to directory containing all site information (in subdirectories), relative to the database position + description = Column(String) # longer names, eg intersection of road1 and road2 + xcoordinate = Column(Float) # ideally moving.Point, but needs to be + ycoordinate = Column(Float) + mapImageFilename = Column(String) # path to map image file, relative to site name, ie sitename/mapImageFilename + nUnitsPerPixel = Column(Float) # number of units of distance per pixel in map image + worldDistanceUnit = Column(String, default = 'm') # make sure it is default in the database + + def __init__(self, name, description = "", xcoordinate = None, ycoordinate = None, mapImageFilename = None, nUnitsPerPixel = 1., worldDistanceUnit = 'm'): + self.name = name + self.description = description + self.xcoordinate = xcoordinate + self.ycoordinate = ycoordinate + self.mapImageFilename = mapImageFilename + self.nUnitsPerPixel = nUnitsPerPixel + self.worldDistanceUnit = worldDistanceUnit + + def getPath(self): + return self.name + + def getMapImageFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.getPath(), self.mapImageFilename) + else: + return self.mapImageFilename + + +class EnvironementalFactors(Base): + '''Represents any environmental factors that may affect the results, in particular + * changing weather conditions + * changing road configuration, geometry, signalization, etc. + ex: sunny, rainy, before counter-measure, after counter-measure''' + __tablename__ = 'environmental_factors' + idx = Column(Integer, primary_key=True) + startTime = Column(DateTime) + endTime = Column(DateTime) + description = Column(String) # eg sunny, before, after + siteIdx = Column(Integer, ForeignKey('sites.idx')) + + site = relationship("Site", backref = backref('environmentalFactors')) + + def __init__(self, startTime, endTime, description, site): + 'startTime is passed as string in utils.datetimeFormat, eg 2011-06-22 10:00:39' + self.startTime = datetime.strptime(startTime, datetimeFormat) + self.endTime = datetime.strptime(endTime, datetimeFormat) + self.description = description + self.site = site + +class CameraType(Base): + ''' Represents parameters of the specific camera used. + + Taken and adapted from tvalib''' + __tablename__ = 'camera_types' + idx = Column(Integer, primary_key=True) + name = Column(String) + resX = Column(Integer) + resY = Column(Integer) + frameRate = Column(Float) + frameRateTimeUnit = Column(String, default = 's') + intrinsicCameraMatrixStr = Column(String) + distortionCoefficientsStr = Column(String) + + def __init__(self, name, resX, resY, frameRate, frameRateTimeUnit = 's', trackingConfigurationFilename = None, intrinsicCameraFilename = None, intrinsicCameraMatrix = None, distortionCoefficients = None): + self.name = name + self.resX = resX + self.resY = resY + self.frameRate = frameRate + self.frameRateTimeUnit = frameRateTimeUnit + self.intrinsicCameraMatrix = None # should be np.array + self.distortionCoefficients = None # list + + if trackingConfigurationFilename is not None: + from storage import ProcessParameters + params = ProcessParameters(trackingConfigurationFilename) + self.intrinsicCameraMatrix = params.intrinsicCameraMatrix + self.distortionCoefficients = params.distortionCoefficients + elif intrinsicCameraFilename is not None: + self.intrinsicCameraMatrix = loadtxt(intrinsicCameraFilename) + self.distortionCoefficients = distortionCoefficients + else: + self.intrinsicCameraMatrix = intrinsicCameraMatrix + self.distortionCoefficients = distortionCoefficients + + if self.intrinsicCameraMatrix is not None: + self.intrinsicCameraMatrixStr = str(self.intrinsicCameraMatrix.tolist()) + if self.distortionCoefficients is not None and len(self.distortionCoefficients) == 5: + self.distortionCoefficientsStr = str(self.distortionCoefficients) + + @orm.reconstructor + def initOnLoad(self): + if self.intrinsicCameraMatrixStr is not None: + from ast import literal_eval + self.intrinsicCameraMatrix = array(literal_eval(self.intrinsicCameraMatrixStr)) + else: + self.intrinsicCameraMatrix = None + if self.distortionCoefficientsStr is not None: + self.distortionCoefficients = literal_eval(self.distortionCoefficientsStr) + else: + self.distortionCoefficients = None + + def computeUndistortMaps(self, undistortedImageMultiplication = None): + if undistortedImageMultiplication is not None and self.intrinsicCameraMatrix is not None and self.distortionCoefficients is not None: + [self.map1, self.map2], newCameraMatrix = computeUndistortMaps(self.resX, self.resY, undistortedImageMultiplication, self.intrinsicCameraMatrix, self.distortionCoefficients) + else: + self.map1 = None + self.map2 = None + + @staticmethod + def getCameraType(session, cameraTypeId, resX = None): + 'Returns the site(s) matching the index or the name' + if str.isdigit(cameraTypeId): + return session.query(CameraType).filter(CameraType.idx == int(cameraTypeId)).all() + else: + if resX is not None: + return session.query(CameraType).filter(CameraType.name.like('%'+cameraTypeId+'%')).filter(CameraType.resX == resX).all() + else: + return session.query(CameraType).filter(CameraType.name.like('%'+cameraTypeId+'%')).all() + +# class SiteDescription(Base): # list of lines and polygons describing the site, eg for sidewalks, center lines + +class CameraView(Base): + __tablename__ = 'camera_views' + idx = Column(Integer, primary_key=True) + description = Column(String) + homographyFilename = Column(String) # path to homograph file, relative to the site name + siteIdx = Column(Integer, ForeignKey('sites.idx')) + cameraTypeIdx = Column(Integer, ForeignKey('camera_types.idx')) + trackingConfigurationFilename = Column(String) # path to configuration .cfg file, relative to site name + maskFilename = Column(String) # path to mask file, relative to site name + virtual = Column(Boolean) # indicates it is not a real camera view, eg merged + + site = relationship("Site", backref = backref('cameraViews')) + cameraType = relationship('CameraType', backref = backref('cameraViews')) + + def __init__(self, description, homographyFilename, site, cameraType, trackingConfigurationFilename, maskFilename, virtual = False): + self.description = description + self.homographyFilename = homographyFilename + self.site = site + self.cameraType = cameraType + self.trackingConfigurationFilename = trackingConfigurationFilename + self.maskFilename = maskFilename + self.virtual = virtual + + def getHomographyFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.site.getPath(), self.homographyFilename) + else: + return self.homographyFilename + + def getTrackingConfigurationFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.site.getPath(), self.trackingConfigurationFilename) + else: + return self.trackingConfigurationFilename + + def getMaskFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.site.getPath(), self.maskFilename) + else: + return self.maskFilename + + def getTrackingParameters(self): + return ProcessParameters(self.getTrackingConfigurationFilename()) + + def getHomographyDistanceUnit(self): + return self.site.worldDistanceUnit + +# class Alignment(Base): +# __tablename__ = 'alignments' +# idx = Column(Integer, primary_key=True) +# siteIdx = Column(Integer, ForeignKey('sites.idx')) + +# cameraView = relationship("Site", backref = backref('alignments')) + +# def __init__(self, cameraView): +# self.cameraView = cameraView + +# class Point(Base): +# __tablename__ = 'points' +# alignmentIdx = Column(Integer, ForeignKey('alignments.idx'), primary_key=True) +# index = Column(Integer, primary_key=True) # order of points in this alignment +# x = Column(Float) +# y = Column(Float) + +# alignment = relationship("Alignment", backref = backref('points', order_by = index)) + +# def __init__(self, alignmentIdx, index, x, y): +# self.alignmentIdx = alignmentIdx +# self.index = index +# self.x = x +# self.y = y + +class VideoSequence(Base): + __tablename__ = 'video_sequences' + idx = Column(Integer, primary_key=True) + name = Column(String) # path to the video file relative to the the site name + startTime = Column(DateTime) + duration = Column(Interval) # video sequence duration + databaseFilename = Column(String) # path to the database file relative to the the site name + virtual = Column(Boolean) # indicates it is not a real video sequence (no video file), eg merged + cameraViewIdx = Column(Integer, ForeignKey('camera_views.idx')) + + cameraView = relationship("CameraView", backref = backref('videoSequences', order_by = idx)) + + def __init__(self, name, startTime, duration, cameraView, databaseFilename = None, virtual = False): + '''startTime is passed as string in utils.datetimeFormat, eg 2011-06-22 10:00:39 + duration is a timedelta object''' + self.name = name + if isinstance(startTime, str): + self.startTime = datetime.strptime(startTime, datetimeFormat) + else: + self.startTime = startTime + self.duration = duration + self.cameraView = cameraView + if databaseFilename is None and len(self.name) > 0: + self.databaseFilename = removeExtension(self.name)+'.sqlite' + else: + self.databaseFilename = databaseFilename + self.virtual = virtual + + def getVideoSequenceFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.cameraView.site.getPath(), self.name) + else: + return self.name + + def getDatabaseFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.cameraView.site.getPath(), self.databaseFilename) + else: + return self.databaseFilename + + def getTimeInterval(self): + return TimeInterval(self.startTime, self.startTime+self.duration) + + def containsInstant(self, instant): + 'instant is a datetime' + return self.startTime <= instant and self.startTime+self.duration + + def intersection(self, startTime, endTime): + 'returns the moving.TimeInterval intersection with [startTime, endTime]' + return TimeInterval.intersection(self.getTimeInterval(), TimeInterval(startTime, endTime)) + + def getFrameNum(self, instant): + 'Warning, there is no check of correct time units' + if self.containsInstant(instant): + return int(floor((instant-self.startTime).seconds*self.cameraView.cameraType.frameRate)) + else: + return None + +class TrackingAnnotation(Base): + __tablename__ = 'tracking_annotations' + idx = Column(Integer, primary_key=True) + description = Column(String) # description + groundTruthFilename = Column(String) + firstFrameNum = Column(Integer) # first frame num of annotated data (could be computed on less data) + lastFrameNum = Column(Integer) + videoSequenceIdx = Column(Integer, ForeignKey('video_sequences.idx')) + maskFilename = Column(String) # path to mask file (can be different from camera view, for annotations), relative to site name + undistorted = Column(Boolean) # indicates whether the annotations were done in undistorted video space + + videoSequence = relationship("VideoSequence", backref = backref('trackingAnnotations')) + + def __init__(self, description, groundTruthFilename, firstFrameNum, lastFrameNum, videoSequence, maskFilename, undistorted = True): + self.description = description + self.groundTruthFilename = groundTruthFilename + self.firstFrameNum = firstFrameNum + self.lastFrameNum = lastFrameNum + self.videoSequence = videoSequence + self.undistorted = undistorted + self.maskFilename = maskFilename + + def getGroundTruthFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.videoSequence.cameraView.site.getPath(), self.groundTruthFilename) + else: + return self.groundTruthFilename + + def getMaskFilename(self, relativeToSiteFilename = True): + if relativeToSiteFilename: + return path.join(self.videoSequence.cameraView.site.getPath(), self.maskFilename) + else: + return self.maskFilename + + def getTimeInterval(self): + return TimeInterval(self.firstFrameNum, self.lastFrameNum) + +# add class for Analysis: foreign key VideoSequenceId, dataFilename, configFilename (get the one from camera view by default), mask? (no, can be referenced in the tracking cfg file) + +# class Analysis(Base): # parameters necessary for processing the data: free form +# eg bounding box depends on camera view, tracking configuration depends on camera view +# results: sqlite + +def createDatabase(filename): + 'creates a session to query the filename' + engine = create_engine('sqlite:///'+filename) + Base.metadata.create_all(engine) + Session = sessionmaker(bind=engine) + return Session() + +def connectDatabase(filename): + 'creates a session to query the filename' + engine = create_engine('sqlite:///'+filename) + Session = sessionmaker(bind=engine) + return Session() + +def getSite(session, siteId = None, name = None, description = None): + 'Returns the site(s) matching the index or the name' + if siteId is not None: + return session.query(Site).filter(Site.idx == int(siteId)).all() + elif name is not None: + return session.query(Site).filter(Site.description.like('%'+name+'%')).all() + elif description is not None: + return session.query(Site).filter(Site.description.like('%'+description+'%')).all() + else: + print('No siteId, name or description have been provided to the function') + return [] + +def getCameraView(session, viewId): + 'Returns the site(s) matching the index' + return session.query(CameraView).filter(CameraView.idx == int(viewId)).first() + +def initializeSites(session, directoryName, nViewsPerSite = 1): + '''Initializes default site objects and n camera views per site + + eg somedirectory/montreal/ contains intersection1, intersection2, etc. + The site names would be somedirectory/montreal/intersection1, somedirectory/montreal/intersection2, etc. + The views should be directories in somedirectory/montreal/intersection1''' + sites = [] + cameraViews = [] + names = sorted(listdir(directoryName)) + for name in names: + if path.isdir(directoryName+sep+name): + sites.append(Site(directoryName+sep+name, None)) + for cameraViewIdx in range(1, nViewsPerSite+1): + cameraViews.append(CameraView('view{}'.format(cameraViewIdx), None, sites[-1], None, None, None)) + session.add_all(sites) + session.add_all(cameraViews) + session.commit() + +def initializeVideos(session, cameraView, directoryName, startTime = None, datetimeFormat = None): + '''Initializes videos with time or tries to guess it from filename + directoryName should contain the videos to find and be the relative path from the site location''' + names = sorted(listdir(directoryName)) + videoSequences = [] + if datetimeFormat is not None: + timeConverter = TimeConverter(datetimeFormat) + for name in names: + prefix = removeExtension(name) + extension = getExtension(name) + if extension in videoFilenameExtensions: + if datetimeFormat is not None: + from argparse import ArgumentTypeError + try: + t1 = timeConverter.convert(name[:name.rfind('_')]) + print('DB time {} / Time from filename {}'.format(startTime, t1)) + except ArgumentTypeError as e: + print('File format error for time {} (prefix {})'.format(name, prefix)) + vidinfo = infoVideo(directoryName+sep+name) + duration = vidinfo['number of frames']#timedelta(minutes = 27, seconds = 33) + fps = vidinfo['fps'] + duration = timedelta(seconds=duration/fps) + videoSequences.append(VideoSequence(directoryName+sep+name, startTime, duration, cameraView, directoryName+sep+prefix+'.sqlite')) + startTime += duration + session.add_all(videoSequences) + session.commit() + +# management +# TODO need to be able to copy everything from a site from one sqlite to another, and delete everything attached to a site
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/ml.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,326 @@ +#! /usr/bin/env python +'''Libraries for machine learning algorithms''' + +from os import path +from random import shuffle +from copy import copy, deepcopy + +import numpy as np +from matplotlib.pylab import text +import matplotlib as mpl +import matplotlib.pyplot as plt +from scipy.cluster.vq import kmeans, whiten, vq +from sklearn import mixture +try: + import cv2 + opencvAvailable = True +except ImportError: + print('OpenCV library could not be loaded (video replay functions will not be available)') # TODO change to logging module + opencvAvailable = False + +from trafficintelligence import utils + +##################### +# OpenCV ML models +##################### + +def computeConfusionMatrix(model, samples, responses): + '''computes the confusion matrix of the classifier (model) + + samples should be n samples by m variables''' + classifications = {} + predictions = model.predict(samples) + for predicted, y in zip(predictions, responses): + classifications[(y, predicted)] = classifications.get((y, predicted), 0)+1 + return classifications + +if opencvAvailable: + class SVM(object): + '''wrapper for OpenCV SimpleVectorMachine algorithm''' + def __init__(self, svmType = cv2.ml.SVM_C_SVC, kernelType = cv2.ml.SVM_RBF, degree = 0, gamma = 1, coef0 = 0, Cvalue = 1, nu = 0, p = 0): + self.model = cv2.ml.SVM_create() + self.model.setType(svmType) + self.model.setKernel(kernelType) + self.model.setDegree(degree) + self.model.setGamma(gamma) + self.model.setCoef0(coef0) + self.model.setC(Cvalue) + self.model.setNu(nu) + self.model.setP(p) + + def save(self, filename): + self.model.save(filename) + + def train(self, samples, layout, responses, computePerformance = False): + self.model.train(samples, layout, responses) + if computePerformance: + return computeConfusionMatrix(self, samples, responses) + + def predict(self, hog): + retval, predictions = self.model.predict(hog) + if hog.shape[0] == 1: + return predictions[0][0] + else: + return np.asarray(predictions, dtype = np.int).ravel().tolist() + + def SVM_load(filename): + if path.exists(filename): + svm = SVM() + svm.model = cv2.ml.SVM_load(filename) + return svm + else: + print('Provided filename {} does not exist: model not loaded!'.format(filename)) + +##################### +# Clustering +##################### + +class Centroid(object): + 'Wrapper around instances to add a counter' + + def __init__(self, instance, nInstances = 1): + self.instance = instance + self.nInstances = nInstances + + # def similar(instance2): + # return self.instance.similar(instance2) + + def add(self, instance2): + self.instance = self.instance.multiply(self.nInstances)+instance2 + self.nInstances += 1 + self.instance = self.instance.multiply(1/float(self.nInstances)) + + def average(c): + inst = self.instance.multiply(self.nInstances)+c.instance.multiply(instance.nInstances) + inst.multiply(1/(self.nInstances+instance.nInstances)) + return Centroid(inst, self.nInstances+instance.nInstances) + + def plot(self, options = ''): + self.instance.plot(options) + text(self.instance.position.x+1, self.instance.position.y+1, str(self.nInstances)) + +def kMedoids(similarityMatrix, initialCentroids = None, k = None): + '''Algorithm that clusters any dataset based on a similarity matrix + Either the initialCentroids or k are passed''' + pass + +def assignCluster(data, similarFunc, initialCentroids = None, shuffleData = True): + '''k-means algorithm with similarity function + Two instances should be in the same cluster if the sameCluster function returns true for two instances. It is supposed that the average centroid of a set of instances can be computed, using the function. + The number of clusters will be determined accordingly + + data: list of instances + averageCentroid: ''' + localdata = copy(data) # shallow copy to avoid modifying data + if shuffleData: + shuffle(localdata) + if initialCentroids is None: + centroids = [Centroid(localdata[0])] + else: + centroids = deepcopy(initialCentroids) + for instance in localdata[1:]: + i = 0 + while i<len(centroids) and not similarFunc(centroids[i].instance, instance): + i += 1 + if i == len(centroids): + centroids.append(Centroid(instance)) + else: + centroids[i].add(instance) + + return centroids + +# TODO recompute centroids for each cluster: instance that minimizes some measure to all other elements + +def spectralClustering(similarityMatrix, k, iter=20): + '''Spectral Clustering algorithm''' + n = len(similarityMatrix) + # create Laplacian matrix + rowsum = np.sum(similarityMatrix,axis=0) + D = np.diag(1 / np.sqrt(rowsum)) + I = np.identity(n) + L = I - np.dot(D,np.dot(similarityMatrix,D)) + # compute eigenvectors of L + U,sigma,V = np.linalg.svd(L) + # create feature vector from k first eigenvectors + # by stacking eigenvectors as columns + features = np.array(V[:k]).T + # k-means + features = whiten(features) + centroids,distortion = kmeans(features,k, iter) + code,distance = vq(features,centroids) # code starting from 0 (represent first cluster) to k-1 (last cluster) + return code,sigma + +def assignToPrototypeClusters(instances, prototypeIndices, similarities, minSimilarity, similarityFunc = None, minClusterSize = 0): + '''Assigns instances to prototypes + if minClusterSize is not 0, the clusters will be refined by removing iteratively the smallest clusters + and reassigning all elements in the cluster until no cluster is smaller than minClusterSize + + labels are indices in the prototypeIndices''' + if similarityFunc is None: + print('similarityFunc is None') + return None + + indices = [i for i in range(len(instances)) if i not in prototypeIndices] + labels = [-1]*len(instances) + assign = True + while assign: + for i in prototypeIndices: + labels[i] = i + for i in indices: + for j in prototypeIndices: + if similarities[i][j] < 0: + similarities[i][j] = similarityFunc(instances[i], instances[j]) + similarities[j][i] = similarities[i][j] + label = similarities[i][prototypeIndices].argmax() + if similarities[i][prototypeIndices[label]] >= minSimilarity: + labels[i] = prototypeIndices[label] + else: + labels[i] = -1 # outlier + clusterSizes = {i: sum(np.array(labels) == i) for i in prototypeIndices} + smallestClusterIndex = min(clusterSizes, key = clusterSizes.get) + assign = (clusterSizes[smallestClusterIndex] < minClusterSize) + if assign: + prototypeIndices.remove(smallestClusterIndex) + indices = [i for i in range(similarities.shape[0]) if labels[i] == smallestClusterIndex] + return prototypeIndices, labels + +def prototypeCluster(instances, similarities, minSimilarity, similarityFunc = None, optimizeCentroid = False, randomInitialization = False, initialPrototypeIndices = None): + '''Finds exemplar (prototype) instance that represent each cluster + Returns the prototype indices (in the instances list) + + the elements in the instances list must have a length (method __len__), or one can use the optimizeCentroid + the positions in the instances list corresponds to the similarities + if similarityFunc is provided, the similarities are calculated as needed (this is faster) if not in similarities (negative if not computed) + similarities must still be allocated with the right size + + if an instance is different enough (<minSimilarity), + it will become a new prototype. + Non-prototype instances will be assigned to an existing prototype + + if optimizeCentroid is True, each time an element is added, we recompute the centroid trajectory as the most similar to all in the cluster + + initialPrototypeIndices are indices in instances + + TODO: check how similarity evolves in clusters''' + if len(instances) == 0: + print('no instances to cluster (empty list)') + return None + if similarityFunc is None: + print('similarityFunc is None') + return None + + # sort instances based on length + indices = range(len(instances)) + if randomInitialization or optimizeCentroid: + indices = np.random.permutation(indices).tolist() + else: + def compare(i, j): + if len(instances[i]) > len(instances[j]): + return -1 + elif len(instances[i]) == len(instances[j]): + return 0 + else: + return 1 + indices.sort(compare) + # initialize clusters + clusters = [] + if initialPrototypeIndices is None: + prototypeIndices = [indices[0]] + else: + prototypeIndices = initialPrototypeIndices # think of the format: if indices, have to be in instances + for i in prototypeIndices: + clusters.append([i]) + indices.remove(i) + # go through all instances + for i in indices: + for j in prototypeIndices: + if similarities[i][j] < 0: + similarities[i][j] = similarityFunc(instances[i], instances[j]) + similarities[j][i] = similarities[i][j] + label = similarities[i][prototypeIndices].argmax() # index in prototypeIndices + if similarities[i][prototypeIndices[label]] < minSimilarity: + prototypeIndices.append(i) + clusters.append([]) + else: + clusters[label].append(i) + if optimizeCentroid: + if len(clusters[label]) >= 2: # no point if only one element in cluster + for j in clusters[label][:-1]: + if similarities[i][j] < 0: + similarities[i][j] = similarityFunc(instances[i], instances[j]) + similarities[j][i] = similarities[i][j] + clusterIndices = clusters[label] + clusterSimilarities = similarities[clusterIndices][:,clusterIndices] + newCentroidIdx = clusterIndices[clusterSimilarities.sum(0).argmax()] + if prototypeIndices[label] != newCentroidIdx: + prototypeIndices[label] = newCentroidIdx + elif len(instances[prototypeIndices[label]]) < len(instances[i]): # replace prototype by current instance i if longer # otherwise, possible to test if randomInitialization or initialPrototypes is not None + prototypeIndices[label] = i + return prototypeIndices + +def computeClusterSizes(labels, prototypeIndices, outlierIndex = -1): + clusterSizes = {i: sum(np.array(labels) == i) for i in prototypeIndices} + clusterSizes['outlier'] = sum(np.array(labels) == outlierIndex) + return clusterSizes + +def computeClusterStatistics(labels, prototypeIndices, instances, similarities, similarityFunc, clusters = None, outlierIndex = -1): + if clusters is None: + clusters = {protoId:[] for protoId in prototypeIndices+[-1]} + for i,l in enumerate(labels): + clusters[l].append(i) + clusters = [clusters[protoId] for protoId in prototypeIndices] + for i, cluster in enumerate(clusters): + n = len(cluster) + print('cluster {}: {} elements'.format(prototypeIndices[i], n)) + if n >=2: + for j,k in enumerate(cluster): + for l in cluster[:j]: + if similarities[k][l] < 0: + similarities[k][l] = similarityFunc(instances[k], instances[l]) + similarities[l][k] = similarities[k][l] + print('Mean similarity to prototype: {}'.format((similarities[prototypeIndices[i]][cluster].sum()+1)/(n-1))) + print('Mean overall similarity: {}'.format((similarities[cluster][:,cluster].sum()+n)/(n*(n-1)))) + +# Gaussian Mixture Models +def plotGMM(mean, covariance, gmmId, fig, color, alpha = 0.3): + v, w = np.linalg.eigh(covariance) + angle = 180*np.arctan2(w[0][1], w[0][0])/np.pi + v *= 4 + ell = mpl.patches.Ellipse(mean, v[0], v[1], 180+angle, color=color) + ell.set_clip_box(fig.bbox) + ell.set_alpha(alpha) + fig.axes[0].add_artist(ell) + plt.plot([mean[0]], [mean[1]], 'x'+color) + plt.annotate(str(gmmId), xy=(mean[0]+1, mean[1]+1)) + +def plotGMMClusters(model, labels = None, dataset = None, fig = None, colors = utils.colors, nUnitsPerPixel = 1., alpha = 0.3): + '''plot the ellipse corresponding to the Gaussians + and the predicted classes of the instances in the dataset''' + if fig is None: + fig = plt.figure() + if len(fig.get_axes()) == 0: + fig.add_subplot(111) + for i in range(model.n_components): + mean = model.means_[i]/nUnitsPerPixel + covariance = model.covariances_[i]/nUnitsPerPixel + # plot points + if dataset is not None: + tmpDataset = dataset/nUnitsPerPixel + plt.scatter(tmpDataset[labels == i, 0], tmpDataset[labels == i, 1], .8, color=colors[i]) + # plot an ellipse to show the Gaussian component + plotGMM(mean, covariance, i, fig, colors[i], alpha) + if dataset is None: # to address issues without points, the axes limits are not redrawn + minima = model.means_.min(0) + maxima = model.means_.max(0) + xwidth = 0.5*(maxima[0]-minima[0]) + ywidth = 0.5*(maxima[1]-minima[1]) + plt.xlim(minima[0]-xwidth,maxima[0]+xwidth) + plt.ylim(minima[1]-ywidth,maxima[1]+ywidth) + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/ml.txt') + unittest.TextTestRunner().run(suite) +# #doctest.testmod() +# #doctest.testfile("example.txt")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/moving.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,1957 @@ +#! /usr/bin/env python +'''Libraries for moving objects, trajectories...''' + +from trafficintelligence import utils, cvutils +from trafficintelligence.base import VideoFilenameAddable + +from math import sqrt, atan2, cos, sin +from numpy import median, mean, array, arange, zeros, ones, hypot, NaN, std, floor, float32, argwhere, minimum +from matplotlib.pyplot import plot, text +from scipy.stats import scoreatpercentile +from scipy.spatial.distance import cdist +from scipy.signal import savgol_filter +import copy + +try: + from shapely.geometry import Polygon, Point as shapelyPoint + from shapely.prepared import prep, PreparedGeometry + shapelyAvailable = True +except ImportError: + print('Shapely library could not be loaded') + shapelyAvailable = False + + +class Interval(object): + '''Generic interval: a subset of real numbers (not iterable)''' + def __init__(self, first=0, last=-1, revert = False): + if revert and last<first: + self.first=last + self.last=first + else: + self.first=first + self.last=last + + def __str__(self): + return '[{0}, {1}]'.format(self.first, self.last) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return ((self.first == other.first) and (self.last == other.last)) or ((self.first == other.last) and (self.last == other.first)) + + def empty(self): + return self.first > self.last + + def center(self): + return (self.first+self.last)/2. + + def length(self): + '''Returns the length of the interval''' + return float(max(0,self.last-self.first)) + + def equal(self, i2): + return self.first==i2.first and self.last == i2.last + + def getList(self): + return [self.first, self.last] + + def contains(self, instant): + return (self.first<=instant and self.last>=instant) + + def inside(self, interval2): + '''Indicates if the temporal interval of self is comprised in interval2''' + return (self.first >= interval2.first) and (self.last <= interval2.last) + + def shift(self, offset): + self.first += offset + self.last += offset + + @classmethod + def union(cls, interval1, interval2): + '''Smallest interval comprising self and interval2''' + return cls(min(interval1.first, interval2.first), max(interval2.last, interval2.last)) + + @classmethod + def intersection(cls, interval1, interval2): + '''Largest interval comprised in both self and interval2''' + return cls(max(interval1.first, interval2.first), min(interval1.last, interval2.last)) + + def distance(self, interval2): + if not Interval.intersection(self, interval2).empty(): + return 0 + elif self.first > interval2.last: + return self.first - interval2.last + elif self.last < interval2.first: + return interval2.first - self.last + else: + return None + + @classmethod + def unionIntervals(cls, intervals): + 'returns the smallest interval containing all intervals' + inter = cls(intervals[0].first, intervals[0].last) + for i in intervals[1:]: + inter = cls.union(inter, i) + return inter + + +class TimeInterval(Interval): + '''Temporal interval: set of instants at fixed time step, between first and last, included + + For example: based on frame numbers (hence the modified length method) + It may be modified directly by setting first and last + It also (mostly) works with datetime.datetime''' + + def __init__(self, first=0, last=-1, revert = False): + super(TimeInterval, self).__init__(first, last, revert) + + @staticmethod + def fromInterval(inter): + return TimeInterval(inter.first, inter.last) + + def __getitem__(self, i): + if not self.empty(): + if isinstance(i, int): + return self.first+i + else: + raise TypeError("Invalid argument type.") + #elif isinstance( key, slice ): + + def __iter__(self): + self.iterInstantNum = -1 + return self + + def __next__(self): + if self.iterInstantNum >= self.length()-1: + raise StopIteration + else: + self.iterInstantNum += 1 + return self[self.iterInstantNum] + + def length(self): + '''Returns the length of the interval''' + return float(max(0,self.last-self.first+1)) + + def __len__(self): + return self.length() + +# class BoundingPolygon: +# '''Class for a polygon bounding a set of points +# with methods to create intersection, unions... +# ''' +# We will use the polygon class of Shapely + +class STObject(object): + '''Class for spatio-temporal object, i.e. with temporal and spatial existence + (time interval and bounding polygon for positions (e.g. rectangle)). + + It may not mean that the object is defined + for all time instants within the time interval''' + + def __init__(self, num = None, timeInterval = None, boundingPolygon = None): + self.num = num + self.timeInterval = timeInterval + self.boundingPolygon = boundingPolygon + + def empty(self): + return self.timeInterval.empty()# or not self.boudingPolygon + + def getNum(self): + return self.num + + def __len__(self): + return self.timeInterval.length() + + def length(self): + return self.timeInterval.length() + + def getFirstInstant(self): + return self.timeInterval.first + + def getLastInstant(self): + return self.timeInterval.last + + def setFirstInstant(self, t): + if t <= self.timeInterval.last: + self.timeInterval.first = t + else: + print('new first instant is after last, not changing') + + def setLastInstant(self, t): + if t >= self.timeInterval.first: + self.timeInterval.last = t + else: + print('new last instant is before first, not changing') + + def getTimeInterval(self): + return self.timeInterval + + def existsAtInstant(self, t): + return self.timeInterval.contains(t) + + def commonTimeInterval(self, obj2): + return TimeInterval.intersection(self.getTimeInterval(), obj2.getTimeInterval()) + + def shiftTimeInterval(self, offset): + self.timeInterval.shift(offset) + +class Point(object): + def __init__(self, x, y): + self.x = x + self.y = y + + def __str__(self): + return '({:f},{:f})'.format(self.x,self.y) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return (self.x == other.x) and (self.y == other.y) + + def __add__(self, other): + return Point(self.x+other.x, self.y+other.y) + + def __sub__(self, other): + return Point(self.x-other.x, self.y-other.y) + + def __neg__(self): + return Point(-self.x, -self.y) + + def __getitem__(self, i): + if i == 0: + return self.x + elif i == 1: + return self.y + else: + raise IndexError() + + def orthogonal(self, clockwise = True): + 'Returns the orthogonal vector' + if clockwise: + return Point(self.y, -self.x) + else: + return Point(-self.y, self.x) + + def projectLocal(self, v, clockwise = True): + 'Projects point projected on v, v.orthogonal()' + e1 = v/v.norm2() + e2 = e1.orthogonal(clockwise) + return Point(Point.dot(self, e1), Point.doc(self, e2)) + + def rotate(self, theta): + return Point(self.x*cos(theta)-self.y*sin(theta), self.x*sin(theta)+self.y*cos(theta)) + + def __mul__(self, alpha): + 'Warning, returns a new Point' + return Point(self.x*alpha, self.y*alpha) + + def divide(self, alpha): + 'Warning, returns a new Point' + return Point(self.x/alpha, self.y/alpha) + + def plot(self, options = 'o', **kwargs): + plot([self.x], [self.y], options, **kwargs) + + @staticmethod + def plotSegment(p1, p2, options = 'o', **kwargs): + plot([p1.x, p2.x], [p1.y, p2.y], options, **kwargs) + + def angle(self): + return atan2(self.y, self.x) + + def norm2Squared(self): + '''2-norm distance (Euclidean distance)''' + return self.x**2+self.y**2 + + def norm2(self): + '''2-norm distance (Euclidean distance)''' + return sqrt(self.norm2Squared()) + + def norm1(self): + return abs(self.x)+abs(self.y) + + def normMax(self): + return max(abs(self.x),abs(self.y)) + + def aslist(self): + return [self.x, self.y] + + def astuple(self): + return (self.x, self.y) + + def asint(self): + return Point(int(self.x), int(self.y)) + + if shapelyAvailable: + def asShapely(self): + return shapelyPoint(self.x, self.y) + + def homographyProject(self, homography): + projected = cvutils.homographyProject(array([[self.x], [self.y]]), homography) + return Point(projected[0], projected[1]) + + def inPolygon(self, polygon): + '''Indicates if the point x, y is inside the polygon + (array of Nx2 coordinates of the polygon vertices) + + taken from http://www.ariel.com.au/a/python-point-int-poly.html + + Use Polygon.contains if Shapely is installed''' + + n = polygon.shape[0]; + counter = 0; + + p1 = polygon[0,:]; + for i in range(n+1): + p2 = polygon[i % n,:]; + if self.y > min(p1[1],p2[1]): + if self.y <= max(p1[1],p2[1]): + if self.x <= max(p1[0],p2[0]): + if p1[1] != p2[1]: + xinters = (self.y-p1[1])*(p2[0]-p1[0])/(p2[1]-p1[1])+p1[0]; + if p1[0] == p2[0] or self.x <= xinters: + counter+=1; + p1=p2 + return (counter%2 == 1); + + @staticmethod + def fromList(p): + return Point(p[0], p[1]) + + @staticmethod + def dot(p1, p2): + 'Scalar product' + return p1.x*p2.x+p1.y*p2.y + + @staticmethod + def cross(p1, p2): + 'Cross product' + return p1.x*p2.y-p1.y*p2.x + + @staticmethod + def parallel(p1, p2): + return Point.cross(p1, p2) == 0. + + @staticmethod + def cosine(p1, p2): + return Point.dot(p1,p2)/(p1.norm2()*p2.norm2()) + + @staticmethod + def distanceNorm2(p1, p2): + return (p1-p2).norm2() + + @staticmethod + def plotAll(points, **kwargs): + from matplotlib.pyplot import scatter + scatter([p.x for p in points],[p.y for p in points], **kwargs) + + def similarOrientation(self, refDirection, cosineThreshold): + 'Indicates whether the cosine of the vector and refDirection is smaller than cosineThreshold' + return Point.cosine(self, refDirection) >= cosineThreshold + + @staticmethod + def timeToCollision(p1, p2, v1, v2, collisionThreshold): + '''Computes exact time to collision with a distance threshold + The unknown of the equation is the time to reach the intersection + between the relative trajectory of one road user + and the circle of radius collisionThreshold around the other road user''' + dv = v1-v2 + dp = p1-p2 + a = dv.norm2Squared()#(v1.x-v2.x)**2 + (v1.y-v2.y)**2 + b = 2*Point.dot(dv, dp)#2 * ((p1.x-p2.x) * (v1.x-v2.x) + (p1.y-p2.y) * (v1.y-v2.y)) + c = dp.norm2Squared() - collisionThreshold**2#(p1.x-p2.x)**2 + (p1.y-p2.y)**2 - collisionThreshold**2 + + delta = b**2 - 4*a*c + if delta >= 0: + deltaRoot = sqrt(delta) + ttc1 = (-b + deltaRoot)/(2*a) + ttc2 = (-b - deltaRoot)/(2*a) + if ttc1 >= 0 and ttc2 >= 0: + return min(ttc1,ttc2) + elif ttc1 >= 0: + return ttc1 + elif ttc2 >= 0: + return ttc2 + else: # ttc1 < 0 and ttc2 < 0: + return None + else: + return None + + @staticmethod + def midPoint(p1, p2): + 'Returns the middle of the segment [p1, p2]' + return Point(0.5*p1.x+0.5*p2.x, 0.5*p1.y+0.5*p2.y) + + @staticmethod + def agg(points, aggFunc = mean): + return Point(aggFunc([p.x for p in points]), aggFunc([p.y for p in points])) + +if shapelyAvailable: + def pointsInPolygon(points, polygon): + '''Optimized tests of a series of points within (Shapely) polygon (not prepared)''' + if type(polygon) == PreparedGeometry: + prepared_polygon = polygon + else: + prepared_polygon = prep(polygon) + return list(filter(prepared_polygon.contains, points)) + +# Functions for coordinate transformation +# From Paul St-Aubin's PVA tools +def subsec_spline_dist(splines): + ''' Prepare list of spline subsegments from a spline list. + + Output: + ======= + ss_spline_d[spline #][mode][station] + + where: + mode=0: incremental distance + mode=1: cumulative distance + mode=2: cumulative distance with trailing distance + ''' + ss_spline_d = [] + #Prepare subsegment distances + for spline in range(len(splines)): + ss_spline_d[spline]=[]#.append([[],[],[]]) + ss_spline_d[spline].append(zeros(len(splines[spline])-1)) #Incremental distance + ss_spline_d[spline].append(zeros(len(splines[spline])-1)) #Cumulative distance + ss_spline_d[spline].append(zeros(len(splines[spline]))) #Cumulative distance with trailing distance + for spline_p in range(len(splines[spline])): + if spline_p > (len(splines[spline]) - 2): + break + ss_spline_d[spline][0][spline_p] = utils.pointDistanceL2(splines[spline][spline_p][0],splines[spline][spline_p][1],splines[spline][(spline_p+1)][0],splines[spline][(spline_p+1)][1]) + ss_spline_d[spline][1][spline_p] = sum(ss_spline_d[spline][0][0:spline_p]) + ss_spline_d[spline][2][spline_p] = ss_spline_d[spline][1][spline_p]#sum(ss_spline_d[spline][0][0:spline_p]) + + ss_spline_d[spline][2][-1] = ss_spline_d[spline][2][-2] + ss_spline_d[spline][0][-1] + + return ss_spline_d + +def prepareSplines(splines): + 'Approximates slope singularity by giving some slope roundoff; account for roundoff error' + for spline in splines: + p1 = spline[0] + for i in range(len(spline)-1): + p2 = spline[i+1] + if(round(p1.x, 10) == round(p2.x, 10)): + p2.x += 0.0000000001 + if(round(p1.y, 10) == round(p2.y, 10)): + p2.y += 0.0000000001 + p1 = p2 + +def ppldb2p(qx,qy, p0x,p0y, p1x,p1y): + ''' Point-projection (Q) on line defined by 2 points (P0,P1). + http://cs.nyu.edu/~yap/classes/visual/03s/hw/h2/math.pdf + ''' + if(p0x == p1x and p0y == p1y): + return None + try: + #Approximate slope singularity by giving some slope roundoff; account for roundoff error + # if(round(p0x, 10) == round(p1x, 10)): + # p1x += 0.0000000001 + # if(round(p0y, 10) == round(p1y, 10)): + # p1y += 0.0000000001 + #make the calculation + Y = (-(qx)*(p0y-p1y)-(qy*(p0y-p1y)**2)/(p0x-p1x)+p0x**2*(p0y-p1y)/(p0x-p1x)-p0x*p1x*(p0y-p1y)/(p0x-p1x)-p0y*(p0x-p1x))/(p1x-p0x-(p0y-p1y)**2/(p0x-p1x)) + X = (-Y*(p1y-p0y)+qx*(p1x-p0x)+qy*(p1y-p0y))/(p1x-p0x) + except ZeroDivisionError: + print('Error: Division by zero in ppldb2p. Please report this error with the full traceback:') + print('qx={0}, qy={1}, p0x={2}, p0y={3}, p1x={4}, p1y={5}...'.format(qx, qy, p0x, p0y, p1x, p1y)) + import pdb; pdb.set_trace() + return Point(X,Y) + +def getSYfromXY(p, splines, goodEnoughSplineDistance = 0.5): + ''' Snap a point p to its nearest subsegment of it's nearest spline (from the list splines). + A spline is a list of points (class Point), most likely a trajectory. + + Output: + ======= + [spline index, + subsegment leading point index, + snapped point, + subsegment distance, + spline distance, + orthogonal point offset] + + or None + ''' + minOffsetY = float('inf') + #For each spline + for splineIdx in range(len(splines)): + #For each spline point index + for spline_p in range(len(splines[splineIdx])-1): + #Get closest point on spline + closestPoint = ppldb2p(p.x,p.y,splines[splineIdx][spline_p][0],splines[splineIdx][spline_p][1],splines[splineIdx][spline_p+1][0],splines[splineIdx][spline_p+1][1]) + if closestPoint is None: + print('Error: Spline {0}, segment {1} has identical bounds and therefore is not a vector. Projection cannot continue.'.format(splineIdx, spline_p)) + return None + # check if the projected point is in between the current segment of the alignment bounds + if utils.inBetween(splines[splineIdx][spline_p][0], splines[splineIdx][spline_p+1][0], closestPoint.x) and utils.inBetween(splines[splineIdx][spline_p][1], splines[splineIdx][spline_p+1][1], closestPoint.y): + offsetY = Point.distanceNorm2(closestPoint, p) + if offsetY < minOffsetY: + minOffsetY = offsetY + snappedSplineIdx = splineIdx + snappedSplineLeadingPoint = spline_p + snappedPoint = Point(closestPoint.x, closestPoint.y) + #Jump loop if significantly close + if offsetY < goodEnoughSplineDistance: + break + + #Get sub-segment distance + if minOffsetY != float('inf'): + subsegmentDistance = Point.distanceNorm2(snappedPoint, splines[snappedSplineIdx][snappedSplineLeadingPoint]) + #Get cumulative alignment distance (total segment distance) + splineDistanceS = splines[snappedSplineIdx].getCumulativeDistance(snappedSplineLeadingPoint) + subsegmentDistance + orthogonalSplineVector = (splines[snappedSplineIdx][snappedSplineLeadingPoint+1]-splines[snappedSplineIdx][snappedSplineLeadingPoint]).orthogonal() + offsetVector = p-snappedPoint + if Point.dot(orthogonalSplineVector, offsetVector) < 0: + minOffsetY = -minOffsetY + return [snappedSplineIdx, snappedSplineLeadingPoint, snappedPoint, subsegmentDistance, splineDistanceS, minOffsetY] + else: + print('Offset for point {} is infinite (check with prepareSplines if some spline segments are aligned with axes)'.format(p)) + return None + +def getXYfromSY(s, y, splineNum, splines, mode = 0): + ''' Find X,Y coordinate from S,Y data. + if mode = 0 : return Snapped X,Y + if mode !=0 : return Real X,Y + ''' + + #(buckle in, it gets ugly from here on out) + ss_spline_d = subsec_spline_dist(splines) + + #Find subsegment + snapped_x = None + snapped_y = None + for spline_ss_index in range(len(ss_spline_d[splineNum][1])): + if(s < ss_spline_d[splineNum][1][spline_ss_index]): + ss_value = s - ss_spline_d[splineNum][1][spline_ss_index-1] + #Get normal vector and then snap + vector_l_x = (splines[splineNum][spline_ss_index][0] - splines[splineNum][spline_ss_index-1][0]) + vector_l_y = (splines[splineNum][spline_ss_index][1] - splines[splineNum][spline_ss_index-1][1]) + magnitude = sqrt(vector_l_x**2 + vector_l_y**2) + n_vector_x = vector_l_x/magnitude + n_vector_y = vector_l_y/magnitude + snapped_x = splines[splineNum][spline_ss_index-1][0] + ss_value*n_vector_x + snapped_y = splines[splineNum][spline_ss_index-1][1] + ss_value*n_vector_y + + #Real values (including orthogonal projection of y)) + real_x = snapped_x - y*n_vector_y + real_y = snapped_y + y*n_vector_x + break + + if mode == 0 or (not snapped_x): + if(not snapped_x): + snapped_x = splines[splineNum][-1][0] + snapped_y = splines[splineNum][-1][1] + return [snapped_x,snapped_y] + else: + return [real_x,real_y] + + +class NormAngle(object): + '''Alternate encoding of a point, by its norm and orientation''' + + def __init__(self, norm, angle): + self.norm = norm + self.angle = angle + + @staticmethod + def fromPoint(p): + norm = p.norm2() + if norm > 0: + angle = p.angle() + else: + angle = 0. + return NormAngle(norm, angle) + + def __add__(self, other): + 'a norm cannot become negative' + return NormAngle(max(self.norm+other.norm, 0), self.angle+other.angle) + + def getPoint(self): + return Point(self.norm*cos(self.angle), self.norm*sin(self.angle)) + + +def predictPositionNoLimit(nTimeSteps, initialPosition, initialVelocity, initialAcceleration = Point(0,0)): + '''Predicts the position in nTimeSteps at constant speed/acceleration''' + return initialVelocity + initialAcceleration.__mul__(nTimeSteps),initialPosition+initialVelocity.__mul__(nTimeSteps) + initialAcceleration.__mul__(nTimeSteps**2*0.5) + +def predictPosition(position, speedOrientation, control, maxSpeed = None): + '''Predicts the position (moving.Point) at the next time step with given control input (deltaSpeed, deltaTheta) + speedOrientation is the other encoding of velocity, (speed, orientation) + speedOrientation and control are NormAngle''' + predictedSpeedTheta = speedOrientation+control + if maxSpeed is not None: + predictedSpeedTheta.norm = min(predictedSpeedTheta.norm, maxSpeed) + predictedPosition = position+predictedSpeedTheta.getPoint() + return predictedPosition, predictedSpeedTheta + + +class FlowVector(object): + '''Class to represent 4-D flow vectors, + ie a position and a velocity''' + def __init__(self, position, velocity): + 'position and velocity should be Point instances' + self.position = position + self.velocity = velocity + + def __add__(self, other): + return FlowVector(self.position+other.position, self.velocity+other.velocity) + + def __mul__(self, alpha): + return FlowVector(self.position.__mul__(alpha), self.velocity.__mul__(alpha)) + + def plot(self, options = '', **kwargs): + plot([self.position.x, self.position.x+self.velocity.x], [self.position.y, self.position.y+self.velocity.y], options, **kwargs) + self.position.plot(options+'x', **kwargs) + + @staticmethod + def similar(f1, f2, maxDistance2, maxDeltavelocity2): + return (f1.position-f2.position).norm2Squared()<maxDistance2 and (f1.velocity-f2.velocity).norm2Squared()<maxDeltavelocity2 + +def intersection(p1, p2, p3, p4): + ''' Intersection point (x,y) of lines formed by the vectors p1-p2 and p3-p4 + http://paulbourke.net/geometry/pointlineplane/''' + dp12 = p2-p1 + dp34 = p4-p3 + #det = (p4.y-p3.y)*(p2.x-p1.x)-(p4.x-p3.x)*(p2.y-p1.y) + det = float(dp34.y*dp12.x-dp34.x*dp12.y) + if det == 0.: + return None + else: + ua = (dp34.x*(p1.y-p3.y)-dp34.y*(p1.x-p3.x))/det + return p1+dp12.__mul__(ua) + +# def intersection(p1, p2, dp1, dp2): +# '''Returns the intersection point between the two lines +# defined by the respective vectors (dp) and origin points (p)''' +# from numpy import matrix +# from numpy.linalg import linalg +# A = matrix([[dp1.y, -dp1.x], +# [dp2.y, -dp2.x]]) +# B = matrix([[dp1.y*p1.x-dp1.x*p1.y], +# [dp2.y*p2.x-dp2.x*p2.y]]) + +# if linalg.det(A) == 0: +# return None +# else: +# intersection = linalg.solve(A,B) +# return Point(intersection[0,0], intersection[1,0]) + +def segmentIntersection(p1, p2, p3, p4): + '''Returns the intersecting point of the segments [p1, p2] and [p3, p4], None otherwise''' + + if (Interval.intersection(Interval(p1.x,p2.x,True), Interval(p3.x,p4.x,True)).empty()) or (Interval.intersection(Interval(p1.y,p2.y,True), Interval(p3.y,p4.y,True)).empty()): + return None + else: + inter = intersection(p1, p2, p3, p4) + if (inter is not None + and utils.inBetween(p1.x, p2.x, inter.x) + and utils.inBetween(p3.x, p4.x, inter.x) + and utils.inBetween(p1.y, p2.y, inter.y) + and utils.inBetween(p3.y, p4.y, inter.y)): + return inter + else: + return None + +def segmentLineIntersection(p1, p2, p3, p4): + '''Indicates if the line going through p1 and p2 intersects inside p3, p4''' + inter = intersection(p1, p2, p3, p4) + if inter is not None and utils.inBetween(p3.x, p4.x, inter.x) and utils.inBetween(p3.y, p4.y, inter.y): + return inter + else: + return None + + +class Trajectory(object): + '''Class for trajectories: temporal sequence of positions + + The class is iterable''' + + def __init__(self, positions=None): + if positions is not None: + self.positions = positions + else: + self.positions = [[],[]] + + @staticmethod + def generate(p, v, nPoints): + t = Trajectory() + p0 = Point(p.x, p.y) + t.addPosition(p0) + for i in range(nPoints-1): + p0 += v + t.addPosition(p0) + return t, Trajectory([[v.x]*nPoints, [v.y]*nPoints]) + + @staticmethod + def load(line1, line2): + return Trajectory([[float(n) for n in line1.split(' ')], + [float(n) for n in line2.split(' ')]]) + + @staticmethod + def fromPointList(points): + t = Trajectory() + if isinstance(points[0], list) or isinstance(points[0], tuple): + for p in points: + t.addPositionXY(p[0],p[1]) + else: + for p in points: + t.addPosition(p) + return t + + def __len__(self): + return len(self.positions[0]) + + def length(self): + return self.__len__() + + def empty(self): + return self.__len__() == 0 + + def __getitem__(self, i): + if isinstance(i, int): + return Point(self.positions[0][i], self.positions[1][i]) + elif isinstance(i, slice): + return Trajectory([self.positions[0][i],self.positions[1][i]]) + else: + raise TypeError("Invalid argument type.") + + def __str__(self): + return ' '.join([self.__getitem__(i).__str__() for i in range(self.length())]) + + def __repr__(self): + return self.__str__() + + def __iter__(self): + self.iterInstantNum = 0 + return self + + def __next__(self): + if self.iterInstantNum >= self.length(): + raise StopIteration + else: + self.iterInstantNum += 1 + return self[self.iterInstantNum-1] + + def __eq__(self, other): + if self.length() == other.length(): + result = True + for p, po in zip(self, other): + result = result and (p == po) + return result + else: + return False + + def setPositionXY(self, i, x, y): + if i < self.__len__(): + self.positions[0][i] = x + self.positions[1][i] = y + + def setPosition(self, i, p): + self.setPositionXY(i, p.x, p.y) + + def addPositionXY(self, x, y): + self.positions[0].append(x) + self.positions[1].append(y) + + def addPosition(self, p): + self.addPositionXY(p.x, p.y) + + def duplicateLastPosition(self): + self.positions[0].append(self.positions[0][-1]) + self.positions[1].append(self.positions[1][-1]) + + @staticmethod + def _plot(positions, options = '', withOrigin = False, lastCoordinate = None, timeStep = 1, objNum = None, **kwargs): + if lastCoordinate is None: + plot(positions[0][::timeStep], positions[1][::timeStep], options, **kwargs) + elif 0 <= lastCoordinate <= len(positions[0]): + plot(positions[0][:lastCoordinate:timeStep], positions[1][:lastCoordinate:timeStep], options, **kwargs) + if withOrigin: + plot([positions[0][0]], [positions[1][0]], 'ro', **kwargs) + if objNum is not None: + text(positions[0][0], positions[1][0], '{}'.format(objNum)) + + def homographyProject(self, homography): + return Trajectory(cvutils.homographyProject(array(self.positions), homography).tolist()) + + def newCameraProject(self, newCameraMatrix): + return Trajectory(cvutils.newCameraProject(array(self.positions), newCameraMatrix).tolist()) + + def plot(self, options = '', withOrigin = False, timeStep = 1, objNum = None, **kwargs): + Trajectory._plot(self.positions, options, withOrigin, None, timeStep, objNum, **kwargs) + + def plotAt(self, lastCoordinate, options = '', withOrigin = False, timeStep = 1, objNum = None, **kwargs): + Trajectory._plot(self.positions, options, withOrigin, lastCoordinate, timeStep, objNum, **kwargs) + + def plotOnWorldImage(self, nPixelsPerUnitDistance, options = '', withOrigin = False, timeStep = 1, objNum = None, **kwargs): + imgPositions = [[x*nPixelsPerUnitDistance for x in self.positions[0]], + [x*nPixelsPerUnitDistance for x in self.positions[1]]] + Trajectory._plot(imgPositions, options, withOrigin, None, timeStep, objNum, **kwargs) + + def getXCoordinates(self): + return self.positions[0] + + def getYCoordinates(self): + return self.positions[1] + + def asArray(self): + return array(self.positions) + + def xBounds(self): + # look for function that does min and max in one pass + return Interval(min(self.getXCoordinates()), max(self.getXCoordinates())) + + def yBounds(self): + # look for function that does min and max in one pass + return Interval(min(self.getYCoordinates()), max(self.getYCoordinates())) + + def add(self, traj2): + '''Returns a new trajectory of the same length''' + if self.length() != traj2.length(): + print('Trajectories of different lengths') + return None + else: + return Trajectory([[a+b for a,b in zip(self.getXCoordinates(),traj2.getXCoordinates())], + [a+b for a,b in zip(self.getYCoordinates(),traj2.getYCoordinates())]]) + + def subtract(self, traj2): + '''Returns a new trajectory of the same length''' + if self.length() != traj2.length(): + print('Trajectories of different lengths') + return None + else: + return Trajectory([[a-b for a,b in zip(self.getXCoordinates(),traj2.getXCoordinates())], + [a-b for a,b in zip(self.getYCoordinates(),traj2.getYCoordinates())]]) + + def __mul__(self, alpha): + '''Returns a new trajectory of the same length''' + return Trajectory([[alpha*x for x in self.getXCoordinates()], + [alpha*y for y in self.getYCoordinates()]]) + + def differentiate(self, doubleLastPosition = False): + diff = Trajectory() + for i in range(1, self.length()): + diff.addPosition(self[i]-self[i-1]) + if doubleLastPosition: + diff.addPosition(diff[-1]) + return diff + + def differentiateSG(self, window_length, polyorder, deriv=0, delta=1.0, axis=-1, mode='interp', cval=0.0, nInstantsIgnoredAtEnds = 2): + '''Differentiates the trajectory using the Savitsky Golay filter + + window_length : The length of the filter window (i.e. the number of coefficients). window_length must be a positive odd integer. + polyorder : The order of the polynomial used to fit the samples. polyorder must be less than window_length. + deriv : The order of the derivative to compute. This must be a nonnegative integer. The default is 0, which means to filter the data without differentiating. + delta : The spacing of the samples to which the filter will be applied. This is only used if deriv > 0. Default is 1.0. + axis : The axis of the array x along which the filter is to be applied. Default is -1. + mode : Must be mirror, constant, nearest, wrap or interp. This determines the type of extension to use for the padded signal to which the filter is applied. When mode is constant, the padding value is given by cval. See the Notes for more details on mirror, constant, wrap, and nearest. When the interp mode is selected (the default), no extension is used. Instead, a degree polyorder polynomial is fit to the last window_length values of the edges, and this polynomial is used to evaluate the last window_length // 2 output values. + cval : Value to fill past the edges of the input if mode is constant. Default is 0.0. + + https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.savgol_filter.html#scipy.signal.savgol_filter''' + if removeBothEnds >=1: + pos = [self.positions[0][nInstantsIgnoredAtEnds:-nInstantsIgnoredAtEnds], + self.positions[1][nInstantsIgnoredAtEnds:-nInstantsIgnoredAtEnds]] + else: + pos = self.positions + filtered = savgol_filter(pos, window_length, polyorder, deriv, delta, axis, mode, cval) + return Trajectory(filtered) + + def norm(self): + '''Returns the list of the norms at each instant''' + return hypot(self.positions[0], self.positions[1]) + + def computeCumulativeDistances(self): + '''Computes the distance from each point to the next and the cumulative distance up to the point + Can be accessed through getDistance(idx) and getCumulativeDistance(idx)''' + self.distances = [] + self.cumulativeDistances = [0.] + p1 = self[0] + cumulativeDistance = 0. + for i in range(self.length()-1): + p2 = self[i+1] + self.distances.append(Point.distanceNorm2(p1,p2)) + cumulativeDistance += self.distances[-1] + self.cumulativeDistances.append(cumulativeDistance) + p1 = p2 + + def getDistance(self,i): + '''Return the distance between points i and i+1''' + if i < self.length()-1: + return self.distances[i] + else: + print('Index {} beyond trajectory length {}-1'.format(i, self.length())) + + def getCumulativeDistance(self, i): + '''Return the cumulative distance between the beginning and point i''' + if i < self.length(): + return self.cumulativeDistances[i] + else: + print('Index {} beyond trajectory length {}'.format(i, self.length())) + + def getMaxDistance(self, metric): + 'Returns the maximum distance between points in the trajectory' + positions = self.getPositions().asArray().T + return cdist(positions, positions, metric = metric).max() + + def getClosestPoint(self, p1, maxDist2 = None): + '''Returns the instant of the closest position in trajectory to p1 (and the point) + if maxDist is not None, will check the distance is smaller + TODO: could use cdist for different metrics''' + distances2 = [] + minDist2 = float('inf') + i = -1 + for p2 in self: + distances2.append(Point.distanceNorm2(p1, p2)) + if distances2[-1] < minDist2: + minDist2 = distances2[-1] + i = len(distances2)-1 + if maxDist2 is not None and minDist2 < maxDist2: + return None + else: + return i + + def similarOrientation(self, refDirection, cosineThreshold, minProportion = 0.5): + '''Indicates whether the minProportion (<=1.) (eg half) of the trajectory elements (vectors for velocity) + have a cosine with refDirection is smaller than cosineThreshold''' + count = 0 + lengthThreshold = float(self.length())*minProportion + for p in self: + if p.similarOrientation(refDirection, cosineThreshold): + count += 1 + return count >= lengthThreshold + + def wiggliness(self): + straightDistance = Point.distanceNorm2(self.__getitem__(0),self.__getitem__(self.length()-1)) + if straightDistance > 0: + return self.getCumulativeDistance(self.length()-1)/float(straightDistance) + else: + return None + + def getIntersections(self, p1, p2): + '''Returns a list of the indices at which the trajectory + intersects with the segment of extremities p1 and p2 + Returns an empty list if there is no crossing''' + indices = [] + intersections = [] + + for i in range(self.length()-1): + q1=self.__getitem__(i) + q2=self.__getitem__(i+1) + p = segmentIntersection(q1, q2, p1, p2) + if p is not None: + if q1.x != q2.x: + ratio = (p.x-q1.x)/(q2.x-q1.x) + elif q1.y != q2.y: + ratio = (p.y-q1.y)/(q2.y-q1.y) + else: + ratio = 0 + indices.append(i+ratio) + intersections.append(p) + return indices, intersections + + def getLineIntersections(self, p1, p2): + '''Returns a list of the indices at which the trajectory + intersects with the line going through p1 and p2 + Returns an empty list if there is no crossing''' + indices = [] + intersections = [] + + for i in range(self.length()-1): + q1=self.__getitem__(i) + q2=self.__getitem__(i+1) + p = segmentLineIntersection(p1, p2, q1, q2) + if p is not None: + if q1.x != q2.x: + ratio = (p.x-q1.x)/(q2.x-q1.x) + elif q1.y != q2.y: + ratio = (p.y-q1.y)/(q2.y-q1.y) + else: + ratio = 0 + indices.append(i+ratio) + intersections.append(p) + return indices, intersections + + def getTrajectoryInInterval(self, inter): + 'Returns all position between index inter.first and index.last (included)' + if inter.first >=0 and inter.last<= self.length(): + return Trajectory([self.positions[0][inter.first:inter.last+1], + self.positions[1][inter.first:inter.last+1]]) + else: + return None + + def subSample(self, step): + 'Returns the positions very step' + return Trajectory([self.positions[0][::step], + self.positions[1][::step]]) + + if shapelyAvailable: + def getInstantsInPolygon(self, polygon): + '''Returns the list of instants at which the trajectory is in the polygon''' + instants = [] + n = self.length() + for t, x, y in zip(range(n), self.positions[0], self.positions[1]): + if polygon.contains(shapelyPoint(x, y)): + instants.append(t) + return instants + + def getTrajectoryInPolygon(self, polygon, t2 = None): + '''Returns the trajectory built with the set of points inside the (shapely) polygon + The polygon could be a prepared polygon (faster) from prepared.prep + + t2 is another trajectory (could be velocities) + which is filtered based on the first (self) trajectory''' + traj = Trajectory() + inPolygon = [] + for x, y in zip(self.positions[0], self.positions[1]): + inPolygon.append(polygon.contains(shapelyPoint(x, y))) + if inPolygon[-1]: + traj.addPositionXY(x, y) + traj2 = Trajectory() + if t2 is not None: + for inp, x, y in zip(inPolygon, t2.positions[0], t2.positions[1]): + if inp: + traj2.addPositionXY(x, y) + return traj, traj2 + + def proportionInPolygon(self, polygon, minProportion = 0.5): + instants = self.getInstantsInPolygon(polygon) + lengthThreshold = float(self.length())*minProportion + return len(instants) >= lengthThreshold + else: + def getTrajectoryInPolygon(self, polygon, t2 = None): + '''Returns the trajectory built with the set of points inside the polygon + (array of Nx2 coordinates of the polygon vertices)''' + traj = Trajectory() + inPolygon = [] + for p in self: + inPolygon.append(p.inPolygon(polygon)) + if inPolygon[-1]: + traj.addPosition(p) + traj2 = Trajectory() + if t2 is not None: + for inp, x, y in zip(inPolygon, t2.positions[0], t2.positions[1]): + if inp: + traj2.addPositionXY(p.x, p.y) + return traj, traj2 + + def proportionInPolygon(self, polygon, minProportion = 0.5): + inPolygon = [p.inPolygon(polygon) for p in self] + lengthThreshold = float(self.length())*minProportion + return sum(inPolygon) >= lengthThreshold + + @staticmethod + def lcss(t1, t2, lcss): + return lcss.compute(t1, t2) + +class CurvilinearTrajectory(Trajectory): + '''Sub class of trajectory for trajectories with curvilinear coordinates and lane assignements + longitudinal coordinate is stored as first coordinate (exterior name S) + lateral coordiante is stored as second coordinate''' + + def __init__(self, S = None, Y = None, lanes = None): + if S is None or Y is None or len(S) != len(Y): + self.positions = [[],[]] + if S is not None and Y is not None and len(S) != len(Y): + print("S and Y coordinates of different lengths\nInitializing to empty lists") + else: + self.positions = [S,Y] + if lanes is None or len(lanes) != self.length(): + self.lanes = [] + else: + self.lanes = lanes + + def __getitem__(self,i): + if isinstance(i, int): + return [self.positions[0][i], self.positions[1][i], self.lanes[i]] + else: + raise TypeError("Invalid argument type.") + #elif isinstance( key, slice ): + + def getSCoordinates(self): + return self.getXCoordinates() + + def getLanes(self): + return self.lanes + + def addPositionSYL(self, s, y, lane): + self.addPositionXY(s,y) + self.lanes.append(lane) + + def addPosition(self, p): + 'Adds position in the point format for curvilinear of list with 3 values' + self.addPositionSYL(p[0], p[1], p[2]) + + def setPosition(self, i, s, y, lane): + self.setPositionXY(i, s, y) + if i < self.__len__(): + self.lanes[i] = lane + + def differentiate(self, doubleLastPosition = False): + diff = CurvilinearTrajectory() + p1 = self[0] + for i in range(1, self.length()): + p2 = self[i] + diff.addPositionSYL(p2[0]-p1[0], p2[1]-p1[1], p1[2]) + p1=p2 + if doubleLastPosition and self.length() > 1: + diff.addPosition(diff[-1]) + return diff + + def getIntersections(self, S1, lane = None): + '''Returns a list of the indices at which the trajectory + goes past the curvilinear coordinate S1 + (in provided lane if lane is not None) + Returns an empty list if there is no crossing''' + indices = [] + for i in range(self.length()-1): + q1=self.__getitem__(i) + q2=self.__getitem__(i+1) + if q1[0] <= S1 < q2[0] and (lane is None or (self.lanes[i] == lane and self.lanes[i+1] == lane)): + indices.append(i+(S1-q1[0])/(q2[0]-q1[0])) + return indices + +################## +# Moving Objects +################## + +userTypeNames = ['unknown', + 'car', + 'pedestrian', + 'motorcycle', + 'bicycle', + 'bus', + 'truck'] + +userType2Num = utils.inverseEnumeration(userTypeNames) + +class CarClassifier: + def predict(self, hog): + return userType2Num['car'] +carClassifier = CarClassifier() + +class MovingObject(STObject, VideoFilenameAddable): + '''Class for moving objects: a spatio-temporal object + with a trajectory and a geometry (constant volume over time) + and a usertype (e.g. road user) coded as a number (see userTypeNames) + ''' + + def __init__(self, num = None, timeInterval = None, positions = None, velocities = None, geometry = None, userType = userType2Num['unknown']): + super(MovingObject, self).__init__(num, timeInterval) + self.positions = positions + self.velocities = velocities + self.geometry = geometry + self.userType = userType + self.features = None + # compute bounding polygon from trajectory + + @staticmethod + def aggregateTrajectories(features, interval = None, aggFunc = mean): + 'Computes the aggregate trajectory from list of MovingObject features' + positions = Trajectory() + velocities = Trajectory() + if interval is None: + inter = TimeInterval.unionIntervals([f.getTimeInterval() for f in features]) + else: + inter = interval + for t in inter: + points = [] + vels = [] + for f in features: + if f.existsAtInstant(t): + points.append(f.getPositionAtInstant(t)) + vels.append(f.getVelocityAtInstant(t)) + positions.addPosition(Point.agg(points, aggFunc)) + velocities.addPosition(Point.agg(vels, aggFunc)) + return inter, positions, velocities + + @staticmethod + def generate(num, p, v, timeInterval): + positions, velocities = Trajectory.generate(p, v, int(timeInterval.length())) + return MovingObject(num = num, timeInterval = timeInterval, positions = positions, velocities = velocities) + + def updatePositions(self): + inter, self.positions, self.velocities = MovingObject.aggregateTrajectories(self.features, self.getTimeInterval()) + + @staticmethod + def concatenate(obj1, obj2, num = None, newFeatureNum = None, computePositions = False): + '''Concatenates two objects, whether overlapping temporally or not + + Positions will be recomputed only if computePositions is True + Otherwise, only featureNumbers and/or features will be merged''' + if num is None: + newNum = obj1.getNum() + else: + newNum = num + commonTimeInterval = obj1.commonTimeInterval(obj2) + if commonTimeInterval.empty(): + #print('The two objects\' time intervals do not overlap: obj1 {} and obj2 {}'.format(obj1.getTimeInterval(), obj2.getTimeInterval())) + emptyInterval = TimeInterval(min(obj1.getLastInstant(),obj2.getLastInstant()), max(obj1.getFirstInstant(),obj2.getFirstInstant())) + if obj1.existsAtInstant(emptyInterval.last): + firstObject = obj2 + secondObject = obj1 + else: + firstObject = obj1 + secondObject = obj2 + v = (secondObject.getPositionAtInstant(emptyInterval.last)-firstObject.getPositionAtInstant(emptyInterval.first)).divide(emptyInterval.length()-1) + positions = copy.deepcopy(firstObject.getPositions()) + velocities = copy.deepcopy(firstObject.getPositions()) + featurePositions = Trajectory() + featureVelocities = Trajectory() + p = firstObject.getPositionAtInstant(emptyInterval.first)+v + for t in range(emptyInterval.first+1, emptyInterval.last): + positions.addPosition(p) + velocities.addPosition(v) + featurePositions.addPosition(p) + featureVelocities.addPosition(v) + p=p+v + for t in secondObject.getTimeInterval(): + positions.addPosition(secondObject.getPositionAtInstant(t)) + velocities.addPosition(secondObject.getVelocityAtInstant(t)) + newObject = MovingObject(newNum, TimeInterval(firstObject.getFirstInstant(), secondObject.getLastInstant()), positions, velocities) + if hasattr(obj1, 'featureNumbers') and hasattr(obj2, 'featureNumbers'): + if newFeatureNum is not None: + newObject.featureNumbers = obj1.featureNumbers+obj2.featureNumbers+[newFeatureNum] + else: + print('Issue, new created feature has no num id') + if obj1.hasFeatures() and obj2.hasFeatures(): + newObject.features = obj1.getFeatures()+obj2.getFeatures()+[MovingObject(newFeatureNum, TimeInterval(emptyInterval.first+1, emptyInterval.last-1), featurePositions, featureVelocities)] + else: # time intervals overlap + newTimeInterval = TimeInterval.union(obj1.getTimeInterval(), obj2.getTimeInterval()) + newObject = MovingObject(newNum, newTimeInterval) + if hasattr(obj1, 'featureNumbers') and hasattr(obj2, 'featureNumbers'): + newObject.featureNumbers = obj1.featureNumbers+obj2.featureNumbers + if obj1.hasFeatures() and obj2.hasFeatures(): + newObject.features = obj1.getFeatures()+obj2.getFeatures() + newObject.updatePositions() + else: + print('Cannot update object positions without features') + # user type + if obj1.getUserType() != obj2.getUserType(): + print('The two moving objects have different user types: obj1 {} obj2 {}'.format(userTypeNames[obj1.getUserType()], userTypeNames[obj2.getUserType()])) + newObject.setUserType(obj1.getUserType()) + return newObject + + def getObjectInTimeInterval(self, inter): + '''Returns a new object extracted from self, + restricted to time interval inter''' + intersection = TimeInterval.intersection(inter, self.getTimeInterval()) + if not intersection.empty(): + trajectoryInterval = TimeInterval(intersection.first-self.getFirstInstant(), intersection.last-self.getFirstInstant()) + obj = MovingObject(self.num, intersection, self.positions.getTrajectoryInInterval(trajectoryInterval), self.geometry, self.userType) + if self.velocities is not None: + obj.velocities = self.velocities.getTrajectoryInInterval(trajectoryInterval) + return obj + else: + print('The object does not exist at {}'.format(inter)) + return None + + def getObjectsInMask(self, mask, homography = None, minLength = 1): + '''Returns new objects made of the positions in the mask + mask is in the destination of the homography space''' + if homography is not None: + self.projectedPositions = self.positions.homographyProject(homography) + else: + self.projectedPositions = self.positions + def inMask(positions, i, mask): + p = positions[i] + return mask[int(p.y), int(p.x)] != 0. + + #subTimeIntervals self.getFirstInstant()+i + filteredIndices = [inMask(self.projectedPositions, i, mask) for i in range(int(self.length()))] + # 'connected components' in subTimeIntervals + l = 0 + intervalLabels = [] + prev = True + for i in filteredIndices: + if i: + if not prev: # new interval + l += 1 + intervalLabels.append(l) + else: + intervalLabels.append(-1) + prev = i + intervalLabels = array(intervalLabels) + subObjects = [] + for l in set(intervalLabels): + if l >= 0: + if sum(intervalLabels == l) >= minLength: + times = [self.getFirstInstant()+i for i in range(len(intervalLabels)) if intervalLabels[i] == l] + subTimeInterval = TimeInterval(min(times), max(times)) + subObjects.append(self.getObjectInTimeInterval(subTimeInterval)) + + return subObjects + + def getPositions(self): + return self.positions + + def getVelocities(self): + return self.velocities + + def getUserType(self): + return self.userType + + def computeCumulativeDistances(self): + self.positions.computeCumulativeDistances() + + def getCurvilinearPositions(self): + if hasattr(self, 'curvilinearPositions'): + return self.curvilinearPositions + else: + return None + + def plotCurvilinearPositions(self, lane = None, options = '', withOrigin = False, **kwargs): + if hasattr(self, 'curvilinearPositions'): + if lane is None: + plot(list(self.getTimeInterval()), self.curvilinearPositions.positions[0], options, **kwargs) + if withOrigin: + plot([self.getFirstInstant()], [self.curvilinearPositions.positions[0][0]], 'ro', **kwargs) + else: + instants = [] + coords = [] + for t, p in zip(self.getTimeInterval(), self.curvilinearPositions): + if p[2] == lane: + instants.append(t) + coords.append(p[0]) + else: + instants.append(NaN) + coords.append(NaN) + plot(instants, coords, options, **kwargs) + if withOrigin and len(instants)>0: + plot([instants[0]], [coords[0]], 'ro', **kwargs) + else: + print('Object {} has no curvilinear positions'.format(self.getNum())) + + def setUserType(self, userType): + self.userType = userType + + def setFeatures(self, features, featuresOrdered = False): + '''Sets the features in the features field based on featureNumbers + if not all features are loaded from 0, one needs to renumber in a dict''' + if featuresOrdered: + tmp = features + else: + tmp = {f.getNum():f for f in features} + self.features = [tmp[i] for i in self.featureNumbers] + + def getFeatures(self): + return self.features + + def hasFeatures(self): + return (self.features is not None) + + def getFeature(self, i): + if self.hasFeatures() and i<len(self.features): + return self.features[i] + else: + return None + + def getNLongestFeatures(self, nFeatures = 1): + if self.features is None: + return [] + else: + tmp = utils.sortByLength(self.getFeatures(), reverse = True) + return tmp[:min(len(tmp), nFeatures)] + + def getFeatureNumbers(self): + '''Returns the number of features at each instant + dict instant -> number of features''' + if self.hasFeatures(): + featureNumbers = {} + for t in self.getTimeInterval(): + n = 0 + for f in self.getFeatures(): + if f.existsAtInstant(t): + n += 1 + featureNumbers[t]=n + return featureNumbers + else: + print('Object {} has no features loaded.'.format(self.getNum())) + return None + + def getSpeeds(self, nInstantsIgnoredAtEnds = 0): + speeds = self.getVelocities().norm() + if nInstantsIgnoredAtEnds > 0: + n = min(nInstantsIgnoredAtEnds, int(floor(self.length()/2.))) + return speeds[n:-n] + else: + return speeds + + def getAccelerations(self, window_length, polyorder, delta=1.0, axis=-1, mode='interp', cval=0.0, speeds = None, nInstantsIgnoredAtEnds = 0): + '''Returns the 1-D acceleration from the 1-D speeds + Caution about previously filtered data''' + if speeds is None: + speeds = self.getSpeeds(nInstantsIgnoredAtEnds) + return savgol_filter(speeds, window_length, polyorder, 1, delta, axis, mode, cval) + + def getSpeedIndicator(self): + from indicators import SeverityIndicator + return SeverityIndicator('Speed', {t:self.getVelocityAtInstant(t).norm2() for t in self.getTimeInterval()}) + + def getPositionAt(self, i): + return self.positions[i] + + def getVelocityAt(self, i): + return self.velocities[i] + + def getPositionAtInstant(self, i): + return self.positions[i-self.getFirstInstant()] + + def getVelocityAtInstant(self, i): + return self.velocities[i-self.getFirstInstant()] + + def getXCoordinates(self): + return self.positions.getXCoordinates() + + def getYCoordinates(self): + return self.positions.getYCoordinates() + + def plot(self, options = '', withOrigin = False, timeStep = 1, withFeatures = False, withIds = False, **kwargs): + if withIds: + objNum = self.getNum() + else: + objNum = None + if withFeatures and self.hasFeatures(): + for f in self.getFeatures(): + f.positions.plot('r', True, timeStep, **kwargs) + self.positions.plot('bx-', True, timeStep, objNum, **kwargs) + else: + self.positions.plot(options, withOrigin, timeStep, objNum, **kwargs) + + def plotOnWorldImage(self, nPixelsPerUnitDistance, options = '', withOrigin = False, timeStep = 1, withIds = False, **kwargs): + if withIds: + self.positions.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, self.getNum(), **kwargs) + else: + self.positions.plotOnWorldImage(nPixelsPerUnitDistance, options, withOrigin, timeStep, None, **kwargs) + + def play(self, videoFilename, homography = None, undistort = False, intrinsicCameraMatrix = None, distortionCoefficients = None, undistortedImageMultiplication = 1.): + cvutils.displayTrajectories(videoFilename, [self], homography = homography, firstFrameNum = self.getFirstInstant(), lastFrameNumArg = self.getLastInstant(), undistort = undistort, intrinsicCameraMatrix = intrinsicCameraMatrix, distortionCoefficients = distortionCoefficients, undistortedImageMultiplication = undistortedImageMultiplication) + + def speedDiagnostics(self, framerate = 1., display = False, nInstantsIgnoredAtEnds=0): + speeds = framerate*self.getSpeeds(nInstantsIgnoredAtEnds) + coef = utils.linearRegression(list(range(len(speeds))), speeds) + print('min/5th perc speed: {} / {}\nspeed diff: {}\nspeed stdev: {}\nregression: {}'.format(min(speeds), scoreatpercentile(speeds, 5), speeds[-2]-speeds[1], std(speeds), coef[0])) + if display: + from matplotlib.pyplot import figure, axis + figure(1) + self.plot() + axis('equal') + figure(2) + plot(list(self.getTimeInterval()), speeds) + figure(3) + plot(list(self.getTimeInterval()), self.getAccelerations(9, 3, speeds = speeds)) # arbitrary parameter + + @staticmethod + def minMaxDistance(obj1, obj2): + '''Computes the min max distance used for feature grouping''' + commonTimeInterval = obj1.commonTimeInterval(obj2) + if not commonTimeInterval.empty(): + minDistance = (obj1.getPositionAtInstant(commonTimeInterval.first)-obj2.getPositionAtInstant(commonTimeInterval.first)).norm2() + maxDistance = minDistance + for t in list(commonTimeInterval)[1:]: + d = (obj1.getPositionAtInstant(t)-obj2.getPositionAtInstant(t)).norm2() + if d<minDistance: + minDistance = d + elif d>maxDistance: + maxDistance = d + return int(commonTimeInterval.length()), minDistance, maxDistance + else: + return int(commonTimeInterval.length()), None, None + + @staticmethod + def distances(obj1, obj2, instant1, _instant2 = None): + '''Returns the distances between all features of the 2 objects + at the same instant instant1 + or at instant1 and instant2''' + if _instant2 is None: + instant2 = instant1 + else: + instant2 = _instant2 + positions1 = [f.getPositionAtInstant(instant1).astuple() for f in obj1.features if f.existsAtInstant(instant1)] + positions2 = [f.getPositionAtInstant(instant2).astuple() for f in obj2.features if f.existsAtInstant(instant2)] + return cdist(positions1, positions2, metric = 'euclidean') + + @staticmethod + def minDistance(obj1, obj2, instant1, instant2 = None): + return MovingObject.distances(obj1, obj2, instant1, instant2).min() + + @staticmethod + def maxDistance(obj1, obj2, instant, instant2 = None): + return MovingObject.distances(obj1, obj2, instant1, instant2).max() + + def maxSize(self): + '''Returns the max distance between features + at instant there are the most features''' + if hasattr(self, 'features'): + nFeatures = -1 + tMaxFeatures = 0 + for t in self.getTimeInterval(): + n = len([f for f in self.features if f.existsAtInstant(t)]) + if n > nFeatures: + nFeatures = n + tMaxFeatures = t + return MovingObject.maxDistance(self, self, tMaxFeatures) + else: + print('Load features to compute a maximum size') + return None + + def setRoutes(self, startRouteID, endRouteID): + self.startRouteID = startRouteID + self.endRouteID = endRouteID + + def getInstantsCrossingLane(self, p1, p2): + '''Returns the instant(s) + at which the object passes from one side of the segment to the other + empty list if there is no crossing''' + indices, intersections = self.positions.getIntersections(p1, p2) + return [t+self.getFirstInstant() for t in indices] + + def computeTrajectorySimilarities(self, prototypes, lcss): + 'Computes the similarities to the prototypes using the LCSS' + if not hasattr(self, 'prototypeSimilarities'): + self.prototypeSimilarities = [] + for proto in prototypes: + lcss.similarities(proto.getMovingObject().getPositions().asArray().T, self.getPositions().asArray().T) + similarities = lcss.similarityTable[-1, :-1].astype(float) + self.prototypeSimilarities.append(similarities/minimum(arange(1.,len(similarities)+1), proto.getMovingObject().length()*ones(len(similarities)))) + + @staticmethod + def computePET(obj1, obj2, collisionDistanceThreshold): + '''Post-encroachment time based on distance threshold + + Returns the smallest time difference when the object positions are within collisionDistanceThreshold + and the instants at which each object is passing through its corresponding position''' + positions1 = [p.astuple() for p in obj1.getPositions()] + positions2 = [p.astuple() for p in obj2.getPositions()] + n1 = len(positions1) + n2 = len(positions2) + pets = zeros((n1, n2)) + for i,t1 in enumerate(obj1.getTimeInterval()): + for j,t2 in enumerate(obj2.getTimeInterval()): + pets[i,j] = abs(t1-t2) + distances = cdist(positions1, positions2, metric = 'euclidean') + smallDistances = (distances <= collisionDistanceThreshold) + if smallDistances.any(): + smallPets = pets[smallDistances] + petIdx = smallPets.argmin() + distanceIndices = argwhere(smallDistances)[petIdx] + return smallPets[petIdx], obj1.getFirstInstant()+distanceIndices[0], obj2.getFirstInstant()+distanceIndices[1] + else: + return None, None, None + + def predictPosition(self, instant, nTimeSteps, externalAcceleration = Point(0,0)): + '''Predicts the position of object at instant+deltaT, + at constant speed''' + return predictPositionNoLimit(nTimeSteps, self.getPositionAtInstant(instant), self.getVelocityAtInstant(instant), externalAcceleration) + + def projectCurvilinear(self, alignments, ln_mv_av_win=3): + ''' Add, for every object position, the class 'moving.CurvilinearTrajectory()' + (curvilinearPositions instance) which holds information about the + curvilinear coordinates using alignment metadata. + From Paul St-Aubin's PVA tools + ====== + + Input: + ====== + alignments = a list of alignments, where each alignment is a list of + points (class Point). + ln_mv_av_win = moving average window (in points) in which to smooth + lane changes. As per tools_math.cat_mvgavg(), this term + is a search *radius* around the center of the window. + + ''' + + self.curvilinearPositions = CurvilinearTrajectory() + + #For each point + for i in range(int(self.length())): + result = getSYfromXY(self.getPositionAt(i), alignments) + + # Error handling + if(result is None): + print('Warning: trajectory {} at point {} {} has alignment errors (spline snapping)\nCurvilinear trajectory could not be computed'.format(self.getNum(), i, self.getPositionAt(i))) + else: + [align, alignPoint, snappedPoint, subsegmentDistance, S, Y] = result + self.curvilinearPositions.addPositionSYL(S, Y, align) + + ## Go back through points and correct lane + #Run through objects looking for outlier point + smoothed_lanes = utils.cat_mvgavg(self.curvilinearPositions.getLanes(),ln_mv_av_win) + ## Recalculate projected point to new lane + lanes = self.curvilinearPositions.getLanes() + if(lanes != smoothed_lanes): + for i in range(len(lanes)): + if(lanes[i] != smoothed_lanes[i]): + result = getSYfromXY(self.getPositionAt(i),[alignments[smoothed_lanes[i]]]) + + # Error handling + if(result is None): + ## This can be triggered by tracking errors when the trajectory jumps around passed another alignment. + print(' Warning: trajectory {} at point {} {} has alignment errors during trajectory smoothing and will not be corrected.'.format(self.getNum(), i, self.getPositionAt(i))) + else: + [align, alignPoint, snappedPoint, subsegmentDistance, S, Y] = result + self.curvilinearPositions.setPosition(i, S, Y, align) + + def computeSmoothTrajectory(self, minCommonIntervalLength): + '''Computes the trajectory as the mean of all features + if a feature exists, its position is + + Warning work in progress + TODO? not use the first/last 1-.. positions''' + nFeatures = len(self.features) + if nFeatures == 0: + print('Empty object features\nCannot compute smooth trajectory') + else: + # compute the relative position vectors + relativePositions = {} # relativePositions[(i,j)] is the position of j relative to i + for i in range(nFeatures): + for j in range(i): + fi = self.features[i] + fj = self.features[j] + inter = fi.commonTimeInterval(fj) + if inter.length() >= minCommonIntervalLength: + xi = array(fi.getXCoordinates()[inter.first-fi.getFirstInstant():int(fi.length())-(fi.getLastInstant()-inter.last)]) + yi = array(fi.getYCoordinates()[inter.first-fi.getFirstInstant():int(fi.length())-(fi.getLastInstant()-inter.last)]) + xj = array(fj.getXCoordinates()[inter.first-fj.getFirstInstant():int(fj.length())-(fj.getLastInstant()-inter.last)]) + yj = array(fj.getYCoordinates()[inter.first-fj.getFirstInstant():int(fj.length())-(fj.getLastInstant()-inter.last)]) + relativePositions[(i,j)] = Point(median(xj-xi), median(yj-yi)) + relativePositions[(j,i)] = -relativePositions[(i,j)] + + ### + # User Type Classification + ### + def classifyUserTypeSpeedMotorized(self, threshold, aggregationFunc = median, nInstantsIgnoredAtEnds = 0): + '''Classifies slow and fast road users + slow: non-motorized -> pedestrians + fast: motorized -> cars + + aggregationFunc can be any function that can be applied to a vector of speeds, including percentile: + aggregationFunc = lambda x: percentile(x, percentileFactor) # where percentileFactor is 85 for 85th percentile''' + speeds = self.getSpeeds(nInstantsIgnoredAtEnds) + if aggregationFunc(speeds) >= threshold: + self.setUserType(userType2Num['car']) + else: + self.setUserType(userType2Num['pedestrian']) + + def classifyUserTypeSpeed(self, speedProbabilities, aggregationFunc = median, nInstantsIgnoredAtEnds = 0): + '''Classifies road user per road user type + speedProbabilities are functions return P(speed|class) + in a dictionary indexed by user type names + Returns probabilities for each class + + for simple threshold classification, simply pass non-overlapping indicator functions (membership) + e.g. def indic(x): + if abs(x-mu) < sigma: + return 1 + else: + return x''' + if not hasattr(self, 'aggregatedSpeed'): + self.aggregatedSpeed = aggregationFunc(self.getSpeeds(nInstantsIgnoredAtEnds)) + userTypeProbabilities = {} + for userTypename in speedProbabilities: + userTypeProbabilities[userType2Num[userTypename]] = speedProbabilities[userTypename](self.aggregatedSpeed) + self.setUserType(utils.argmaxDict(userTypeProbabilities)) + return userTypeProbabilities + + def initClassifyUserTypeHoGSVM(self, aggregationFunc, pedBikeCarSVM, bikeCarSVM = None, pedBikeSpeedTreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), nInstantsIgnoredAtEnds = 0, homography = None, intrinsicCameraMatrix = None, distortionCoefficients = None): + '''Initializes the data structures for classification + + TODO? compute speed for longest feature?''' + self.aggregatedSpeed = aggregationFunc(self.getSpeeds(nInstantsIgnoredAtEnds)) + if self.aggregatedSpeed < pedBikeSpeedTreshold or bikeCarSVM is None: + self.appearanceClassifier = pedBikeCarSVM + elif self.aggregatedSpeed < bikeCarSpeedThreshold: + self.appearanceClassifier = bikeCarSVM + else: + self.appearanceClassifier = carClassifier + # project feature positions + if self.hasFeatures(): + for f in self.getFeatures(): + pp = cvutils.worldToImageProject(f.getPositions().asArray(), intrinsicCameraMatrix, distortionCoefficients, homography).tolist() + f.positions = Trajectory(pp) + self.userTypes = {} + + def classifyUserTypeHoGSVMAtInstant(self, img, instant, width, height, px, py, minNPixels, rescaleSize, orientations, pixelsPerCell, cellsPerBlock, blockNorm): + '''Extracts the image box around the object + (of square size max(width, height) of the box around the features, + with an added px or py for width and height (around the box)) + computes HOG on this cropped image (with parameters rescaleSize, orientations, pixelsPerCell, cellsPerBlock) + and applies the SVM model on it''' + croppedImg = cvutils.imageBox(img, self, instant, width, height, px, py, minNPixels) + if croppedImg is not None and len(croppedImg) > 0: + hog = cvutils.HOG(croppedImg, rescaleSize, orientations, pixelsPerCell, cellsPerBlock, blockNorm) + self.userTypes[instant] = self.appearanceClassifier.predict(hog.reshape(1,hog.size)) + else: + self.userTypes[instant] = userType2Num['unknown'] + + def classifyUserTypeHoGSVM(self, pedBikeCarSVM = None, width = 0, height = 0, homography = None, images = None, bikeCarSVM = None, pedBikeSpeedTreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), minSpeedEquiprobable = -1, speedProbabilities = None, aggregationFunc = median, maxPercentUnknown = 0.5, nInstantsIgnoredAtEnds = 0, px = 0.2, py = 0.2, minNPixels = 800, rescaleSize = (64, 64), orientations = 9, pixelsPerCell = (8,8), cellsPerBlock = (2,2)): + '''Agregates SVM detections in each image and returns probability + (proportion of instants with classification in each category) + + images is a dictionary of images indexed by instant + With default parameters, the general (ped-bike-car) classifier will be used + + Considered categories are the keys of speedProbabilities''' + if not hasattr(self, 'aggregatedSpeed') or not hasattr(self, 'userTypes'): + print('Initializing the data structures for classification by HoG-SVM') + self.initClassifyUserTypeHoGSVM(aggregationFunc, pedBikeCarSVM, bikeCarSVM, pedBikeSpeedTreshold, bikeCarSpeedThreshold, nInstantsIgnoredAtEnds) + + if len(self.userTypes) != self.length() and images is not None: # if classification has not been done previously + for t in self.getTimeInterval(): + if t not in self.userTypes: + self.classifyUserTypeHoGSVMAtInstant(images[t], t, homography, width, height, px, py, minNPixels, rescaleSize, orientations, pixelsPerCell, cellsPerBlock) + # compute P(Speed|Class) + if speedProbabilities is None or self.aggregatedSpeed < minSpeedEquiprobable: # equiprobable information from speed + userTypeProbabilities = {userType2Num['car']: 1., userType2Num['pedestrian']: 1., userType2Num['bicycle']: 1.} + else: + userTypeProbabilities = {userType2Num[userTypename]: speedProbabilities[userTypename](self.aggregatedSpeed) for userTypename in speedProbabilities} + # compute P(Class|Appearance) + nInstantsUserType = {userTypeNum: 0 for userTypeNum in userTypeProbabilities}# number of instants the object is classified as userTypename + nInstantsUserType[userType2Num['unknown']] = 0 + for t in self.userTypes: + nInstantsUserType[self.userTypes[t]] += 1 #nInstantsUserType.get(self.userTypes[t], 0) + 1 + # result is P(Class|Appearance) x P(Speed|Class) + if nInstantsUserType[userType2Num['unknown']] < maxPercentUnknown*self.length(): # if not too many unknowns + for userTypeNum in userTypeProbabilities: + userTypeProbabilities[userTypeNum] *= nInstantsUserType[userTypeNum] + # class is the user type that maximizes usertype probabilities + if nInstantsUserType[userType2Num['unknown']] >= maxPercentUnknown*self.length() and (speedProbabilities is None or self.aggregatedSpeed < minSpeedEquiprobable): # if no speed information and too many unknowns + self.setUserType(userType2Num['unknown']) + else: + self.setUserType(utils.argmaxDict(userTypeProbabilities)) + + def classifyUserTypeArea(self, areas, homography): + '''Classifies the object based on its location (projected to image space) + areas is a dictionary of matrix of the size of the image space + for different road users possible locations, indexed by road user type names + + TODO: areas could be a wrapper object with a contains method that would work for polygons and images (with wrapper class) + skip frames at beginning/end?''' + print('not implemented/tested yet') + if not hasattr(self, projectedPositions): + if homography is not None: + self.projectedPositions = obj.positions.homographyProject(homography) + else: + self.projectedPositions = obj.positions + possibleUserTypes = {userType: 0 for userType in range(len(userTypenames))} + for p in self.projectedPositions: + for userTypename in areas: + if areas[userTypename][p.x, p.y] != 0: + possibleUserTypes[userType2Enum[userTypename]] += 1 + # what to do: threshold for most common type? self.setUserType() + return possibleUserTypes + + @staticmethod + def collisionCourseDotProduct(movingObject1, movingObject2, instant): + 'A positive result indicates that the road users are getting closer' + deltap = movingObject1.getPositionAtInstant(instant)-movingObject2.getPositionAtInstant(instant) + deltav = movingObject2.getVelocityAtInstant(instant)-movingObject1.getVelocityAtInstant(instant) + return Point.dot(deltap, deltav) + + @staticmethod + def collisionCourseCosine(movingObject1, movingObject2, instant): + 'A positive result indicates that the road users are getting closer' + return Point.cosine(movingObject1.getPositionAtInstant(instant)-movingObject2.getPositionAtInstant(instant), #deltap + movingObject2.getVelocityAtInstant(instant)-movingObject1.getVelocityAtInstant(instant)) #deltav + + +class Prototype(object): + 'Class for a prototype' + + def __init__(self, filename, num, trajectoryType, nMatchings = None): + self.filename = filename + self.num = num + self.trajectoryType = trajectoryType + self.nMatchings = nMatchings + self.movingObject = None + + def getFilename(self): + return self.filename + def getNum(self): + return self.num + def getTrajectoryType(self): + return self.trajectoryType + def getNMatchings(self): + return self.nMatchings + def getMovingObject(self): + return self.movingObject + def setMovingObject(self, o): + self.movingObject = o + + +################## +# Annotations +################## + +class BBMovingObject(MovingObject): + '''Class for a moving object represented as a bounding box + used for series of ground truth annotations using bounding boxes + and for the output of Urban Tracker http://www.jpjodoin.com/urbantracker/ + + By default in image space + + Its center is the center of the box (generalize to other shapes?) + (computed after projecting if homography available) + ''' + + def __init__(self, num = None, timeInterval = None, topLeftPositions = None, bottomRightPositions = None, userType = userType2Num['unknown']): + super(BBMovingObject, self).__init__(num, timeInterval, userType = userType) + self.topLeftPositions = topLeftPositions.getPositions() + self.bottomRightPositions = bottomRightPositions.getPositions() + + def computeCentroidTrajectory(self, homography = None): + self.positions = self.topLeftPositions.add(self.bottomRightPositions).__mul__(0.5) + if homography is not None: + self.positions = self.positions.homographyProject(homography) + + def matches(self, obj, instant, matchingDistance): + '''Indicates if the annotation matches obj (MovingObject) + with threshold matchingDistance + Returns distance if below matchingDistance, matchingDistance+1 otherwise + (returns an actual value, otherwise munkres does not terminate)''' + d = Point.distanceNorm2(self.getPositionAtInstant(instant), obj.getPositionAtInstant(instant)) + if d < matchingDistance: + return d + else: + return matchingDistance + 1 + +def computeClearMOT(annotations, objects, matchingDistance, firstInstant, lastInstant, returnMatches = False, debug = False): + '''Computes the CLEAR MOT metrics + + Reference: + Keni, Bernardin, and Stiefelhagen Rainer. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008 (2008) + + objects and annotations are supposed to in the same space + current implementation is BBMovingObject (bounding boxes) + mathingDistance is threshold on matching between annotation and object + + TO: tracker output (objects) + GT: ground truth (annotations) + + Output: returns motp, mota, mt, mme, fpt, gt + mt number of missed GT.frames (sum of the number of GT not detected in each frame) + mme number of mismatches + fpt number of false alarm.frames (tracker objects without match in each frame) + gt number of GT.frames + + if returnMatches is True, return as 2 new arguments the GT and TO matches + matches is a dict + matches[i] is the list of matches for GT/TO i + the list of matches is a dict, indexed by time, for the TO/GT id matched at time t + (an instant t not present in matches[i] at which GT/TO exists means a missed detection or false alarm) + + TODO: Should we use the distance as weights or just 1/0 if distance below matchingDistance? + (add argument useDistanceForWeights = False)''' + from munkres import Munkres + + munk = Munkres() + dist = 0. # total distance between GT and TO + ct = 0 # number of associations between GT and tracker output in each frame + gt = 0 # number of GT.frames + mt = 0 # number of missed GT.frames (sum of the number of GT not detected in each frame) + fpt = 0 # number of false alarm.frames (tracker objects without match in each frame) + mme = 0 # number of mismatches + matches = {} # match[i] is the tracker track associated with GT i (using object references) + if returnMatches: + gtMatches = {a.getNum():{} for a in annotations} + toMatches = {o.getNum():{} for o in objects} + else: + gtMatches = None + toMatches = None + for t in range(firstInstant, lastInstant+1): + previousMatches = matches.copy() + # go through currently matched GT-TO and check if they are still matched withing matchingDistance + toDelete = [] + for a in matches: + if a.existsAtInstant(t) and matches[a].existsAtInstant(t): + d = a.matches(matches[a], t, matchingDistance) + if d < matchingDistance: + dist += d + else: + toDelete.append(a) + else: + toDelete.append(a) + for a in toDelete: + del matches[a] + + # match all unmatched GT-TO + matchedGTs = list(matches.keys()) + matchedTOs = list(matches.values()) + costs = [] + unmatchedGTs = [a for a in annotations if a.existsAtInstant(t) and a not in matchedGTs] + unmatchedTOs = [o for o in objects if o.existsAtInstant(t) and o not in matchedTOs] + nGTs = len(matchedGTs)+len(unmatchedGTs) + nTOs = len(matchedTOs)+len(unmatchedTOs) + if len(unmatchedTOs) > 0: + for a in unmatchedGTs: + costs.append([a.matches(o, t, matchingDistance) for o in unmatchedTOs]) + if len(costs) > 0: + newMatches = munk.compute(costs) + for k,v in newMatches: + if costs[k][v] < matchingDistance: + matches[unmatchedGTs[k]]=unmatchedTOs[v] + dist += costs[k][v] + if debug: + print('{} '.format(t)+', '.join(['{} {}'.format(k.getNum(), v.getNum()) for k,v in matches.items()])) + if returnMatches: + for a,o in matches.items(): + gtMatches[a.getNum()][t] = o.getNum() + toMatches[o.getNum()][t] = a.getNum() + + # compute metrics elements + ct += len(matches) + mt += nGTs-len(matches) + fpt += nTOs-len(matches) + gt += nGTs + # compute mismatches + # for gt that do not appear in both frames, check if the corresponding to was matched to another gt in previous/next frame + mismatches = [] + for a in matches: + if a in previousMatches: + if matches[a] != previousMatches[a]: + mismatches.append(a) + elif matches[a] in list(previousMatches.values()): + mismatches.append(matches[a]) + for a in previousMatches: + if a not in matches and previousMatches[a] in list(matches.values()): + mismatches.append(previousMatches[a]) + if debug: + for mm in set(mismatches): + print('{} {}'.format(type(mm), mm.getNum())) + # some object mismatches may appear twice + mme += len(set(mismatches)) + + if ct > 0: + motp = dist/ct + else: + motp = None + if gt > 0: + mota = 1.-float(mt+fpt+mme)/gt + else: + mota = None + return motp, mota, mt, mme, fpt, gt, gtMatches, toMatches + +def plotRoadUsers(objects, colors): + '''Colors is a PlottingPropertyValues instance''' + from matplotlib.pyplot import figure, axis + figure() + for obj in objects: + obj.plot(colors.get(obj.userType)) + axis('equal') + + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/moving.txt') + #suite = doctest.DocTestSuite() + unittest.TextTestRunner().run(suite) + #doctest.testmod() + #doctest.testfile("example.txt") + if shapelyAvailable: + suite = doctest.DocFileSuite('tests/moving_shapely.txt') + unittest.TextTestRunner().run(suite)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/objectsmoothing.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,246 @@ +from trafficintelligence import storage, moving, utils + +from math import atan2, degrees, sin, cos, pi +from numpy import median + +import matplotlib.pyplot as plt + +def findNearest(feat, featureSet,t,reverse=True): + dist={} + for f in featureSet: + if reverse: + dist[f]= moving.Point.distanceNorm2(feat.getPositionAtInstant(t+1),f.getPositionAtInstant(t)) + else: + dist[f]= moving.Point.distanceNorm2(feat.getPositionAtInstant(t-1),f.getPositionAtInstant(t)) + return min(dist, key=dist.get) # = utils.argmaxDict(dist) + +def getFeatures(obj, featureID): + currentFeature = obj.getFeature(featureID) + first = currentFeature.getFirstInstant() + last = currentFeature.getLastInstant() + featureList=[[currentFeature,first,last,moving.Point(0,0)]] + # find the features to fill in the beginning of the object existence + while first != obj.getFirstInstant(): + delta=featureList[-1][3] + featureSet = [f for f in obj.getFeatures() if f.existsAtInstant(first-1)] + feat = findNearest(currentFeature,featureSet,first-1,reverse=True) + if feat.existsAtInstant(first): + featureList.append([feat,feat.getFirstInstant(),first-1,(currentFeature.getPositionAtInstant(first)-feat.getPositionAtInstant(first))+delta]) + else: + featureList.append([feat,feat.getFirstInstant(),first-1,(currentFeature.getPositionAtInstant(first)-feat.getPositionAtInstant(first-1))+delta]) + currentFeature = feat + first= feat.getFirstInstant() + # find the features to fill in the end of the object existence + delta=moving.Point(0,0) + currentFeature = obj.getFeature(featureID) # need to reinitialize + while last!= obj.getLastInstant(): + featureSet = [f for f in obj.getFeatures() if f.existsAtInstant(last+1)] + feat = findNearest(currentFeature,featureSet,last+1,reverse=False) + if feat.existsAtInstant(last): + featureList.append([feat,last+1,feat.getLastInstant(),(currentFeature.getPositionAtInstant(last)-feat.getPositionAtInstant(last))+delta]) + else: + featureList.append([feat,last+1,feat.getLastInstant(),(currentFeature.getPositionAtInstant(last)-feat.getPositionAtInstant(last+1))+delta]) + currentFeature = feat + last= feat.getLastInstant() + delta=featureList[-1][3] + return featureList + +def buildFeature(obj, featureID, num = 1): + featureList= getFeatures(obj, featureID) + tmp={} + delta={} + for i in featureList: + for t in range(i[1],i[2]+1): + tmp[t]=[i[0],i[3]] + newTraj = moving.Trajectory() + + for instant in obj.getTimeInterval(): + newTraj.addPosition(tmp[instant][0].getPositionAtInstant(instant)+tmp[instant][1]) + newFeature= moving.MovingObject(num,timeInterval=obj.getTimeInterval(),positions=newTraj) + return newFeature + +def getBearing(p1,p2,p3): + angle = degrees(atan2(p3.y -p1.y, p3.x -p1.x)) + bearing1 = (90 - angle) % 360 + angle2 = degrees(atan2(p2.y -p1.y, p2.x -p1.x)) + bearing2 = (90 - angle2) % 360 + dist= moving.Point.distanceNorm2(p1, p2) + return [dist,bearing1,bearing2,bearing2-bearing1] + +#Quantitative analysis "CSJ" functions +def computeVelocities(obj, smoothing=True, halfWidth=3): #compute velocities from positions + velocities={} + for i in list(obj.timeInterval)[:-1]: + p1= obj.getPositionAtInstant(i) + p2= obj.getPositionAtInstant(i+1) + velocities[i]=p2-p1 + velocities[obj.getLastInstant()]= velocities[obj.getLastInstant()-1] # duplicate last point + if smoothing: + velX= [velocities[y].aslist()[0] for y in sorted(velocities.keys())] + velY= [velocities[y].aslist()[1] for y in sorted(velocities.keys())] + v1= list(utils.filterMovingWindow(velX, halfWidth)) + v2= list(utils.filterMovingWindow(velY, halfWidth)) + smoothedVelocity={} + for t,i in enumerate(sorted(velocities.keys())): + smoothedVelocity[i]=moving.Point(v1[t], v2[t]) + velocities=smoothedVelocity + return velocities + +def computeAcceleration(obj,fromPosition=True): + acceleration={} + if fromPosition: + velocities=computeVelocities(obj,False,1) + for i in sorted(velocities.keys()): + if i != sorted(velocities.keys())[-1]: + acceleration[i]= velocities[i+1]-velocities[i] + else: + for i in list(obj.timeInterval)[:-1]: + v1= obj.getVelocityAtInstant(i) + v2= obj.getVelocityAtInstant(i+1) + acceleration[i]= v2-v1 + return acceleration + +def computeJerk(obj,fromPosition=True): + jerk={} + acceleration=computeAcceleration(obj,fromPosition=fromPosition) + for i in sorted(acceleration.keys()): + if i != sorted(acceleration.keys())[-1]: + jerk[i] = (acceleration[i+1]-acceleration[i]).norm2() + return jerk + +def sumSquaredJerk(obj,fromPosition=True): + jerk= computeJerk(obj,fromPosition=fromPosition) + t=0 + for i in sorted(jerk.keys()): + t+= jerk[i]* jerk[i] + return t + +def smoothObjectTrajectory(obj, featureID,newNum,smoothing=False,halfWidth=3,create=False): + results=[] + bearing={} + if create: + feature = buildFeature(obj, featureID , num=1) # why num=1 + else: + feature = obj.getFeature(featureID) + for t in feature.getTimeInterval(): + p1= feature.getPositionAtInstant(t) + p2= obj.getPositionAtInstant(t) + if t!=feature.getLastInstant(): + p3= feature.getPositionAtInstant(t+1) + else: + p1= feature.getPositionAtInstant(t-1) + p3= feature.getPositionAtInstant(t) + bearing[t]= getBearing(p1,p2,p3)[1] + results.append(getBearing(p1,p2,p3)) + + medianResults=median(results,0) + dist= medianResults[0] + angle= medianResults[3] + + for i in sorted(bearing.keys()): + bearing[i]= bearing[i]+angle + + if smoothing: + bearingInput=[] + for i in sorted(bearing.keys()): + bearingInput.append(bearing[i]) + import utils + bearingOut=utils.filterMovingWindow(bearingInput, halfWidth) + for t,i in enumerate(sorted(bearing.keys())): + bearing[i]=bearingOut[t] + + #solve a smoothing problem in case of big drop in computing bearing (0,360) + for t,i in enumerate(sorted(bearing.keys())): + if i!= max(bearing.keys()) and abs(bearingInput[t] - bearingInput[t+1])>=340: + for x in range(max(i-halfWidth,min(bearing.keys())),min(i+halfWidth,max(bearing.keys()))+1): + bearing[x]=bearingInput[t-i+x] + + translated = moving.Trajectory() + for t in feature.getTimeInterval(): + p1= feature.getPositionAtInstant(t) + p1.x = p1.x + dist*sin(bearing[t]*pi/180) + p1.y = p1.y + dist*cos(bearing[t]*pi/180) + translated.addPosition(p1) + + #modify first and last un-smoothed positions (half width) + if smoothing: + d1= translated[halfWidth]- feature.positions[halfWidth] + d2= translated[-halfWidth-1]- feature.positions[-halfWidth-1] + for i in range(halfWidth): + p1= feature.getPositionAt(i)+d1 + p2= feature.getPositionAt(-i-1)+d2 + translated.setPosition(i,p1) + translated.setPosition(-i-1,p2) + + newObj= moving.MovingObject(newNum,timeInterval=feature.getTimeInterval(),positions=translated) + return newObj + +def smoothObject(obj, newNum, minLengthParam = 0.7, smoothing = False, plotResults = True, halfWidth = 3, _computeVelocities = True, optimize = True, create = False): + '''Computes a smoother trajectory for the object + and optionnally smoother velocities + + The object should have its features in obj.features + TODO: check whether features are necessary''' + if not obj.hasFeatures(): + print('Object {} has an empty list of features: please load and add them using obj.setFeatures(features)'.format(obj.getNum())) + from sys import exit + exit() + + featureList=[i for i,f in enumerate(obj.getFeatures()) if f.length() >= minLengthParam*obj.length()] + if featureList==[]: + featureList.append(utils.argmaxDict({i:f.length() for i,f in enumerate(obj.getFeatures())})) + create = True + newObjects = [] + for featureID in featureList: # featureID should be the index in the list of obj.features + newObjects.append(smoothObjectTrajectory(obj, featureID, newNum, smoothing = smoothing, halfWidth = halfWidth, create = create)) + + newTranslated = moving.Trajectory() + newInterval = [] + for t in obj.getTimeInterval(): + xCoord=[] + yCoord=[] + for i in newObjects: + if i.existsAtInstant(t): + p1= i.getPositionAtInstant(t) + xCoord.append(p1.x) + yCoord.append(p1.y) + if xCoord != []: + tmp= moving.Point(median(xCoord), median(yCoord)) + newInterval.append(t) + newTranslated.addPosition(tmp) + + newObj= moving.MovingObject(newNum, timeInterval = moving.TimeInterval(min(newInterval),max(newInterval)),positions=newTranslated) + + if _computeVelocities: + tmpTraj = moving.Trajectory() + velocities= computeVelocities(newObj,True,5) + for i in sorted(velocities.keys()): + tmpTraj.addPosition(velocities[i]) + newObj.velocities=tmpTraj + else: + newObj.velocities=obj.velocities + + if optimize: + csj1= sumSquaredJerk(obj,fromPosition=True) + csj2= sumSquaredJerk(newObj,fromPosition=True) + if csj1<csj2: + newObj=obj + newObj.velocities=obj.velocities + if _computeVelocities and csj1>=csj2: + csj3= sumSquaredJerk(obj,fromPosition=False) + csj4= sumSquaredJerk(newObj,fromPosition=False) + if csj4<=csj3: + newObj.velocities= obj.velocities + + newObj.featureNumbers=obj.featureNumbers + newObj.features=obj.getFeatures() + newObj.userType=obj.userType + + if plotResults: + plt.figure() + plt.title('objects_id = {}'.format(obj.num)) + for i in featureList: + obj.getFeature(i).plot('cx-') + obj.plot('rx-') + newObj.plot('gx-') + return newObj
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/pavement.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,313 @@ +#! /usr/bin/env python +'''Tools for processing and analyzing pavement marking data''' + +from trafficintelligence import utils + +import numpy as np + + +paintTypes = {0: "Non-existant", + 1: "Eau", + 2: "Epoxy", + 3: "Alkyde", + 4: "Autre"} + +durabilities = {1: 98, #96 to 100 + 2: 85, #75 to 96 + 3: 62, #50 to 75 + 4: 32, #15 to 50 + 5: 7 #0 to 15 + } + +roadFunctionalClasses = {40: "Collectrice", + 20: "Nationale", + 30: "Regionale", + 10: "Autoroute", + 60: "Acces ressources", + 51: "Local 1", + 52: "Local 2", + 53: "Local 3", + 15: "Aut (PRN)", + 25: "Nat (PRN)", + 70: "Acces isolees", + 99: "Autres"} + +def caracteristiques(rtss, maintenanceLevel, rtssWeatherStation, fmr, paintType): + '''Computes characteristic data for the RTSS (class rtss) + maintenanceLevel = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\BDD_access\\analyse_donnees_deneigement\\exigence_circuits.txt', delimiter = ';') + rtssWeatherStation = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\stations_environnement_canada\\rtssWeatherStation\juste_pour_rtss_avec_donnees_entretien_hiv\\rtssWeatherStation_EC3.txt', delimiter = ',') + fmr = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\BDD_access\\analyse_donnees_deneigement\\fmr.txt', delimiter = ';') + paintType = pylab.csv2rec('C:\\Users\Alexandre\Desktop\Projet_maitrise_recherche\BDD_access\\analyse_donnees_deneigement\\type_peinture.txt', delimiter = ';') + ''' + # determination exigence deneigement + if rtss.id in maintenanceLevel['rtss_debut']: + for i in range(len(maintenanceLevel)): + if maintenanceLevel['rtss_debut'][i] == rtss.id: + exigence = maintenanceLevel['exigence'][i] + else: + exigence = '' + + # determination x/y + if rtss.id in rtssWeatherStation['rtss']: + for i in range(len(rtssWeatherStation)): + if rtssWeatherStation['rtss'][i] == rtss.id: + x_moy = rtssWeatherStation['x_moy'][i] + y_moy = rtssWeatherStation['y_moy'][i] + else: + x_moy, y_moy = '','' + + # determination info fmr + age_revtm, classe_fonct, type_revtm, milieu, djma, pourc_camions, vit_max = [], [], [], [], [], [], [] + if rtss.id in fmr['rtss_debut']: + for i in range(len(fmr)): + if fmr['rtss_debut'][i] == rtss.id: + age_revtm.append(fmr['age_revtm'][i]) + classe_fonct.append(fmr['des_clasf_fonct'][i]) + type_revtm.append(fmr['des_type_revtm'][i]) + milieu.append(fmr['des_cod_mil'][i]) + djma.append(fmr['val_djma'][i]) + pourc_camions.append(fmr['val_pourc_camns'][i]) + vit_max.append(fmr['val_limt_vitss'][i]) + age_revtm = utils.mostCommon(age_revtm) + classe_fonct = utils.mostCommon(classe_fonct) + type_revtm = utils.mostCommon(type_revtm) + milieu = utils.mostCommon(milieu) + djma = utils.mostCommon(djma) + vit_max = utils.mostCommon(vit_max) + if vit_max < 0: + vit_max = '' + pourc_camions = utils.mostCommon(pourc_camions) + if pourc_camions == "" or pourc_camions < 0: + djma_camions = "" + else: + djma_camions = pourc_camions*djma/100 + else: + age_revtm, classe_fonct, type_revtm, milieu, djma, djma_camions, vit_max = '','','','','','','' + + # determination type peinture + peinture_rd, peinture_rg, peinture_cl = [], [], [] + peinture_lrd, peinture_lrg, peinture_lc = 0,0,0 + if rtss.id in paintType['rtss_debut_orig']: + for i in range(len(paintType)): + if paintType['rtss_debut_orig'][i] == rtss.id: + peinture_rd.append((paintType['peinture_rd'][i])) + peinture_rg.append((paintType['peinture_rg'][i])) + peinture_cl.append((paintType['peinture_cl'][i])) + peinture_lrd = utils.mostCommon(peinture_rd) + peinture_lrg = utils.mostCommon(peinture_rg) + peinture_lc = utils.mostCommon(peinture_cl) + else: + peinture_lrd, peinture_lrg, peinture_lc = '','','' + + return (exigence, x_moy, y_moy, age_revtm, classe_fonct, type_revtm, milieu, djma, djma_camions, vit_max, peinture_lrd, peinture_lrg, peinture_lc) + +def winterMaintenanceIndicators(data, startDate, endDate, circuitReference, snowThreshold): + '''Computes several winter maintenance indicators + data = entretien_hivernal = pylab.csv2rec('C:\\Users\Alexandre\Documents\Cours\Poly\Projet\mesures_entretien_hivernal\mesures_deneigement.txt', delimiter = ',')''' + import datetime + somme_eau, somme_neige, somme_abrasif, somme_sel, somme_lc, somme_lrg, somme_lrd, compteur_premiere_neige, compteur_somme_abrasif = 0,0,0,0,0,0,0,0,0 + + if circuitReference in data['ref_circuit']: + for i in range(len(data)): + if data['ref_circuit'][i] == circuitReference and (data['date'][i] + datetime.timedelta(days = 6)) <= endDate and (data['date'][i] + datetime.timedelta(days = 6)) > startDate: + compteur_premiere_neige += float(data['premiere_neige'][i]) + somme_neige += float(data['neige'][i]) + somme_eau += float(data['eau'][i]) + somme_abrasif += float(data['abrasif'][i]) + somme_sel += float(data['sel'][i]) + somme_lc += float(data['lc'][i]) + somme_lrg += float(data['lrg'][i]) + somme_lrd += float(data['lrd'][i]) + compteur_somme_abrasif += float(data['autre_abrasif_binaire'][i]) + if compteur_premiere_neige >= 1: + premiere_neige = 1 + else: + premiere_neige = 0 + if compteur_somme_abrasif >= 1: + autres_abrasifs = 1 + else: + autres_abrasifs = 0 + if somme_neige < snowThreshold: + neigeMTQ_sup_seuil = 0 + else: + neigeMTQ_sup_seuil = 1 + else: + somme_eau, somme_neige, somme_abrasif, somme_sel, somme_lc, somme_lrg, somme_lrd, premiere_neige, autres_abrasifs, neigeMTQ_sup_seuil = '','','','','','','','','','' + + return (somme_eau, somme_neige, neigeMTQ_sup_seuil, somme_abrasif, somme_sel, somme_lc, somme_lrg, somme_lrd, premiere_neige, autres_abrasifs) + +def weatherIndicators(data, startDate, endDate, snowThreshold, weatherDatatype, minProportionMeasures = 0.): + '''Computes the indicators from Environment Canada files + (loaded as a recarray using csv2rec in data), + between start and end dates (datetime.datetime objects) + + weatherDataType is to indicate Environnement Canada data ('ec') or else MTQ + minProportionMeasures is proportion of measures necessary to consider the indicators''' + from matplotlib.mlab import find + nbre_jours_T_negatif,nbre_jours_gel_degel,pluie_tot,neige_tot,ecart_type_T = 0,0,0,0,0 + compteur,nbre_jours_gel_consecutifs=0,0 + tmoys = [] + seuils_T = [20,15,10,5] + deltas_T = [0,0,0,0] + startIndex = find(data['date'] == startDate) + nDays = int((endDate - startDate).days)+1 + if len(startIndex) > 0 and startIndex+nDays <= len(data): + startIndex = startIndex[0] + for i in range(startIndex, startIndex+nDays): + if not np.isnan(data['tmax'][i]): + tmax = data['tmax'][i] + else: + tmax = None + if not np.isnan(data['tmin'][i]): + tmin = data['tmin'][i] + else: + tmin = None + if weatherDatatype == 'ec': + if data['pluie_tot'][i] is not None and not np.isnan(data['pluie_tot'][i]): + pluie_tot += data['pluie_tot'][i] + if data['neige_tot'][i] is not None and not np.isnan(data['neige_tot'][i]): + neige_tot += data['neige_tot'][i] + if tmax is not None: + if tmax < 0: + nbre_jours_T_negatif += 1 + if tmax is not None and tmin is not None: + if tmax > 0 and tmin < 0: + nbre_jours_gel_degel += 1 + for l in range(len(seuils_T)): + if tmax - tmin >=seuils_T[l]: + deltas_T[l] += 1 + if not np.isnan(data['tmoy'][i]): + tmoys.append(data['tmoy'][i]) + if tmax is not None: + if tmax < 0: + compteur += 1 + elif tmax >= 0 and compteur >= nbre_jours_gel_consecutifs: + nbre_jours_gel_consecutifs = compteur + compteur = 0 + else: + compteur = 0 + nbre_jours_gel_consecutifs = max(nbre_jours_gel_consecutifs,compteur) + if len(tmoys) > 0 and float(len(tmoys))/nDays >= minProportionMeasures: + if tmoys != []: + ecart_type_T = np.std(tmoys) + else: + ecart_type = None + if neige_tot < snowThreshold: + neigeEC_sup_seuil = 0 + else: + neigeEC_sup_seuil = 1 + return (nbre_jours_T_negatif,nbre_jours_gel_degel, deltas_T, nbre_jours_gel_consecutifs, pluie_tot, neige_tot, neigeEC_sup_seuil, ecart_type_T) + else: + return [None]*2+[[None]*len(seuils_T)]+[None]*5 + +def mtqWeatherIndicators(data, startDate, endDate,tmax,tmin,tmoy): + print("Deprecated, use weatherIndicators") + from matplotlib.mlab import find + nbre_jours_T_negatif,nbre_jours_gel_degel,ecart_type_T = 0,0,0 + compteur,nbre_jours_gel_consecutifs=0,0 + tmoys = [] + seuils_T = [20,15,10,5] + deltas_T = [0,0,0,0] + startIndex = find(data['date'] == startDate) + nDays = (endDate - startDate).days+1 + for i in range(startIndex, startIndex+nDays): + if tmax[i] < 0: + nbre_jours_T_negatif += 1 + if tmax[i] > 0 and tmin[i] < 0: + nbre_jours_gel_degel += 1 + for l in range(len(seuils_T)): + if tmax[i] - tmin[i] >=seuils_T[l]: + deltas_T[l] += 1 + tmoys.append(tmoy[i]) + if tmax[i] < 0: + compteur += 1 + elif tmax[i] >= 0 and compteur >= nbre_jours_gel_consecutifs: + nbre_jours_gel_consecutifs = compteur + compteur = 0 + else: + compteur = 0 + nbre_jours_gel_consecutifs = max(nbre_jours_gel_consecutifs,compteur) + if tmoys != []: + ecart_type_T = np.std(tmoys) + else: + ecart_type = None + + return (nbre_jours_T_negatif,nbre_jours_gel_degel, deltas_T, nbre_jours_gel_consecutifs, ecart_type_T) + +class RTSS(object): + '''class for data related to a RTSS: + - agregating pavement marking measurements + - RTSS characteristics from FMR: pavement type, age, AADT, truck AADT + - winter maintenance level from V155 + + If divided highway, the RTSS ends with G or D and are distinct: there is no ambiguity + - retroreflectivity types: there are CB, RJ and RB + If undivided, ending with C + - durability is fine: ETAT_MARQG_RG ETAT_MARQG_CL ETAT_MARQG_RD (+SG/SD, but recent) + - retroreflectivity: CJ is center line, RB and SB are left/right if DEBUT-FIN>0 or <0 + ''' + + def __init__(self, _id, name, data): + self.id = _id + self.name = name + self.data = data + +class MarkingTest(object): + '''class for a test site for a given product + + including the series of measurements over the years''' + + def __init__(self, _id, paintingDate, paintingType, color, data): + self.id = _id + self.paintingDate = paintingDate + self.paintingType = paintingType + self.color = color + self.data = data + self.nMeasures = len(data) + + def getSite(self): + return int(self.id[:2]) + + def getTestAttributes(self): + return [self.paintingType, self.color, self.paintingDate.year] + + def plot(self, measure, options = 'o', dayRatio = 1., **kwargs): + from matplotlib.pyplot import plot + plot(self.data['jours']/float(dayRatio), + self.data[measure], options, **kwargs) + + def getMarkingMeasures(self, dataLabel): + nonZeroIndices = ~np.isnan(self.data[dataLabel]) + return self.data[nonZeroIndices]['jours'], self.data[nonZeroIndices][dataLabel] + + def plotMarkingMeasures(self, measure, options = 'o', dayRatio = 1., **kwargs): + for i in range(1,7): + self.plot('{}_{}'.format(measure, i), options, dayRatio, **kwargs) + + def computeMarkingMeasureVariations(self, dataLabel, lanePositions, weatherData, snowThreshold, weatherDataType = 'ec', minProportionMeasures = 0.): + '''Computes for each successive measurement + lanePositions = None + measure variation, initial measure, time duration, weather indicators + + TODO if measurements per lane, add a variable for lane position (position1 to 6) + lanePositions = list of integers (range(1,7)) + measure variation, initial measure, time duration, lane position1, weather indicators + measure variation, initial measure, time duration, lane position2, weather indicators + ...''' + variationData = [] + if lanePositions is None: + nonZeroIndices = ~np.isnan(self.data[dataLabel]) + days = self.data[nonZeroIndices]['jours'] + dates = self.data[nonZeroIndices]['date_mesure'] + measures = self.data[nonZeroIndices][dataLabel] + for i in range(1, len(dates)): + nDaysTNegative, nDaysThawFreeze, deltaTemp, nConsecutiveFrozenDays, totalRain, totalSnow, snowAboveThreshold, stdevTemp = weatherIndicators(weatherData, dates[i-1], dates[i], snowThreshold, weatherDataType, minProportionMeasures) + if dates[i-1].year+1 == dates[i].year: + winter = 1 + if days[i-1]<365: + firstWinter = 1 + else: + winter = 0 + firstWinter = 0 + variationData.append([measures[i-1]-measures[i], measures[i-1], days[i]-days[i-1], days[i-1], winter, firstWinter, nDaysTNegative, nDaysThawFreeze] + deltaTemp + [nConsecutiveFrozenDays, totalRain, totalSnow, snowAboveThreshold, stdevTemp]) + return variationData
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/poly-utils.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,125 @@ +#! /usr/bin/env python +'''Various utilities to load data saved by the POLY new output(s)''' + +from moving import TimeInterval +from indicators import SeverityIndicator + +import sys, utils +import numpy as np + + +def loadNewInteractions(videoFilename,interactionType,dirname, extension, indicatorsNames, roaduserNum1,roaduserNum2, selectedIndicators=[]): + '''Loads interactions from the POLY traffic event format''' + from events import Interaction + filename= dirname + videoFilename + extension + #filename= dirname + interactionType+ '-' + videoFilename + extension # case of min distance todo: change the saving format to be matched with all outputs + file = utils.openCheck(filename) + if (not file): + return [] + #interactions = [] + interactionNum = 0 + data= np.loadtxt(filename) + indicatorFrameNums= data[:,0] + inter = Interaction(interactionNum, TimeInterval(indicatorFrameNums[0],indicatorFrameNums[-1]), roaduserNum1, roaduserNum2) + inter.addVideoFilename(videoFilename) + inter.addInteractionType(interactionType) + for key in indicatorsNames: + values= {} + for i,t in enumerate(indicatorFrameNums): + values[t] = data[i,key] + inter.addIndicator(SeverityIndicator(indicatorsNames[key], values)) + if selectedIndicators !=[]: + values= {} + for i,t in enumerate(indicatorFrameNums): + values[t] = [data[i,index] for index in selectedIndicators] + inter.addIndicator(SeverityIndicator('selectedIndicators', values)) + + #interactions.append(inter) + file.close() + #return interactions + return inter + +# Plotting results + +frameRate = 15. + +# To run in directory that contains the directories that contain the results (Miss-xx and Incident-xx) +#dirname = '/home/nicolas/Research/Data/kentucky-db/' + +interactingRoadUsers = {'Miss/0404052336': [(0,3)] # 0,2 and 1 vs 3 + #, + #'Incident/0306022035': [(1,3)] + #, + #'Miss/0208030956': [(4,5),(5,7)] + } + + +def getIndicatorName(filename, withUnit = False): + if withUnit: + unit = ' (s)' + else: + unit = '' + if 'collision-point' in filename: + return 'TTC'+unit + elif 'crossing' in filename: + return 'pPET'+unit + elif 'probability' in filename: + return 'P(UEA)' + +def getMethodName(fileprefix): + if fileprefix == 'constant-velocity': + return 'Con. Vel.' + elif fileprefix == 'normal-adaptation': + return 'Norm. Ad.' + elif fileprefix == 'point-set': + return 'Pos. Set' + elif fileprefix == 'evasive-action': + return 'Ev. Act.' + elif fileprefix == 'point-set-evasive-action': + return 'Pos. Set' + +indicator2TimeIdx = {'TTC':2,'pPET':2, 'P(UEA)':3} + +def getDataAtInstant(data, i): + return data[data[:,2] == i] + +def getPointsAtInstant(data, i): + return getDataAtInstant(i)[3:5] + +def getIndicator(data, roadUserNumbers, indicatorName): + if data.ndim ==1: + data.shape = (1,data.shape[0]) + + # find the order for the roadUserNumbers + uniqueObj1 = np.unique(data[:,0]) + uniqueObj2 = np.unique(data[:,1]) + found = False + if roadUserNumbers[0] in uniqueObj1 and roadUserNumbers[1] in uniqueObj2: + objNum1 = roadUserNumbers[0] + objNum2 = roadUserNumbers[1] + found = True + if roadUserNumbers[1] in uniqueObj1 and roadUserNumbers[0] in uniqueObj2: + objNum1 = roadUserNumbers[1] + objNum2 = roadUserNumbers[0] + found = True + + # get subset of data for road user numbers + if found: + roadUserData = data[np.logical_and(data[:,0] == objNum1, data[:,1] == objNum2),:] + if roadUserData.size > 0: + time = np.unique(roadUserData[:,indicator2TimeIdx[indicatorName]]) + values = {} + if indicatorName == 'P(UEA)': + tmp = roadUserData[:,4] + for k,v in zip(time, tmp): + values[k]=v + return SeverityIndicator(indicatorName, values, mostSevereIsMax = False, maxValue = 1.), roadUserData + else: + for i in range(time[0],time[-1]+1): + try: + tmp = getDataAtInstant(roadUserData, i) + values[i] = np.sum(tmp[:,5]*tmp[:,6])/np.sum(tmp[:,5])/frameRate + except IOError: + values[i] = np.inf + return SeverityIndicator(indicatorName, values, mostSevereIsMax = False), roadUserData + return None, None
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/prediction.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,598 @@ +#! /usr/bin/env python +'''Library for motion prediction methods''' + +import moving +from utils import LCSS + +import math, random +from copy import copy +import numpy as np +#from multiprocessing import Pool + + +class PredictedTrajectory(object): + '''Class for predicted trajectories with lazy evaluation + if the predicted position has not been already computed, compute it + + it should also have a probability''' + + def __init__(self): + self.probability = 0. + self.predictedPositions = {} + self.predictedSpeedOrientations = {} + #self.collisionPoints = {} + #self.crossingZones = {} + + def predictPosition(self, nTimeSteps): + if nTimeSteps > 0 and not nTimeSteps in self.predictedPositions: + self.predictPosition(nTimeSteps-1) + self.predictedPositions[nTimeSteps], self.predictedSpeedOrientations[nTimeSteps] = moving.predictPosition(self.predictedPositions[nTimeSteps-1], self.predictedSpeedOrientations[nTimeSteps-1], self.getControl(), self.maxSpeed) + return self.predictedPositions[nTimeSteps] + + def getPredictedTrajectory(self): + return moving.Trajectory.fromPointList(list(self.predictedPositions.values())) + + def getPredictedSpeeds(self): + return [so.norm for so in self.predictedSpeedOrientations.values()] + + def plot(self, options = '', withOrigin = False, timeStep = 1, **kwargs): + self.getPredictedTrajectory().plot(options, withOrigin, timeStep, **kwargs) + +class PredictedTrajectoryConstant(PredictedTrajectory): + '''Predicted trajectory at constant speed or acceleration + TODO generalize by passing a series of velocities/accelerations''' + + def __init__(self, initialPosition, initialVelocity, control = moving.NormAngle(0,0), probability = 1., maxSpeed = None): + self.control = control + self.maxSpeed = maxSpeed + self.probability = probability + self.predictedPositions = {0: initialPosition} + self.predictedSpeedOrientations = {0: moving.NormAngle.fromPoint(initialVelocity)} + + def getControl(self): + return self.control + +class PredictedTrajectoryPrototype(PredictedTrajectory): + '''Predicted trajectory that follows a prototype trajectory + The prototype is in the format of a moving.Trajectory: it could be + 1. an observed trajectory (extracted from video) + 2. a generic polyline (eg the road centerline) that a vehicle is supposed to follow + + Prediction can be done + 1. at constant speed (the instantaneous user speed) + 2. following the trajectory path, at the speed of the user + (applying a constant ratio equal + to the ratio of the user instantaneous speed and the trajectory closest speed)''' + + def __init__(self, initialPosition, initialVelocity, prototype, constantSpeed = False, nFramesIgnore = 3, probability = 1.): + ''' prototype is a MovingObject + + Prediction at constant speed will not work for unrealistic trajectories + that do not follow a slowly changing velocity (eg moving object trajectories, + but is good for realistic motion (eg features)''' + self.prototype = prototype + self.constantSpeed = constantSpeed + self.nFramesIgnore = nFramesIgnore + self.probability = probability + self.predictedPositions = {0: initialPosition} + self.closestPointIdx = prototype.getPositions().getClosestPoint(initialPosition) + self.deltaPosition = initialPosition-prototype.getPositionAt(self.closestPointIdx) #should be computed in relative coordinates to position + self.theta = prototype.getVelocityAt(self.closestPointIdx).angle() + self.initialSpeed = initialVelocity.norm2() + if not constantSpeed: + self.ratio = self.initialSpeed/prototype.getVelocityAt(self.closestPointIdx).norm2() + + def predictPosition(self, nTimeSteps): + if nTimeSteps > 0 and not nTimeSteps in self.predictedPositions: + deltaPosition = copy(self.deltaPosition) + if self.constantSpeed: + traj = self.prototype.getPositions() + trajLength = traj.length() + traveledDistance = nTimeSteps*self.initialSpeed + traj.getCumulativeDistance(self.closestPointIdx) + i = self.closestPointIdx + while i < trajLength and traj.getCumulativeDistance(i) < traveledDistance: + i += 1 + if i == trajLength: + v = self.prototype.getVelocityAt(-1-self.nFramesIgnore) + self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[i-1]+v*((traveledDistance-traj.getCumulativeDistance(i-1))/v.norm2()) + else: + v = self.prototype.getVelocityAt(min(i-1, int(self.prototype.length())-1-self.nFramesIgnore)) + self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[i-1]+(traj[i]-traj[i-1])*((traveledDistance-traj.getCumulativeDistance(i-1))/traj.getDistance(i-1)) + else: + traj = self.prototype.getPositions() + trajLength = traj.length() + nSteps = self.ratio*nTimeSteps+self.closestPointIdx + i = int(np.floor(nSteps)) + if nSteps < trajLength-1: + v = self.prototype.getVelocityAt(min(i, int(self.prototype.length())-1-self.nFramesIgnore)) + self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[i]+(traj[i+1]-traj[i])*(nSteps-i) + else: + v = self.prototype.getVelocityAt(-1-self.nFramesIgnore) + self.predictedPositions[nTimeSteps] = deltaPosition.rotate(v.angle()-self.theta)+traj[-1]+v*(nSteps-trajLength+1) + return self.predictedPositions[nTimeSteps] + +class PredictedTrajectoryRandomControl(PredictedTrajectory): + '''Random vehicle control: suitable for normal adaptation''' + def __init__(self, initialPosition, initialVelocity, accelerationDistribution, steeringDistribution, probability = 1., maxSpeed = None): + '''Constructor + accelerationDistribution and steeringDistribution are distributions + that return random numbers drawn from them''' + self.accelerationDistribution = accelerationDistribution + self.steeringDistribution = steeringDistribution + self.maxSpeed = maxSpeed + self.probability = probability + self.predictedPositions = {0: initialPosition} + self.predictedSpeedOrientations = {0: moving.NormAngle.fromPoint(initialVelocity)} + + def getControl(self): + return moving.NormAngle(self.accelerationDistribution(),self.steeringDistribution()) + +class SafetyPoint(moving.Point): + '''Can represent a collision point or crossing zone + with respective safety indicator, TTC or pPET''' + def __init__(self, p, probability = 1., indicator = -1): + self.x = p.x + self.y = p.y + self.probability = probability + self.indicator = indicator + + def __str__(self): + return '{0} {1} {2} {3}'.format(self.x, self.y, self.probability, self.indicator) + + @staticmethod + def save(out, points, predictionInstant, objNum1, objNum2): + for p in points: + out.write('{0} {1} {2} {3}\n'.format(objNum1, objNum2, predictionInstant, p)) + + @staticmethod + def computeExpectedIndicator(points): + return np.sum([p.indicator*p.probability for p in points])/sum([p.probability for p in points]) + +def computeCollisionTime(predictedTrajectory1, predictedTrajectory2, collisionDistanceThreshold, timeHorizon): + '''Computes the first instant + at which two predicted trajectories are within some distance threshold + Computes all the times including timeHorizon + + User has to check the first variable collision to know about a collision''' + t = 1 + p1 = predictedTrajectory1.predictPosition(t) + p2 = predictedTrajectory2.predictPosition(t) + collision = (p1-p2).norm2() <= collisionDistanceThreshold + while t < timeHorizon and not collision: + t += 1 + p1 = predictedTrajectory1.predictPosition(t) + p2 = predictedTrajectory2.predictPosition(t) + collision = (p1-p2).norm2() <= collisionDistanceThreshold + return collision, t, p1, p2 + +def savePredictedTrajectoriesFigure(currentInstant, obj1, obj2, predictedTrajectories1, predictedTrajectories2, timeHorizon, printFigure = True): + from matplotlib.pyplot import figure, axis, title, clf, savefig + if printFigure: + clf() + else: + figure() + for et in predictedTrajectories1: + for t in range(int(np.round(timeHorizon))): + et.predictPosition(t) + et.plot('rx') + for et in predictedTrajectories2: + for t in range(int(np.round(timeHorizon))): + et.predictPosition(t) + et.plot('bx') + obj1.plot('r', withOrigin = True) + obj2.plot('b', withOrigin = True) + title('instant {0}'.format(currentInstant)) + axis('equal') + if printFigure: + savefig('predicted-trajectories-t-{0}.png'.format(currentInstant)) + +def calculateProbability(nMatching,similarity,objects): + sumFrequencies=sum([nMatching[p] for p in similarity]) + prototypeProbability={} + for i in similarity: + prototypeProbability[i]= similarity[i] * float(nMatching[i])/sumFrequencies + sumProbabilities= sum([prototypeProbability[p] for p in prototypeProbability]) + probabilities={} + for i in prototypeProbability: + probabilities[objects[i]]= float(prototypeProbability[i])/sumProbabilities + return probabilities + +def findPrototypes(prototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity=0.1,mostMatched=None,spatialThreshold=1.0, delta=180): + ''' behaviour prediction first step''' + if route[0] not in noiseEntryNums: + prototypesRoutes= [ x for x in sorted(prototypes.keys()) if route[0]==x[0]] + elif route[1] not in noiseExitNums: + prototypesRoutes=[ x for x in sorted(prototypes.keys()) if route[1]==x[1]] + else: + prototypesRoutes=[x for x in sorted(prototypes.keys())] + lcss = LCSS(similarityFunc=lambda x,y: (distanceForLCSS(x,y) <= spatialThreshold),delta=delta) + similarity={} + for y in prototypesRoutes: + if y in prototypes: + prototypesIDs=prototypes[y] + for x in prototypesIDs: + s=lcss.computeNormalized(partialObjPositions, objects[x].positions) + if s >= minSimilarity: + similarity[x]=s + + if mostMatched==None: + probabilities= calculateProbability(nMatching,similarity,objects) + return probabilities + else: + mostMatchedValues=sorted(similarity.values(),reverse=True)[:mostMatched] + keys=[k for k in similarity if similarity[k] in mostMatchedValues] + newSimilarity={} + for i in keys: + newSimilarity[i]=similarity[i] + probabilities= calculateProbability(nMatching,newSimilarity,objects) + return probabilities + +def findPrototypesSpeed(prototypes,secondStepPrototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity=0.1,mostMatched=None,useDestination=True,spatialThreshold=1.0, delta=180): + if useDestination: + prototypesRoutes=[route] + else: + if route[0] not in noiseEntryNums: + prototypesRoutes= [ x for x in sorted(prototypes.keys()) if route[0]==x[0]] + elif route[1] not in noiseExitNums: + prototypesRoutes=[ x for x in sorted(prototypes.keys()) if route[1]==x[1]] + else: + prototypesRoutes=[x for x in sorted(prototypes.keys())] + lcss = LCSS(similarityFunc=lambda x,y: (distanceForLCSS(x,y) <= spatialThreshold),delta=delta) + similarity={} + for y in prototypesRoutes: + if y in prototypes: + prototypesIDs=prototypes[y] + for x in prototypesIDs: + s=lcss.computeNormalized(partialObjPositions, objects[x].positions) + if s >= minSimilarity: + similarity[x]=s + + newSimilarity={} + for i in similarity: + if i in secondStepPrototypes: + for j in secondStepPrototypes[i]: + newSimilarity[j]=similarity[i] + probabilities= calculateProbability(nMatching,newSimilarity,objects) + return probabilities + +def getPrototypeTrajectory(obj,route,currentInstant,prototypes,secondStepPrototypes,nMatching,objects,noiseEntryNums,noiseExitNums,minSimilarity=0.1,mostMatched=None,useDestination=True,useSpeedPrototype=True): + partialInterval=moving.Interval(obj.getFirstInstant(),currentInstant) + partialObjPositions= obj.getObjectInTimeInterval(partialInterval).positions + if useSpeedPrototype: + prototypeTrajectories=findPrototypesSpeed(prototypes,secondStepPrototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity,mostMatched,useDestination) + else: + prototypeTrajectories=findPrototypes(prototypes,nMatching,objects,route,partialObjPositions,noiseEntryNums,noiseExitNums,minSimilarity,mostMatched) + return prototypeTrajectories + + +class PredictionParameters(object): + def __init__(self, name, maxSpeed): + self.name = name + self.maxSpeed = maxSpeed + + def __str__(self): + return '{0} {1}'.format(self.name, self.maxSpeed) + + def generatePredictedTrajectories(self, obj, instant): + return None + + def computeCrossingsCollisionsAtInstant(self, currentInstant, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False): + '''returns the lists of collision points and crossing zones''' + predictedTrajectories1 = self.generatePredictedTrajectories(obj1, currentInstant) + predictedTrajectories2 = self.generatePredictedTrajectories(obj2, currentInstant) + + collisionPoints = [] + if computeCZ: + crossingZones = [] + else: + crossingZones = None + for et1 in predictedTrajectories1: + for et2 in predictedTrajectories2: + collision, t, p1, p2 = computeCollisionTime(et1, et2, collisionDistanceThreshold, timeHorizon) + if collision: + collisionPoints.append(SafetyPoint((p1+p2)*0.5, et1.probability*et2.probability, t)) + elif computeCZ: # check if there is a crossing zone + # TODO same computation as PET with metric + concatenate past trajectory with future trajectory + cz = None + t1 = 0 + while not cz and t1 < timeHorizon: # t1 <= timeHorizon-1 + t2 = 0 + while not cz and t2 < timeHorizon: + cz = moving.segmentIntersection(et1.predictPosition(t1), et1.predictPosition(t1+1), et2.predictPosition(t2), et2.predictPosition(t2+1)) + if cz is not None: + deltaV= (et1.predictPosition(t1)- et1.predictPosition(t1+1) - et2.predictPosition(t2)+ et2.predictPosition(t2+1)).norm2() + crossingZones.append(SafetyPoint(cz, et1.probability*et2.probability, abs(t1-t2)-(float(collisionDistanceThreshold)/deltaV))) + t2 += 1 + t1 += 1 + + if debug: + savePredictedTrajectoriesFigure(currentInstant, obj1, obj2, predictedTrajectories1, predictedTrajectories2, timeHorizon) + + return collisionPoints, crossingZones + + def computeCrossingsCollisions(self, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, timeInterval = None):#, nProcesses = 1): + '''Computes all crossing and collision points at each common instant for two road users. ''' + collisionPoints = {} + if computeCZ: + crossingZones = {} + else: + crossingZones = None + if timeInterval is not None: + commonTimeInterval = timeInterval + else: + commonTimeInterval = obj1.commonTimeInterval(obj2) + #if nProcesses == 1: + for i in list(commonTimeInterval)[:-1]: # do not look at the 1 last position/velocities, often with errors + cp, cz = self.computeCrossingsCollisionsAtInstant(i, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ, debug) + if len(cp) != 0: + collisionPoints[i] = cp + if computeCZ and len(cz) != 0: + crossingZones[i] = cz + return collisionPoints, crossingZones + + def computeCollisionProbability(self, obj1, obj2, collisionDistanceThreshold, timeHorizon, debug = False, timeInterval = None): + '''Computes only collision probabilities + Returns for each instant the collision probability and number of samples drawn''' + collisionProbabilities = {} + if timeInterval is not None: + commonTimeInterval = timeInterval + else: + commonTimeInterval = obj1.commonTimeInterval(obj2) + for i in list(commonTimeInterval)[:-1]: + nCollisions = 0 + predictedTrajectories1 = self.generatePredictedTrajectories(obj1, i) + predictedTrajectories2 = self.generatePredictedTrajectories(obj2, i) + for et1 in predictedTrajectories1: + for et2 in predictedTrajectories2: + collision, t, p1, p2 = computeCollisionTime(et1, et2, collisionDistanceThreshold, timeHorizon) + if collision: + nCollisions += 1 + # take into account probabilities ?? + nSamples = float(len(predictedTrajectories1)*len(predictedTrajectories2)) + collisionProbabilities[i] = [nSamples, float(nCollisions)/nSamples] + + if debug: + savePredictedTrajectoriesFigure(i, obj1, obj2, predictedTrajectories1, predictedTrajectories2, timeHorizon) + + return collisionProbabilities + +class ConstantPredictionParameters(PredictionParameters): + def __init__(self, maxSpeed): + PredictionParameters.__init__(self, 'constant velocity', maxSpeed) + + def generatePredictedTrajectories(self, obj, instant): + return [PredictedTrajectoryConstant(obj.getPositionAtInstant(instant), obj.getVelocityAtInstant(instant), maxSpeed = self.maxSpeed)] + +class NormalAdaptationPredictionParameters(PredictionParameters): + def __init__(self, maxSpeed, nPredictedTrajectories, accelerationDistribution, steeringDistribution, useFeatures = False): + '''An example of acceleration and steering distributions is + lambda: random.triangular(-self.maxAcceleration, self.maxAcceleration, 0.) + ''' + if useFeatures: + name = 'point set normal adaptation' + else: + name = 'normal adaptation' + PredictionParameters.__init__(self, name, maxSpeed) + self.nPredictedTrajectories = nPredictedTrajectories + self.useFeatures = useFeatures + self.accelerationDistribution = accelerationDistribution + self.steeringDistribution = steeringDistribution + + def __str__(self): + return PredictionParameters.__str__(self)+' {0} {1} {2}'.format(self.nPredictedTrajectories, + self.maxAcceleration, + self.maxSteering) + + def generatePredictedTrajectories(self, obj, instant): + predictedTrajectories = [] + if self.useFeatures and obj.hasFeatures(): + features = [f for f in obj.getFeatures() if f.existsAtInstant(instant)] + positions = [f.getPositionAtInstant(instant) for f in features] + velocities = [f.getVelocityAtInstant(instant) for f in features] + else: + positions = [obj.getPositionAtInstant(instant)] + velocities = [obj.getVelocityAtInstant(instant)] + probability = 1./float(len(positions)*self.nPredictedTrajectories) + for i in range(self.nPredictedTrajectories): + for initialPosition,initialVelocity in zip(positions, velocities): + predictedTrajectories.append(PredictedTrajectoryRandomControl(initialPosition, + initialVelocity, + self.accelerationDistribution, + self.steeringDistribution, + probability, + maxSpeed = self.maxSpeed)) + return predictedTrajectories + +class PointSetPredictionParameters(PredictionParameters): + def __init__(self, maxSpeed): + PredictionParameters.__init__(self, 'point set', maxSpeed) + + def generatePredictedTrajectories(self, obj, instant): + predictedTrajectories = [] + if obj.hasFeatures(): + features = [f for f in obj.getFeatures() if f.existsAtInstant(instant)] + positions = [f.getPositionAtInstant(instant) for f in features] + velocities = [f.getVelocityAtInstant(instant) for f in features] + probability = 1./float(len(positions)) + for initialPosition,initialVelocity in zip(positions, velocities): + predictedTrajectories.append(PredictedTrajectoryConstant(initialPosition, initialVelocity, probability = probability, maxSpeed = self.maxSpeed)) + return predictedTrajectories + else: + print('Object {} has no features'.format(obj.getNum())) + return None + + +class EvasiveActionPredictionParameters(PredictionParameters): + def __init__(self, maxSpeed, nPredictedTrajectories, accelerationDistribution, steeringDistribution, useFeatures = False): + '''Suggested acceleration distribution may not be symmetric, eg + lambda: random.triangular(self.minAcceleration, self.maxAcceleration, 0.)''' + + if useFeatures: + name = 'point set evasive action' + else: + name = 'evasive action' + PredictionParameters.__init__(self, name, maxSpeed) + self.nPredictedTrajectories = nPredictedTrajectories + self.useFeatures = useFeatures + self.accelerationDistribution = accelerationDistribution + self.steeringDistribution = steeringDistribution + + def __str__(self): + return PredictionParameters.__str__(self)+' {0} {1} {2} {3}'.format(self.nPredictedTrajectories, self.minAcceleration, self.maxAcceleration, self.maxSteering) + + def generatePredictedTrajectories(self, obj, instant): + predictedTrajectories = [] + if self.useFeatures and obj.hasFeatures(): + features = [f for f in obj.getFeatures() if f.existsAtInstant(instant)] + positions = [f.getPositionAtInstant(instant) for f in features] + velocities = [f.getVelocityAtInstant(instant) for f in features] + else: + positions = [obj.getPositionAtInstant(instant)] + velocities = [obj.getVelocityAtInstant(instant)] + probability = 1./float(self.nPredictedTrajectories) + for i in range(self.nPredictedTrajectories): + for initialPosition,initialVelocity in zip(positions, velocities): + predictedTrajectories.append(PredictedTrajectoryConstant(initialPosition, + initialVelocity, + moving.NormAngle(self.accelerationDistribution(), + self.steeringDistribution()), + probability, + self.maxSpeed)) + return predictedTrajectories + + +class CVDirectPredictionParameters(PredictionParameters): + '''Prediction parameters of prediction at constant velocity + using direct computation of the intersecting point + Warning: the computed time to collision may be higher than timeHorizon (not used)''' + + def __init__(self): + PredictionParameters.__init__(self, 'constant velocity (direct computation)', None) + + def computeCrossingsCollisionsAtInstant(self, currentInstant, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, *kwargs): + collisionPoints = [] + if computeCZ: + crossingZones = [] + else: + crossingZones = None + + p1 = obj1.getPositionAtInstant(currentInstant) + p2 = obj2.getPositionAtInstant(currentInstant) + if (p1-p2).norm2() <= collisionDistanceThreshold: + collisionPoints = [SafetyPoint((p1+p2)*0.5, 1., 0.)] + else: + v1 = obj1.getVelocityAtInstant(currentInstant) + v2 = obj2.getVelocityAtInstant(currentInstant) + intersection = moving.intersection(p1, p1+v1, p2, p2+v2) + + if intersection is not None: + dp1 = intersection-p1 + dp2 = intersection-p2 + dot1 = moving.Point.dot(dp1, v1) + dot2 = moving.Point.dot(dp2, v2) + if (computeCZ and (dot1 > 0 or dot2 > 0)) or (dot1 > 0 and dot2 > 0): # if the road users are moving towards the intersection or if computing pPET + dist1 = dp1.norm2() + dist2 = dp2.norm2() + s1 = math.copysign(v1.norm2(), dot1) + s2 = math.copysign(v2.norm2(), dot2) + halfCollisionDistanceThreshold = collisionDistanceThreshold/2. + timeInterval1 = moving.TimeInterval(max(0,dist1-halfCollisionDistanceThreshold)/s1, (dist1+halfCollisionDistanceThreshold)/s1) + timeInterval2 = moving.TimeInterval(max(0,dist2-halfCollisionDistanceThreshold)/s2, (dist2+halfCollisionDistanceThreshold)/s2) + collisionTimeInterval = moving.TimeInterval.intersection(timeInterval1, timeInterval2) + + if collisionTimeInterval.empty(): + if computeCZ: + crossingZones = [SafetyPoint(intersection, 1., timeInterval1.distance(timeInterval2))] + else: + collisionPoints = [SafetyPoint(intersection, 1., collisionTimeInterval.center())] + + if debug and intersection is not None: + from matplotlib.pyplot import plot, figure, axis, title + figure() + plot([p1.x, intersection.x], [p1.y, intersection.y], 'r') + plot([p2.x, intersection.x], [p2.y, intersection.y], 'b') + intersection.plot() + obj1.plot('r') + obj2.plot('b') + title('instant {0}'.format(currentInstant)) + axis('equal') + + return collisionPoints, crossingZones + +class CVExactPredictionParameters(PredictionParameters): + '''Prediction parameters of prediction at constant velocity + using direct computation of the intersecting point (solving the equation) + Warning: the computed time to collision may be higher than timeHorizon (not used)''' + + def __init__(self): + PredictionParameters.__init__(self, 'constant velocity (direct exact computation)', None) + + def computeCrossingsCollisionsAtInstant(self, currentInstant, obj1, obj2, collisionDistanceThreshold, timeHorizon, computeCZ = False, debug = False, *kwargs): + 'TODO compute pPET' + collisionPoints = [] + crossingZones = [] + + p1 = obj1.getPositionAtInstant(currentInstant) + p2 = obj2.getPositionAtInstant(currentInstant) + v1 = obj1.getVelocityAtInstant(currentInstant) + v2 = obj2.getVelocityAtInstant(currentInstant) + #intersection = moving.intersection(p1, p1+v1, p2, p2+v2) + + if not moving.Point.parallel(v1, v2): + ttc = moving.Point.timeToCollision(p1, p2, v1, v2, collisionDistanceThreshold) + if ttc is not None: + collisionPoints = [SafetyPoint((p1+(v1*ttc)+p2+(v2*ttc))*0.5, 1., ttc)] + else: + pass # compute pPET + + return collisionPoints, crossingZones + +class PrototypePredictionParameters(PredictionParameters): + def __init__(self, prototypes, nPredictedTrajectories, pointSimilarityDistance, minSimilarity, lcssMetric = 'cityblock', minFeatureTime = 10, constantSpeed = False, useFeatures = True): + PredictionParameters.__init__(self, 'prototypes', None) + self.prototypes = prototypes + self.nPredictedTrajectories = nPredictedTrajectories + self.lcss = LCSS(metric = lcssMetric, epsilon = pointSimilarityDistance) + self.minSimilarity = minSimilarity + self.minFeatureTime = minFeatureTime + self.constantSpeed = constantSpeed + self.useFeatures = useFeatures + + def getLcss(self): + return self.lcss + + def addPredictedTrajectories(self, predictedTrajectories, obj, instant): + obj.computeTrajectorySimilarities(self.prototypes, self.lcss) + for proto, similarities in zip(self.prototypes, obj.prototypeSimilarities): + if similarities[instant-obj.getFirstInstant()] >= self.minSimilarity: + initialPosition = obj.getPositionAtInstant(instant) + initialVelocity = obj.getVelocityAtInstant(instant) + predictedTrajectories.append(PredictedTrajectoryPrototype(initialPosition, initialVelocity, proto.getMovingObject(), constantSpeed = self.constantSpeed, probability = proto.getNMatchings())) + + def generatePredictedTrajectories(self, obj, instant): + predictedTrajectories = [] + if instant-obj.getFirstInstant()+1 >= self.minFeatureTime: + if self.useFeatures and obj.hasFeatures(): + if not hasattr(obj, 'currentPredictionFeatures'): + obj.currentPredictionFeatures = [] + else: + obj.currentPredictionFeatures[:] = [f for f in obj.currentPredictionFeatures if f.existsAtInstant(instant)] + firstInstants = [(f,f.getFirstInstant()) for f in obj.getFeatures() if f.existsAtInstant(instant) and f not in obj.currentPredictionFeatures] + firstInstants.sort(key = lambda t: t[1]) + for f,t1 in firstInstants[:min(self.nPredictedTrajectories, len(firstInstants), self.nPredictedTrajectories-len(obj.currentPredictionFeatures))]: + obj.currentPredictionFeatures.append(f) + for f in obj.currentPredictionFeatures: + self.addPredictedTrajectories(predictedTrajectories, f, instant) + else: + self.addPredictedTrajectories(predictedTrajectories, obj, instant) + return predictedTrajectories + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/prediction.txt') + #suite = doctest.DocTestSuite() + unittest.TextTestRunner().run(suite) + #doctest.testmod() + #doctest.testfile("example.txt") +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/processing.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,20 @@ +#! /usr/bin/env python +'''Algorithms to process trajectories and moving objects''' + +import moving + +import numpy as np + + +def extractSpeeds(objects, zone): + speeds = {} + objectsNotInZone = [] + import matplotlib.nxutils as nx + for o in objects: + inPolygon = nx.points_inside_poly(o.getPositions().asArray().T, zone.T) + if inPolygon.any(): + objspeeds = [o.getVelocityAt(i).norm2() for i in range(int(o.length()-1)) if inPolygon[i]] + speeds[o.num] = np.mean(objspeeds) # km/h + else: + objectsNotInZone.append(o) + return speeds, objectsNotInZone
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/requirements.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,10 @@ +matplotlib +numpy + +The following libraries are optional. They are necessary for (sometimes very) specific classes/functions. + +Computer Vision (cvutils.py): opencv, scikit-image +Statistics and machine learning (ml.py): scipy, scikit-learn +Moving object geometry (currently commented) (moving.py) and plotting shapely polygons (utils.py): shapely +Tabular data loading/processing (storage.py): pandas +Optimization: munkres
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/run-tests.sh Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,10 @@ +#!/bin/sh +# for file in tests/*... basename +for f in ./*.py +do + python3 $f +done +for f in ./tests/*.py +do + python3 $f +done
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/sensors.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,65 @@ +#! /usr/bin/env python +'''Libraries for detecting, counting, etc., road users''' + +from numpy import mean, isnan + +from trafficintelligence import moving + +# TODO graphical user interface for creation + +class Sensor: + def detect(self, o): + print("Detect method not implemented") + return False + + def detectInstants(self, o): + print("DetectInstants method not implemented") + return [] + +class BoxSensor(Sensor): + def __init__(self, polygon, minNPointsInBox = 1): + self.polygon = polygon # check 2xN? + self.minNPointsInBox = minNPointsInBox + + def detectInstants(self, obj): + indices = obj.getPositions().getInstantsInPolygon(self.polygon) + firstInstant = obj.getFirstInstant() + return [i+firstInstant for i in indices] + + def detect(self, obj): + instants = self.detectInstants(obj) + return len(instants) >= self.minNPointsInBox + +def detectAnd(sensors, obj): + 'Returns True if all sensors detect the object' + result = True + for s in sensors: + result = result and s.detect(obj) + if not result: + return result + return result + +def detectOr(sensors, obj): + 'Returns True if any sensor detects the object' + result = False + for s in sensors: + result = result or s.detect(obj) + if result: + return result + return result + +def detectAndOrder(sensors, obj): + 'Returns True if all sensors are detected and in their order' + detectionInstants = [] + for s in sensors: + instants = s.detectInstants(obj) + if len(instants) == 0: + return False + else: + detectionInstants.append(mean(instants)) + result = True + for i in range(len(sensors)-1): + result = result and (detectionInstants[i] <= detectionInstants[i+1]) + if not result: + return result + return result
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/storage.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,1478 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +'''Various utilities to save and load data''' + +from trafficintelligence import utils, moving, events, indicators +from trafficintelligence.base import VideoFilenameAddable + +from pathlib import Path +import shutil +from copy import copy +import sqlite3, logging +from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg +from pandas import read_csv, merge + + +commentChar = '#' + +delimiterChar = '%'; + +ngsimUserTypes = {'twowheels':1, + 'car':2, + 'truck':3} + +tableNames = {'feature':'positions', + 'object': 'objects', + 'objectfeatures': 'positions'} + +######################### +# Sqlite +######################### + +# utils +def printDBError(error): + print('DB Error: {}'.format(error)) + +def dropTables(connection, tableNames): + 'deletes the table with names in tableNames' + try: + cursor = connection.cursor() + for tableName in tableNames: + cursor.execute('DROP TABLE IF EXISTS '+tableName) + except sqlite3.OperationalError as error: + printDBError(error) + +def deleteFromSqlite(filename, dataType): + 'Deletes (drops) some tables in the filename depending on type of data' + if Path(filename).is_file(): + with sqlite3.connect(filename) as connection: + if dataType == 'object': + dropTables(connection, ['objects', 'objects_features']) + elif dataType == 'interaction': + dropTables(connection, ['interactions', 'indicators']) + elif dataType == 'bb': + dropTables(connection, ['bounding_boxes']) + elif dataType == 'pois': + dropTables(connection, ['gaussians2d', 'objects_pois']) + elif dataType == 'prototype': + dropTables(connection, ['prototypes', 'objects_prototypes']) + else: + print('Unknown data type {} to delete from database'.format(dataType)) + else: + print('{} does not exist'.format(filename)) + +def tableExists(connection, tableName): + 'indicates if the table exists in the database' + try: + cursor = connection.cursor() + cursor.execute('SELECT COUNT(*) FROM SQLITE_MASTER WHERE type = \'table\' AND name = \''+tableName+'\'') + return cursor.fetchone()[0] == 1 + except sqlite3.OperationalError as error: + printDBError(error) + +def createTrajectoryTable(cursor, tableName): + if tableName.endswith('positions') or tableName.endswith('velocities'): + cursor.execute("CREATE TABLE IF NOT EXISTS "+tableName+" (trajectory_id INTEGER, frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") + else: + print('Unallowed name {} for trajectory table'.format(tableName)) + +def createObjectsTable(cursor): + cursor.execute("CREATE TABLE IF NOT EXISTS objects (object_id INTEGER, road_user_type INTEGER, n_objects INTEGER, PRIMARY KEY(object_id))") + +def createAssignmentTable(cursor, objectType1, objectType2, objectIdColumnName1, objectIdColumnName2): + cursor.execute("CREATE TABLE IF NOT EXISTS "+objectType1+"s_"+objectType2+"s ("+objectIdColumnName1+" INTEGER, "+objectIdColumnName2+" INTEGER, PRIMARY KEY("+objectIdColumnName1+","+objectIdColumnName2+"))") + +def createObjectsFeaturesTable(cursor): + cursor.execute("CREATE TABLE IF NOT EXISTS objects_features (object_id INTEGER, trajectory_id INTEGER, PRIMARY KEY(object_id, trajectory_id))") + + +def createCurvilinearTrajectoryTable(cursor): + cursor.execute("CREATE TABLE IF NOT EXISTS curvilinear_positions (trajectory_id INTEGER, frame_number INTEGER, s_coordinate REAL, y_coordinate REAL, lane TEXT, PRIMARY KEY(trajectory_id, frame_number))") + +def createFeatureCorrespondenceTable(cursor): + cursor.execute('CREATE TABLE IF NOT EXISTS feature_correspondences (trajectory_id INTEGER, source_dbname VARCHAR, db_trajectory_id INTEGER, PRIMARY KEY(trajectory_id))') + +def createInteractionTable(cursor): + cursor.execute('CREATE TABLE IF NOT EXISTS interactions (id INTEGER PRIMARY KEY, object_id1 INTEGER, object_id2 INTEGER, first_frame_number INTEGER, last_frame_number INTEGER, FOREIGN KEY(object_id1) REFERENCES objects(id), FOREIGN KEY(object_id2) REFERENCES objects(id))') + +def createIndicatorTable(cursor): + cursor.execute('CREATE TABLE IF NOT EXISTS indicators (interaction_id INTEGER, indicator_type INTEGER, frame_number INTEGER, value REAL, FOREIGN KEY(interaction_id) REFERENCES interactions(id), PRIMARY KEY(interaction_id, indicator_type, frame_number))') + +def insertTrajectoryQuery(tableName): + return "INSERT INTO "+tableName+" VALUES (?,?,?,?)" + +def insertObjectQuery(): + return "INSERT INTO objects VALUES (?,?,?)" + +def insertObjectFeatureQuery(): + return "INSERT INTO objects_features VALUES (?,?)" + +def createIndex(connection, tableName, columnName, unique = False): + '''Creates an index for the column in the table + I will make querying with a condition on this column faster''' + try: + cursor = connection.cursor() + s = "CREATE " + if unique: + s += "UNIQUE " + cursor.execute(s+"INDEX IF NOT EXISTS "+tableName+"_"+columnName+"_index ON "+tableName+"("+columnName+")") + connection.commit() + #connection.close() + except sqlite3.OperationalError as error: + printDBError(error) + +def getNumberRowsTable(connection, tableName, columnName = None): + '''Returns the number of rows for the table + If columnName is not None, means we want the number of distinct values for that column + (otherwise, we can just count(*))''' + try: + cursor = connection.cursor() + if columnName is None: + cursor.execute("SELECT COUNT(*) from "+tableName) + else: + cursor.execute("SELECT COUNT(DISTINCT "+columnName+") from "+tableName) + return cursor.fetchone()[0] + except sqlite3.OperationalError as error: + printDBError(error) + +def getMinMax(connection, tableName, columnName, minmax): + '''Returns max/min or both for given column in table + minmax must be string max, min or minmax''' + try: + cursor = connection.cursor() + if minmax == 'min' or minmax == 'max': + cursor.execute("SELECT "+minmax+"("+columnName+") from "+tableName) + elif minmax == 'minmax': + cursor.execute("SELECT MIN("+columnName+"), MAX("+columnName+") from "+tableName) + else: + print("Argument minmax unknown: {}".format(minmax)) + return cursor.fetchone()[0] + except sqlite3.OperationalError as error: + printDBError(error) + +def getObjectCriteria(objectNumbers): + if objectNumbers is None: + query = '' + elif type(objectNumbers) == int: + query = '<= {0}'.format(objectNumbers-1) + elif type(objectNumbers) == list: + query = 'in ('+', '.join([str(n) for n in objectNumbers])+')' + else: + print('objectNumbers {} are not a known type ({})'.format(objectNumbers, type(objectNumbers))) + query = '' + return query + +def loadTrajectoriesFromTable(connection, tableName, trajectoryType, objectNumbers = None, timeStep = None): + '''Loads trajectories (in the general sense) from the given table + can be positions or velocities + + returns a moving object''' + cursor = connection.cursor() + + try: + objectCriteria = getObjectCriteria(objectNumbers) + queryStatement = None + if trajectoryType == 'feature': + queryStatement = 'SELECT * from '+tableName + if objectNumbers is not None and timeStep is not None: + queryStatement += ' WHERE trajectory_id '+objectCriteria+' AND frame_number%{} = 0'.format(timeStep) + elif objectNumbers is not None: + queryStatement += ' WHERE trajectory_id '+objectCriteria + elif timeStep is not None: + queryStatement += ' WHERE frame_number%{} = 0'.format(timeStep) + queryStatement += ' ORDER BY trajectory_id, frame_number' + elif trajectoryType == 'object': + queryStatement = 'SELECT OF.object_id, P.frame_number, avg(P.x_coordinate), avg(P.y_coordinate) from '+tableName+' P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id' + if objectNumbers is not None: + queryStatement += ' AND OF.object_id '+objectCriteria + if timeStep is not None: + queryStatement += ' AND P.frame_number%{} = 0'.format(timeStep) + queryStatement += ' GROUP BY OF.object_id, P.frame_number ORDER BY OF.object_id, P.frame_number' + elif trajectoryType in ['bbtop', 'bbbottom']: + if trajectoryType == 'bbtop': + corner = 'top_left' + elif trajectoryType == 'bbbottom': + corner = 'bottom_right' + queryStatement = 'SELECT object_id, frame_number, x_'+corner+', y_'+corner+' FROM '+tableName + if objectNumbers is not None and timeStep is not None: + queryStatement += ' WHERE object_id '+objectCriteria+' AND frame_number%{} = 0'.format(timeStep) + elif objectNumbers is not None: + queryStatement += ' WHERE object_id '+objectCriteria + elif timeStep is not None: + queryStatement += ' WHERE frame_number%{} = 0'.format(timeStep) + queryStatement += ' ORDER BY object_id, frame_number' + else: + print('Unknown trajectory type {}'.format(trajectoryType)) + if queryStatement is not None: + cursor.execute(queryStatement) + logging.debug(queryStatement) + except sqlite3.OperationalError as error: + printDBError(error) + return [] + + objId = -1 + obj = None + objects = [] + for row in cursor: + if row[0] != objId: + objId = row[0] + if obj is not None and (obj.length() == obj.positions.length() or (timeStep is not None and npceil(obj.length()/timeStep) == obj.positions.length())): + objects.append(obj) + elif obj is not None: + print('Object {} is missing {} positions'.format(obj.getNum(), int(obj.length())-obj.positions.length())) + obj = moving.MovingObject(row[0], timeInterval = moving.TimeInterval(row[1], row[1]), positions = moving.Trajectory([[row[2]],[row[3]]])) + else: + obj.timeInterval.last = row[1] + obj.positions.addPositionXY(row[2],row[3]) + + if obj is not None and (obj.length() == obj.positions.length() or (timeStep is not None and npceil(obj.length()/timeStep) == obj.positions.length())): + objects.append(obj) + elif obj is not None: + print('Object {} is missing {} positions'.format(obj.getNum(), int(obj.length())-obj.positions.length())) + + return objects + +def loadUserTypesFromTable(cursor, objectNumbers): + objectCriteria = getObjectCriteria(objectNumbers) + queryStatement = 'SELECT object_id, road_user_type FROM objects' + if objectNumbers is not None: + queryStatement += ' WHERE object_id '+objectCriteria + cursor.execute(queryStatement) + userTypes = {} + for row in cursor: + userTypes[row[0]] = row[1] + return userTypes + +def loadTrajectoriesFromSqlite(filename, trajectoryType, objectNumbers = None, withFeatures = False, timeStep = None, tablePrefix = None): + '''Loads the trajectories (in the general sense, + either features, objects (feature groups) or bounding box series) + The number loaded is either the first objectNumbers objects, + or the indices in objectNumbers from the database''' + objects = [] + with sqlite3.connect(filename) as connection: + if tablePrefix is None: + prefix = '' + else: + prefix = tablePrefix + '_' + objects = loadTrajectoriesFromTable(connection, prefix+'positions', trajectoryType, objectNumbers, timeStep) + objectVelocities = loadTrajectoriesFromTable(connection, prefix+'velocities', trajectoryType, objectNumbers, timeStep) + + if len(objectVelocities) > 0: + for o,v in zip(objects, objectVelocities): + if o.getNum() == v.getNum(): + o.velocities = v.positions + o.velocities.duplicateLastPosition() # avoid having velocity shorter by one position than positions + else: + print('Could not match positions {0} with velocities {1}'.format(o.getNum(), v.getNum())) + + if trajectoryType == 'object': + cursor = connection.cursor() + try: + # attribute feature numbers to objects + queryStatement = 'SELECT trajectory_id, object_id FROM objects_features' + if objectNumbers is not None: + queryStatement += ' WHERE object_id '+getObjectCriteria(objectNumbers) + queryStatement += ' ORDER BY object_id' # order is important to group all features per object + logging.debug(queryStatement) + cursor.execute(queryStatement) + + featureNumbers = {} + for row in cursor: + objId = row[1] + if objId not in featureNumbers: + featureNumbers[objId] = [row[0]] + else: + featureNumbers[objId].append(row[0]) + + for obj in objects: + obj.featureNumbers = featureNumbers[obj.getNum()] + + # load userType + userTypes = loadUserTypesFromTable(cursor, objectNumbers) + for obj in objects: + obj.userType = userTypes[obj.getNum()] + + if withFeatures: + nFeatures = 0 + for obj in objects: + nFeatures = max(nFeatures, max(obj.featureNumbers)) + features = loadTrajectoriesFromSqlite(filename, 'feature', nFeatures+1, timeStep = timeStep) + for obj in objects: + obj.setFeatures(features) + + except sqlite3.OperationalError as error: + printDBError(error) + return objects + +def loadObjectFeatureFrameNumbers(filename, objectNumbers = None): + 'Loads the feature frame numbers for each object' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + queryStatement = 'SELECT OF.object_id, TL.trajectory_id, TL.length FROM (SELECT trajectory_id, max(frame_number)-min(frame_number) AS length FROM positions GROUP BY trajectory_id) TL, objects_features OF WHERE TL.trajectory_id = OF.trajectory_id' + if objectNumbers is not None: + queryStatement += ' AND object_id '+getObjectCriteria(objectNumbers) + queryStatement += ' ORDER BY OF.object_id, TL.length DESC' + logging.debug(queryStatement) + cursor.execute(queryStatement) + objectFeatureNumbers = {} + for row in cursor: + objId = row[0] + if objId in objectFeatureNumbers: + objectFeatureNumbers[objId].append(row[1]) + else: + objectFeatureNumbers[objId] = [row[1]] + return objectFeatureNumbers + except sqlite3.OperationalError as error: + printDBError(error) + return None + +def addCurvilinearTrajectoriesFromSqlite(filename, objects): + '''Adds curvilinear positions (s_coordinate, y_coordinate, lane) + from a database to an existing MovingObject dict (indexed by each objects's num)''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from curvilinear_positions order by trajectory_id, frame_number') + except sqlite3.OperationalError as error: + printDBError(error) + return [] + + missingObjectNumbers = [] + objNum = None + for row in cursor: + if objNum != row[0]: + objNum = row[0] + if objNum in objects: + objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() + else: + missingObjectNumbers.append(objNum) + if objNum in objects: + objects[objNum].curvilinearPositions.addPositionSYL(row[2],row[3],row[4]) + if len(missingObjectNumbers) > 0: + print('List of missing objects to attach corresponding curvilinear trajectories: {}'.format(missingObjectNumbers)) + +def saveTrajectoriesToTable(connection, objects, trajectoryType, tablePrefix = None): + 'Saves trajectories in table tableName' + cursor = connection.cursor() + # Parse feature and/or object structure and commit to DB + if(trajectoryType == 'feature' or trajectoryType == 'object'): + # Extract features from objects + if trajectoryType == 'object': + features = [] + for obj in objects: + if obj.hasFeatures(): + features += obj.getFeatures() + if len(features) == 0: + print('Warning, objects have no features') # todo save centroid trajectories? + elif trajectoryType == 'feature': + features = objects + # Setup feature queries + if tablePrefix is None: + prefix = '' + else: + prefix = tablePrefix+'_' + createTrajectoryTable(cursor, prefix+"positions") + createTrajectoryTable(cursor, prefix+"velocities") + positionQuery = insertTrajectoryQuery(prefix+"positions") + velocityQuery = insertTrajectoryQuery(prefix+"velocities") + # Setup object queries + if trajectoryType == 'object': + createObjectsTable(cursor) + createObjectsFeaturesTable(cursor) + objectQuery = insertObjectQuery() + objectFeatureQuery = insertObjectFeatureQuery() + for feature in features: + num = feature.getNum() + frameNum = feature.getFirstInstant() + for p in feature.getPositions(): + cursor.execute(positionQuery, (num, frameNum, p.x, p.y)) + frameNum += 1 + velocities = feature.getVelocities() + if velocities is not None: + frameNum = feature.getFirstInstant() + for v in velocities[:-1]: + cursor.execute(velocityQuery, (num, frameNum, v.x, v.y)) + frameNum += 1 + if trajectoryType == 'object': + for obj in objects: + if obj.hasFeatures(): + for feature in obj.getFeatures(): + featureNum = feature.getNum() + cursor.execute(objectFeatureQuery, (obj.getNum(), featureNum)) + cursor.execute(objectQuery, (obj.getNum(), obj.getUserType(), 1)) + # Parse curvilinear position structure + elif(trajectoryType == 'curvilinear'): + createCurvilinearTrajectoryTable(cursor) + curvilinearQuery = "INSERT INTO curvilinear_positions VALUES (?,?,?,?,?)" + for obj in objects: + num = obj.getNum() + frameNum = obj.getFirstInstant() + for p in obj.getCurvilinearPositions(): + cursor.execute(curvilinearQuery, (num, frameNum, p[0], p[1], p[2])) + frameNum += 1 + else: + print('Unknown trajectory type {}'.format(trajectoryType)) + connection.commit() + +def saveTrajectoriesToSqlite(outputFilename, objects, trajectoryType): + '''Writes features, ie the trajectory positions (and velocities if exist) + with their instants to a specified sqlite file + Either feature positions (and velocities if they exist) + or curvilinear positions will be saved at a time''' + + with sqlite3.connect(outputFilename) as connection: + try: + saveTrajectoriesToTable(connection, objects, trajectoryType, None) + except sqlite3.OperationalError as error: + printDBError(error) + +def setRoadUserTypes(filename, objects): + '''Saves the user types of the objects in the sqlite database stored in filename + The objects should exist in the objects table''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + for obj in objects: + cursor.execute('update objects set road_user_type = {} WHERE object_id = {}'.format(obj.getUserType(), obj.getNum())) + connection.commit() + +def loadBBMovingObjectsFromSqlite(filename, objectType = 'bb', objectNumbers = None, timeStep = None): + '''Loads bounding box moving object from an SQLite + (format of SQLite output by the ground truth annotation tool + or Urban Tracker + + Load descriptions?''' + objects = [] + with sqlite3.connect(filename) as connection: + if objectType == 'bb': + topCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbtop', objectNumbers, timeStep) + bottomCorners = loadTrajectoriesFromTable(connection, 'bounding_boxes', 'bbbottom', objectNumbers, timeStep) + userTypes = loadUserTypesFromTable(connection.cursor(), objectNumbers) # string format is same as object + + for t, b in zip(topCorners, bottomCorners): + num = t.getNum() + if t.getNum() == b.getNum(): + annotation = moving.BBMovingObject(num, t.getTimeInterval(), t, b, userTypes[num]) + objects.append(annotation) + else: + print ('Unknown type of bounding box {}'.format(objectType)) + return objects + +def saveInteraction(cursor, interaction): + roadUserNumbers = list(interaction.getRoadUserNumbers()) + cursor.execute('INSERT INTO interactions VALUES({}, {}, {}, {}, {})'.format(interaction.getNum(), roadUserNumbers[0], roadUserNumbers[1], interaction.getFirstInstant(), interaction.getLastInstant())) + +def saveInteractionsToSqlite(filename, interactions): + 'Saves the interactions in the table' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + createInteractionTable(cursor) + for inter in interactions: + saveInteraction(cursor, inter) + except sqlite3.OperationalError as error: + printDBError(error) + connection.commit() + +def saveIndicator(cursor, interactionNum, indicator): + for instant in indicator.getTimeInterval(): + if indicator[instant]: + cursor.execute('INSERT INTO indicators VALUES({}, {}, {}, {})'.format(interactionNum, events.Interaction.indicatorNameToIndices[indicator.getName()], instant, indicator[instant])) + +def saveIndicatorsToSqlite(filename, interactions, indicatorNames = events.Interaction.indicatorNames): + 'Saves the indicator values in the table' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + createInteractionTable(cursor) + createIndicatorTable(cursor) + for inter in interactions: + saveInteraction(cursor, inter) + for indicatorName in indicatorNames: + indicator = inter.getIndicator(indicatorName) + if indicator is not None: + saveIndicator(cursor, inter.getNum(), indicator) + except sqlite3.OperationalError as error: + printDBError(error) + connection.commit() + +def loadInteractionsFromSqlite(filename): + '''Loads interaction and their indicators + + TODO choose the interactions to load''' + interactions = [] + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('SELECT INT.id, INT.object_id1, INT.object_id2, INT.first_frame_number, INT.last_frame_number, IND.indicator_type, IND.frame_number, IND.value from interactions INT, indicators IND WHERE INT.id = IND.interaction_id ORDER BY INT.id, IND.indicator_type, IND.frame_number') + interactionNum = -1 + indicatorTypeNum = -1 + tmpIndicators = {} + for row in cursor: + if row[0] != interactionNum: + interactionNum = row[0] + interactions.append(events.Interaction(interactionNum, moving.TimeInterval(row[3],row[4]), row[1], row[2])) + interactions[-1].indicators = {} + if indicatorTypeNum != row[5] or row[0] != interactionNum: + indicatorTypeNum = row[5] + indicatorName = events.Interaction.indicatorNames[indicatorTypeNum] + indicatorValues = {row[6]:row[7]} + interactions[-1].indicators[indicatorName] = indicators.SeverityIndicator(indicatorName, indicatorValues, mostSevereIsMax = not indicatorName in events.Interaction.timeIndicators) + else: + indicatorValues[row[6]] = row[7] + interactions[-1].indicators[indicatorName].timeInterval.last = row[6] + except sqlite3.OperationalError as error: + printDBError(error) + return [] + return interactions +# load first and last object instants +# CREATE TEMP TABLE IF NOT EXISTS object_instants AS SELECT OF.object_id, min(frame_number) as first_instant, max(frame_number) as last_instant from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id group by OF.object_id order by OF.object_id + +def createBoundingBoxTable(filename, invHomography = None): + '''Create the table to store the object bounding boxes in image space + ''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('CREATE TABLE IF NOT EXISTS bounding_boxes (object_id INTEGER, frame_number INTEGER, x_top_left REAL, y_top_left REAL, x_bottom_right REAL, y_bottom_right REAL, PRIMARY KEY(object_id, frame_number))') + cursor.execute('INSERT INTO bounding_boxes SELECT object_id, frame_number, min(x), min(y), max(x), max(y) from ' + '(SELECT object_id, frame_number, (x*{}+y*{}+{})/w as x, (x*{}+y*{}+{})/w as y from ' + '(SELECT OF.object_id, P.frame_number, P.x_coordinate as x, P.y_coordinate as y, P.x_coordinate*{}+P.y_coordinate*{}+{} as w from positions P, objects_features OF WHERE P.trajectory_id = OF.trajectory_id)) '.format(invHomography[0,0], invHomography[0,1], invHomography[0,2], invHomography[1,0], invHomography[1,1], invHomography[1,2], invHomography[2,0], invHomography[2,1], invHomography[2,2])+ + 'GROUP BY object_id, frame_number') + except sqlite3.OperationalError as error: + printDBError(error) + connection.commit() + +def loadBoundingBoxTableForDisplay(filename): + '''Loads bounding boxes from bounding_boxes table for display over trajectories''' + boundingBoxes = {} # list of bounding boxes for each instant + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('SELECT name FROM sqlite_master WHERE type=\'table\' AND name=\'bounding_boxes\'') + result = cursor.fetchall() + if len(result) > 0: + cursor.execute('SELECT * FROM bounding_boxes') + for row in cursor: + boundingBoxes.setdefault(row[1], []).append([moving.Point(row[2], row[3]), moving.Point(row[4], row[5])]) + except sqlite3.OperationalError as error: + printDBError(error) + return boundingBoxes + return boundingBoxes + +######################### +# saving and loading for scene interpretation: POIs and Prototypes +######################### + +def savePrototypesToSqlite(filename, prototypes): + '''save the prototypes (a prototype is defined by a filename, a number (id) and type''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_filename VARCHAR, prototype_id INTEGER, trajectory_type VARCHAR CHECK (trajectory_type IN (\"feature\", \"object\")), nmatchings INTEGER, PRIMARY KEY (prototype_filename, prototype_id, trajectory_type))') + for p in prototypes: + cursor.execute('INSERT INTO prototypes VALUES(?,?,?,?)', (p.getFilename(), p.getNum(), p.getTrajectoryType(), p.getNMatchings())) + except sqlite3.OperationalError as error: + printDBError(error) + connection.commit() + +def savePrototypeAssignmentsToSqlite(filename, objects, objectType, labels, prototypes): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + if objectType == 'feature': + tableName = 'features_prototypes' + objectIdColumnName = 'trajectory_id' + elif objectType == 'object': + tableName = 'objects_prototypes' + objectIdColumnName = 'object_id' + cursor.execute('CREATE TABLE IF NOT EXISTS '+tableName+' ('+objectIdColumnName+' INTEGER, prototype_filename VARCHAR, prototype_id INTEGER, trajectory_type VARCHAR CHECK (trajectory_type IN (\"feature\", \"object\")), PRIMARY KEY('+objectIdColumnName+', prototype_filename, prototype_id, trajectory_type))') + for obj, label in zip(objects, labels): + proto = prototypes[label] + cursor.execute('INSERT INTO objects_prototypes VALUES(?,?,?,?)', (obj.getNum(), proto.getFilename(), proto.getNum(), proto.getTrajectoryType())) + except sqlite3.OperationalError as error: + printDBError(error) + connection.commit() + +def loadPrototypesFromSqlite(filename, withTrajectories = True): + 'Loads prototype ids and matchings (if stored)' + prototypes = [] + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + objects = [] + try: + cursor.execute('SELECT * FROM prototypes') + for row in cursor: + prototypes.append(moving.Prototype(row[0], row[1], row[2], row[3])) + if withTrajectories: + for p in prototypes: + p.setMovingObject(loadTrajectoriesFromSqlite(p.getFilename(), p.getTrajectoryType(), [p.getNum()])[0]) + # loadingInformation = {} # complicated slightly optimized + # for p in prototypes: + # dbfn = p.getFilename() + # trajType = p.getTrajectoryType() + # if (dbfn, trajType) in loadingInformation: + # loadingInformation[(dbfn, trajType)].append(p) + # else: + # loadingInformation[(dbfn, trajType)] = [p] + # for k, v in loadingInformation.iteritems(): + # objects += loadTrajectoriesFromSqlite(k[0], k[1], [p.getNum() for p in v]) + except sqlite3.OperationalError as error: + printDBError(error) + if len(set([p.getTrajectoryType() for p in prototypes])) > 1: + print('Different types of prototypes in database ({}).'.format(set([p.getTrajectoryType() for p in prototypes]))) + return prototypes + +def savePOIsToSqlite(filename, gmm, gmmType, gmmId): + '''Saves a Gaussian mixture model (of class sklearn.mixture.GaussianMixture) + gmmType is a type of GaussianMixture, learnt either from beginnings or ends of trajectories''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + if gmmType not in ['beginning', 'end']: + print('Unknown POI type {}. Exiting'.format(gmmType)) + import sys + sys.exit() + try: + cursor.execute('CREATE TABLE IF NOT EXISTS gaussians2d (poi_id INTEGER, id INTEGER, type VARCHAR, x_center REAL, y_center REAL, covariance VARCHAR, covariance_type VARCHAR, weight, precisions_cholesky VARCHAR, PRIMARY KEY(poi_id, id))') + for i in range(gmm.n_components): + cursor.execute('INSERT INTO gaussians2d VALUES(?,?,?,?,?,?,?,?,?)', (gmmId, i, gmmType, gmm.means_[i][0], gmm.means_[i][1], str(gmm.covariances_[i].tolist()), gmm.covariance_type, gmm.weights_[i], str(gmm.precisions_cholesky_[i].tolist()))) + connection.commit() + except sqlite3.OperationalError as error: + printDBError(error) + +def savePOIAssignmentsToSqlite(filename, objects): + 'save the od fields of objects' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('CREATE TABLE IF NOT EXISTS objects_pois (object_id INTEGER, origin_poi_id INTEGER, destination_poi_id INTEGER, PRIMARY KEY(object_id))') + for o in objects: + cursor.execute('INSERT INTO objects_pois VALUES(?,?,?)', (o.getNum(), o.od[0], o.od[1])) + connection.commit() + except sqlite3.OperationalError as error: + printDBError(error) + +def loadPOIsFromSqlite(filename): + 'Loads all 2D Gaussians in the database' + from sklearn import mixture # todo if not avalaible, load data in duck-typed class with same fields + from ast import literal_eval + pois = [] + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + try: + cursor.execute('SELECT * from gaussians2d') + gmmId = None + gmm = [] + for row in cursor: + if gmmId is None or row[0] != gmmId: + if len(gmm) > 0: + tmp = mixture.GaussianMixture(len(gmm), covarianceType) + tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) + tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) + tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) + tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] + tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) + pois.append(tmp) + gaussian = {'type': row[2], + 'mean': row[3:5], + 'covar': array(literal_eval(row[5])), + 'weight': row[7], + 'precisions': array(literal_eval(row[8]))} + gmm = [gaussian] + covarianceType = row[6] + gmmId = row[0] + else: + gmm.append({'type': row[2], + 'mean': row[3:5], + 'covar': array(literal_eval(row[5])), + 'weight': row[7], + 'precisions': array(literal_eval(row[8]))}) + if len(gmm) > 0: + tmp = mixture.GaussianMixture(len(gmm), covarianceType) + tmp.means_ = array([gaussian['mean'] for gaussian in gmm]) + tmp.covariances_ = array([gaussian['covar'] for gaussian in gmm]) + tmp.weights_ = array([gaussian['weight'] for gaussian in gmm]) + tmp.gmmTypes = [gaussian['type'] for gaussian in gmm] + tmp.precisions_cholesky_ = array([gaussian['precisions'] for gaussian in gmm]) + pois.append(tmp) + except sqlite3.OperationalError as error: + printDBError(error) + return pois + +######################### +# saving and loading for scene interpretation (Mohamed Gomaa Mohamed's PhD) +######################### + +def writePrototypesToSqlite(prototypes,nMatching, outputFilename): + ''' prototype dataset is a dictionary with keys== routes, values== prototypes Ids ''' + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute('CREATE TABLE IF NOT EXISTS prototypes (prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))') + + for route in prototypes: + if prototypes[route]!=[]: + for i in prototypes[route]: + cursor.execute('insert into prototypes (prototype_id, routeIDstart,routeIDend, nMatching) values (?,?,?,?)',(i,route[0],route[1],nMatching[route][i])) + + connection.commit() + connection.close() + +def readPrototypesFromSqlite(filename): + ''' + This function loads the prototype file in the database + It returns a dictionary for prototypes for each route and nMatching + ''' + prototypes = {} + nMatching={} + + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from prototypes order by prototype_id, routeIDstart,routeIDend, nMatching') + except sqlite3.OperationalError as error: + printDBError(error) + return [] + + for row in cursor: + route=(row[1],row[2]) + if route not in prototypes: + prototypes[route]=[] + prototypes[route].append(row[0]) + nMatching[row[0]]=row[3] + + connection.close() + return prototypes,nMatching + +def writeLabelsToSqlite(labels, outputFilename): + """ labels is a dictionary with keys: routes, values: prototypes Ids + """ + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS labels (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") + + for route in labels: + if labels[route]!=[]: + for i in labels[route]: + for j in labels[route][i]: + cursor.execute("insert into labels (object_id, routeIDstart,routeIDend, prototype_id) values (?,?,?,?)",(j,route[0],route[1],i)) + + connection.commit() + connection.close() + +def loadLabelsFromSqlite(filename): + labels = {} + + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from labels order by object_id, routeIDstart,routeIDend, prototype_id') + except sqlite3.OperationalError as error: + printDBError(error) + return [] + + for row in cursor: + route=(row[1],row[2]) + p=row[3] + if route not in labels: + labels[route]={} + if p not in labels[route]: + labels[route][p]=[] + labels[route][p].append(row[0]) + + connection.close() + return labels + +def writeSpeedPrototypeToSqlite(prototypes,nmatching, outFilename): + """ to match the format of second layer prototypes""" + connection = sqlite3.connect(outFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS speedprototypes (spdprototype_id INTEGER,prototype_id INTEGER,routeID_start INTEGER, routeID_end INTEGER, nMatching INTEGER, PRIMARY KEY(spdprototype_id))") + + for route in prototypes: + if prototypes[route]!={}: + for i in prototypes[route]: + if prototypes[route][i]!= []: + for j in prototypes[route][i]: + cursor.execute("insert into speedprototypes (spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching) values (?,?,?,?,?)",(j,i,route[0],route[1],nmatching[j])) + + connection.commit() + connection.close() + +def loadSpeedPrototypeFromSqlite(filename): + """ + This function loads the prototypes table in the database of name <filename>. + """ + prototypes = {} + nMatching={} + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from speedprototypes order by spdprototype_id,prototype_id, routeID_start, routeID_end, nMatching') + except sqlite3.OperationalError as error: + printDBError(error) + return [] + + for row in cursor: + route=(row[2],row[3]) + if route not in prototypes: + prototypes[route]={} + if row[1] not in prototypes[route]: + prototypes[route][row[1]]=[] + prototypes[route][row[1]].append(row[0]) + nMatching[row[0]]=row[4] + + connection.close() + return prototypes,nMatching + + +def writeRoutesToSqlite(Routes, outputFilename): + """ This function writes the activity path define by start and end IDs""" + connection = sqlite3.connect(outputFilename) + cursor = connection.cursor() + + cursor.execute("CREATE TABLE IF NOT EXISTS routes (object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") + + for route in Routes: + if Routes[route]!=[]: + for i in Routes[route]: + cursor.execute("insert into routes (object_id, routeIDstart,routeIDend) values (?,?,?)",(i,route[0],route[1])) + + connection.commit() + connection.close() + +def loadRoutesFromSqlite(filename): + Routes = {} + + connection = sqlite3.connect(filename) + cursor = connection.cursor() + + try: + cursor.execute('SELECT * from routes order by object_id, routeIDstart,routeIDend') + except sqlite3.OperationalError as error: + printDBError(error) + return [] + + for row in cursor: + route=(row[1],row[2]) + if route not in Routes: + Routes[route]=[] + Routes[route].append(row[0]) + + connection.close() + return Routes + +def setRoutes(filename, objects): + connection = sqlite3.connect(filename) + cursor = connection.cursor() + for obj in objects: + cursor.execute('update objects set startRouteID = {} WHERE object_id = {}'.format(obj.startRouteID, obj.getNum())) + cursor.execute('update objects set endRouteID = {} WHERE object_id = {}'.format(obj.endRouteID, obj.getNum())) + connection.commit() + connection.close() + +######################### +# txt files +######################### + +def openCheck(filename, option = 'r', quitting = False): + '''Open file filename in read mode by default + and checks it is open''' + try: + return open(filename, option) + except IOError: + print('File {} could not be opened.'.format(filename)) + if quitting: + from sys import exit + exit() + return None + +def readline(f, commentCharacters = commentChar): + '''Modified readline function to skip comments + Can take a list of characters or a string (in will work in both)''' + s = f.readline() + while (len(s) > 0) and s[0] in commentCharacters: + s = f.readline() + return s.strip() + +def getLines(f, delimiterChar = delimiterChar, commentCharacters = commentChar): + '''Gets a complete entry (all the lines) in between delimiterChar.''' + dataStrings = [] + s = readline(f, commentCharacters) + while len(s) > 0 and s[0] != delimiterChar: + dataStrings += [s.strip()] + s = readline(f, commentCharacters) + return dataStrings + +def saveList(filename, l): + f = openCheck(filename, 'w') + for x in l: + f.write('{}\n'.format(x)) + f.close() + +def loadListStrings(filename, commentCharacters = commentChar): + f = openCheck(filename, 'r') + result = getLines(f, commentCharacters) + f.close() + return result + +def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar): + values = [] + for l in loadListStrings(filename, commentCharacters): + if l.startswith(option): + values.append(l.split(delimiterChar)[1].strip()) + return values + +def addSectionHeader(propertiesFile, headerName = 'main'): + '''Add fake section header + + from http://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/2819788#2819788 + use read_file in Python 3.2+ + ''' + yield '[{}]\n'.format(headerName) + for line in propertiesFile: + yield line + +def loadPemsTraffic(filename): + '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse + into pandas dataframe''' + f = openCheck(filename) + l = f.readline().strip() + items = l.split(',') + headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] + nLanes = (len(items)-len(headers))/3 + for i in range(nLanes): + headers += ['flow{}'.format(i+1), 'occupancy{}'.format(i+1), 'speed{}'.format(i+1)] + f.close() + return read_csv(filename, delimiter = ',', names = headers) + +def generatePDLaneColumn(data): + data['LANE'] = data['LANE\\LINK\\NO'].astype(str)+'_'+data['LANE\\INDEX'].astype(str) + +def convertTrajectoriesVissimToSqlite(filename): + '''Relies on a system call to sqlite3 + sqlite3 [file.sqlite] < import_fzp.sql''' + sqlScriptFilename = "import_fzp.sql" + # create sql file + out = openCheck(sqlScriptFilename, "w") + out.write(".separator \";\"\n"+ + "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+ + ".import "+filename+" curvilinear_positions\n"+ + "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") + out.close() + # system call + from subprocess import run + out = openCheck("err.log", "w") + run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) + out.close() + shutil.os.remove(sqlScriptFilename) + +def loadObjectNumbersInLinkFromVissimFile(filename, linkIds): + '''Finds the ids of the objects that go through any of the link in the list linkIds''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + queryStatement = 'SELECT DISTINCT trajectory_id FROM curvilinear_positions where link_id IN ('+','.join([str(id) for id in linkIds])+')' + try: + cursor.execute(queryStatement) + return [row[0] for row in cursor] + except sqlite3.OperationalError as error: + printDBError(error) + +def getNObjectsInLinkFromVissimFile(filename, linkIds): + '''Returns the number of objects that traveled through the link ids''' + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + queryStatement = 'SELECT link_id, COUNT(DISTINCT trajectory_id) FROM curvilinear_positions where link_id IN ('+','.join([str(id) for id in linkIds])+') GROUP BY link_id' + try: + cursor.execute(queryStatement) + return {row[0]:row[1] for row in cursor} + except sqlite3.OperationalError as error: + printDBError(error) + +def loadTrajectoriesFromVissimFile(filename, simulationStepsPerTimeUnit, objectNumbers = None, warmUpLastInstant = None, usePandas = False, nDecimals = 2, lowMemory = True): + '''Reads data from VISSIM .fzp trajectory file + simulationStepsPerTimeUnit is the number of simulation steps per unit of time used by VISSIM (second) + for example, there seems to be 10 simulation steps per simulated second in VISSIM, + so simulationStepsPerTimeUnit should be 10, + so that all times correspond to the number of the simulation step (and can be stored as integers) + + Objects positions will be considered only after warmUpLastInstant + (if the object has no such position, it won't be loaded) + + Assumed to be sorted over time + Warning: if reading from SQLite a limited number of objects, objectNumbers will be the maximum object id''' + objects = {} # dictionary of objects index by their id + + if usePandas: + data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory) + generatePDLaneColumn(data) + data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit + if warmUpLastInstant is not None: + data = data[data['TIME']>=warmUpLastInstant] + grouped = data.loc[:,['NO','TIME']].groupby(['NO'], as_index = False) + instants = grouped['TIME'].agg({'first': npmin, 'last': npmax}) + for row_index, row in instants.iterrows(): + objNum = int(row['NO']) + tmp = data[data['NO'] == objNum] + objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(row['first'], row['last'])) + # positions should be rounded to nDecimals decimals only + objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) + if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: + return list(objects.values()) + else: + if filename.endswith(".fzp"): + inputfile = openCheck(filename, quitting = True) + line = readline(inputfile, '*$') + while len(line) > 0:#for line in inputfile: + data = line.strip().split(';') + objNum = int(data[1]) + instant = float(data[0])*simulationStepsPerTimeUnit + s = float(data[4]) + y = float(data[5]) + lane = data[2]+'_'+data[3] + if objNum not in objects: + if warmUpLastInstant is None or instant >= warmUpLastInstant: + if objectNumbers is None or len(objects) < objectNumbers: + objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) + objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() + if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: + objects[objNum].timeInterval.last = instant + objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) + line = readline(inputfile, '*$') + elif filename.endswith(".sqlite"): + with sqlite3.connect(filename) as connection: + cursor = connection.cursor() + queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions' + if objectNumbers is not None: + queryStatement += ' WHERE trajectory_id '+getObjectCriteria(objectNumbers) + queryStatement += ' ORDER BY trajectory_id, t' + try: + cursor.execute(queryStatement) + for row in cursor: + objNum = row[1] + instant = row[0]*simulationStepsPerTimeUnit + s = row[4] + y = row[5] + lane = '{}_{}'.format(row[2], row[3]) + if objNum not in objects: + if warmUpLastInstant is None or instant >= warmUpLastInstant: + if objectNumbers is None or len(objects) < objectNumbers: + objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) + objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() + if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: + objects[objNum].timeInterval.last = instant + objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) + except sqlite3.OperationalError as error: + printDBError(error) + else: + print("File type of "+filename+" not supported (only .sqlite and .fzp files)") + return list(objects.values()) + +def selectPDLanes(data, lanes = None): + '''Selects the subset of data for the right lanes + + Lane format is a string 'x_y' where x is link index and y is lane index''' + if lanes is not None: + if 'LANE' not in data.columns: + generatePDLaneColumn(data) + indices = (data['LANE'] == lanes[0]) + for l in lanes[1:]: + indices = indices | (data['LANE'] == l) + return data[indices] + else: + return data + +def countStoppedVehiclesVissim(filename, lanes = None, proportionStationaryTime = 0.7): + '''Counts the number of vehicles stopped for a long time in a VISSIM trajectory file + and the total number of vehicles + + Vehicles are considered finally stationary + if more than proportionStationaryTime of their total time + If lanes is not None, only the data for the selected lanes will be provided + (format as string x_y where x is link index and y is lane index)''' + if filename.endswith(".fzp"): + columns = ['NO', '$VEHICLE:SIMSEC', 'POS'] + if lanes is not None: + columns += ['LANE\\LINK\\NO', 'LANE\\INDEX'] + data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory) + data = selectPDLanes(data, lanes) + data.sort(['$VEHICLE:SIMSEC'], inplace = True) + + nStationary = 0 + nVehicles = 0 + for name, group in data.groupby(['NO'], sort = False): + nVehicles += 1 + positions = array(group['POS']) + diff = positions[1:]-positions[:-1] + if npsum(diff == 0.) >= proportionStationaryTime*(len(positions)-1): + nStationary += 1 + elif filename.endswith(".sqlite"): + # select trajectory_id, t, s_coordinate, speed from curvilinear_positions where trajectory_id between 1860 and 1870 and speed < 0.1 + # pb of the meaning of proportionStationaryTime in arterial network? Why proportion of existence time? + pass + else: + print("File type of "+filename+" not supported (only .sqlite and .fzp files)") + + return nStationary, nVehicles + +def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2, lowMemory = True): + '''Counts the number of collisions per lane in a VISSIM trajectory file + + To distinguish between cars passing and collision, + one checks when the sign of the position difference inverts + (if the time are closer than collisionTimeDifference) + If lanes is not None, only the data for the selected lanes will be provided + (format as string x_y where x is link index and y is lane index)''' + data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory) + data = selectPDLanes(data, lanes) + data = data.convert_objects(convert_numeric=True) + + merged = merge(data, data, how='inner', left_on=['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\\LINK\\NO', 'LANE\\INDEX', '$VEHICLE:SIMSEC'], sort = False) + merged = merged[merged['NO_x']>merged['NO_y']] + + nCollisions = 0 + for name, group in merged.groupby(['LANE\\LINK\\NO', 'LANE\\INDEX', 'NO_x', 'NO_y']): + diff = group['POS_x']-group['POS_y'] + # diff = group['POS_x']-group['POS_y'] # to check the impact of convert_objects and the possibility of using type conversion in read_csv or function to convert strings if any + if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0: + xidx = diff[diff < 0].argmax() + yidx = diff[diff > 0].argmin() + if abs(group.loc[xidx, '$VEHICLE:SIMSEC'] - group.loc[yidx, '$VEHICLE:SIMSEC']) <= collisionTimeDifference: + nCollisions += 1 + + # select TD1.link_id, TD1.lane_id from temp.diff_positions as TD1, temp.diff_positions as TD2 where TD1.link_id = TD2.link_id and TD1.lane_id = TD2.lane_id and TD1.id1 = TD2.id1 and TD1.id2 = TD2.id2 and TD1.t = TD2.t+0.1 and TD1.diff*TD2.diff < 0; # besoin de faire un group by?? + # create temp table diff_positions as select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id; + # SQL select link_id, lane_id, id1, id2, min(diff), max(diff) from (select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id) group by link_id, lane_id, id1, id2 having min(diff)*max(diff) < 0 + return nCollisions + +def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): + '''Reads data from the trajectory data provided by NGSIM project + and returns the list of Feature objects''' + objects = [] + + inputfile = openCheck(filename, quitting = True) + + def createObject(numbers): + firstFrameNum = int(numbers[1]) + # do the geometry and usertype + + firstFrameNum = int(numbers[1]) + lastFrameNum = firstFrameNum+int(numbers[2])-1 + #time = moving.TimeInterval(firstFrameNum, firstFrameNum+int(numbers[2])-1) + obj = moving.MovingObject(num = int(numbers[0]), + timeInterval = moving.TimeInterval(firstFrameNum, lastFrameNum), + positions = moving.Trajectory([[float(numbers[6])],[float(numbers[7])]]), + userType = int(numbers[10])) + obj.userType = int(numbers[10]) + obj.laneNums = [int(numbers[13])] + obj.precedingVehicles = [int(numbers[14])] # lead vehicle (before) + obj.followingVehicles = [int(numbers[15])] # following vehicle (after) + obj.spaceHeadways = [float(numbers[16])] # feet + obj.timeHeadways = [float(numbers[17])] # seconds + obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate + obj.speeds = [float(numbers[11])] + obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object + return obj + + numbers = readline(inputfile).strip().split() + if (len(numbers) > 0): + obj = createObject(numbers) + + for line in inputfile: + numbers = line.strip().split() + if obj.getNum() != int(numbers[0]): + # check and adapt the length to deal with issues in NGSIM data + if (obj.length() != obj.positions.length()): + print('length pb with object {} ({},{})'.format(obj.getNum(),obj.length(),obj.positions.length())) + obj.last = obj.getFirstInstant()+obj.positions.length()-1 + #obj.velocities = utils.computeVelocities(f.positions) # compare norm to speeds ? + objects.append(obj) + if (nObjects>0) and (len(objects)>=nObjects): + break + obj = createObject(numbers) + else: + obj.laneNums.append(int(numbers[13])) + obj.positions.addPositionXY(float(numbers[6]), float(numbers[7])) + obj.curvilinearPositions.addPositionSYL(float(numbers[5]), float(numbers[4]), obj.laneNums[-1]) + obj.speeds.append(float(numbers[11])) + obj.precedingVehicles.append(int(numbers[14])) + obj.followingVehicles.append(int(numbers[15])) + obj.spaceHeadways.append(float(numbers[16])) + obj.timeHeadways.append(float(numbers[17])) + + if (obj.size[0] != float(numbers[8])): + print('changed length obj {}'.format(obj.getNum())) + if (obj.size[1] != float(numbers[9])): + print('changed width obj {}'.format(obj.getNum())) + + inputfile.close() + return objects + +def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): + '''Reads data from the trajectory data provided by NGSIM project + and converts to our current format.''' + if append: + out = openCheck(outputfile,'a') + else: + out = openCheck(outputfile,'w') + nObjectsPerType = [0,0,0] + + features = loadNgsimFile(inputfile, sequenceNum) + for f in features: + nObjectsPerType[f.userType-1] += 1 + f.write(out) + + print(nObjectsPerType) + + out.close() + +def loadPinholeCameraModel(filename, tanalystFormat = True): + '''Loads the data from a file containing the camera parameters + (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) + and returns a dictionary''' + if tanalystFormat: + f = openCheck(filename, quitting = True) + content = getLines(f) + cameraData = {} + for l in content: + tmp = l.split(':') + cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.')) + return cameraData + else: + print('Unknown camera model (not tanalyst format') + return None + +def savePositionsToCsv(f, obj): + timeInterval = obj.getTimeInterval() + positions = obj.getPositions() + curvilinearPositions = obj.getCurvilinearPositions() + for i in range(int(obj.length())): + p1 = positions[i] + s = '{},{},{},{}'.format(obj.num,timeInterval[i],p1.x,p1.y) + if curvilinearPositions is not None: + p2 = curvilinearPositions[i] + s += ',{},{}'.format(p2[0],p2[1]) + f.write(s+'\n') + +def saveTrajectoriesToCsv(filename, objects): + f = openCheck(filename, 'w') + for i,obj in enumerate(objects): + savePositionsToCsv(f, obj) + f.close() + + +######################### +# Utils to read .ini type text files for configuration, meta data... +######################### + +class ClassifierParameters(VideoFilenameAddable): + 'Class for the parameters of object classifiers' + def loadConfigFile(self, filename): + from configparser import ConfigParser + + config = ConfigParser() + config.read_file(addSectionHeader(openCheck(filename))) + + parentPath = Path(filename).parent + self.sectionHeader = config.sections()[0] + + self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename')) + self.bikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'bv-svm-filename')) + self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop') + self.minNPixels = config.getint(self.sectionHeader, 'min-npixels-crop') + x = config.getint(self.sectionHeader, 'hog-rescale-size') + self.hogRescaleSize = (x, x) + self.hogNOrientations = config.getint(self.sectionHeader, 'hog-norientations') + x = config.getint(self.sectionHeader, 'hog-npixels-cell') + self.hogNPixelsPerCell = (x, x) + x = config.getint(self.sectionHeader, 'hog-ncells-block') + self.hogNCellsPerBlock = (x, x) + self.hogBlockNorm = config.get(self.sectionHeader, 'hog-block-norm') + + self.speedAggregationMethod = config.get(self.sectionHeader, 'speed-aggregation-method') + self.nFramesIgnoreAtEnds = config.getint(self.sectionHeader, 'nframes-ignore-at-ends') + self.speedAggregationCentile = config.getint(self.sectionHeader, 'speed-aggregation-centile') + self.minSpeedEquiprobable = config.getfloat(self.sectionHeader, 'min-speed-equiprobable') + self.maxPercentUnknown = config.getfloat(self.sectionHeader, 'max-prop-unknown-appearance') + self.maxPedestrianSpeed = config.getfloat(self.sectionHeader, 'max-ped-speed') + self.maxCyclistSpeed = config.getfloat(self.sectionHeader, 'max-cyc-speed') + self.meanPedestrianSpeed = config.getfloat(self.sectionHeader, 'mean-ped-speed') + self.stdPedestrianSpeed = config.getfloat(self.sectionHeader, 'std-ped-speed') + self.locationCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-loc') + self.scaleCyclistSpeed = config.getfloat(self.sectionHeader, 'cyc-speed-scale') + self.meanVehicleSpeed = config.getfloat(self.sectionHeader, 'mean-veh-speed') + self.stdVehicleSpeed = config.getfloat(self.sectionHeader, 'std-veh-speed') + + def __init__(self, filename = None): + if filename is not None and Path(filename).exists(): + self.loadConfigFile(filename) + else: + print('Configuration filename {} could not be loaded.'.format(filename)) + + def convertToFrames(self, frameRate, speedRatio = 3.6): + '''Converts parameters with a relationship to time in 'native' frame time + speedRatio is the conversion from the speed unit in the config file + to the distance per second + + ie param(config file) = speedRatio x fps x param(used in program) + eg km/h = 3.6 (m/s to km/h) x frame/s x m/frame''' + denominator = frameRate*speedRatio + #denominator2 = denominator**2 + self.minSpeedEquiprobable = self.minSpeedEquiprobable/denominator + self.maxPedestrianSpeed = self.maxPedestrianSpeed/denominator + self.maxCyclistSpeed = self.maxCyclistSpeed/denominator + self.meanPedestrianSpeed = self.meanPedestrianSpeed/denominator + self.stdPedestrianSpeed = self.stdPedestrianSpeed/denominator + self.meanVehicleSpeed = self.meanVehicleSpeed/denominator + self.stdVehicleSpeed = self.stdVehicleSpeed/denominator + # special case for the lognormal distribution + self.locationCyclistSpeed = self.locationCyclistSpeed-log(denominator) + #self.scaleCyclistSpeed = self.scaleCyclistSpeed # no modification of scale + + +class ProcessParameters(VideoFilenameAddable): + '''Class for all parameters controlling data processing: input, + method parameters, etc. for tracking and safety + + Note: framerate is already taken into account''' + + def loadConfigFile(self, filename): + from configparser import ConfigParser + + config = ConfigParser(strict=False) + config.read_file(addSectionHeader(openCheck(filename))) + + parentPath = Path(filename).parent + self.sectionHeader = config.sections()[0] + # Tracking/display parameters + self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) + self.databaseFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'database-filename')) + self.homographyFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'homography-filename')) + if Path(self.homographyFilename).exists(): + self.homography = loadtxt(self.homographyFilename) + else: + self.homography = None + self.intrinsicCameraFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'intrinsic-camera-filename')) + if Path(self.intrinsicCameraFilename).exists(): + self.intrinsicCameraMatrix = loadtxt(self.intrinsicCameraFilename) + else: + self.intrinsicCameraMatrix = None + distortionCoefficients = getValuesFromINIFile(filename, 'distortion-coefficients', '=') + self.distortionCoefficients = [float(x) for x in distortionCoefficients] + self.undistortedImageMultiplication = config.getfloat(self.sectionHeader, 'undistorted-size-multiplication') + self.undistort = config.getboolean(self.sectionHeader, 'undistort') + self.firstFrameNum = config.getint(self.sectionHeader, 'frame1') + self.videoFrameRate = config.getfloat(self.sectionHeader, 'video-fps') + + self.minFeatureTime = config.getfloat(self.sectionHeader, 'min-feature-time') + + self.classifierFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'classifier-filename')) + + # Safety parameters + self.maxPredictedSpeed = config.getfloat(self.sectionHeader, 'max-predicted-speed')/3.6/self.videoFrameRate + self.predictionTimeHorizon = config.getfloat(self.sectionHeader, 'prediction-time-horizon')*self.videoFrameRate + self.collisionDistance = config.getfloat(self.sectionHeader, 'collision-distance') + self.crossingZones = config.getboolean(self.sectionHeader, 'crossing-zones') + self.predictionMethod = config.get(self.sectionHeader, 'prediction-method') + self.nPredictedTrajectories = config.getint(self.sectionHeader, 'npredicted-trajectories') + self.maxNormalAcceleration = config.getfloat(self.sectionHeader, 'max-normal-acceleration')/self.videoFrameRate**2 + self.maxNormalSteering = config.getfloat(self.sectionHeader, 'max-normal-steering')/self.videoFrameRate + self.minExtremeAcceleration = config.getfloat(self.sectionHeader, 'min-extreme-acceleration')/self.videoFrameRate**2 + self.maxExtremeAcceleration = config.getfloat(self.sectionHeader, 'max-extreme-acceleration')/self.videoFrameRate**2 + self.maxExtremeSteering = config.getfloat(self.sectionHeader, 'max-extreme-steering')/self.videoFrameRate + self.useFeaturesForPrediction = config.getboolean(self.sectionHeader, 'use-features-prediction') + self.constantSpeedPrototypePrediction = config.getboolean(self.sectionHeader, 'constant-speed') + self.maxLcssDistance = config.getfloat(self.sectionHeader, 'max-lcss-distance') + self.lcssMetric = config.get(self.sectionHeader, 'lcss-metric') + self.minLcssSimilarity = config.getfloat(self.sectionHeader, 'min-lcss-similarity') + + def __init__(self, filename = None): + if filename is not None and Path(filename).exists(): + self.loadConfigFile(filename) + else: + print('Configuration filename {} could not be loaded.'.format(filename)) + +def processVideoArguments(args): + '''Loads information from configuration file + then checks what was passed on the command line + for override (eg video filename and database filename''' + parentPath = Path(args.configFilename).parent + if args.configFilename is not None: # consider there is a configuration file + params = ProcessParameters(args.configFilename) + videoFilename = params.videoFilename + databaseFilename = params.databaseFilename + if params.homography is not None: + invHomography = linalg.inv(params.homography) + else: + invHomography = None + intrinsicCameraMatrix = params.intrinsicCameraMatrix + distortionCoefficients = array(params.distortionCoefficients) + undistortedImageMultiplication = params.undistortedImageMultiplication + undistort = params.undistort + firstFrameNum = params.firstFrameNum + else: + invHomography = None + undistort = False + intrinsicCameraMatrix = None + distortionCoefficients = [] + undistortedImageMultiplication = None + undistort = False + firstFrameNum = 0 + + # override video and database filenames if present on command line + # if not absolute, make all filenames relative to the location of the configuration filename + if args.videoFilename is not None: + videoFilename = args.videoFilename + else: + videoFilename = params.videoFilename + if args.databaseFilename is not None: + databaseFilename = args.databaseFilename + else: + databaseFilename = params.databaseFilename + + return params, videoFilename, databaseFilename, invHomography, intrinsicCameraMatrix, distortionCoefficients, undistortedImageMultiplication, undistort, firstFrameNum + +# deprecated +class SceneParameters(object): + def __init__(self, config, sectionName): + from configparser import NoOptionError + from ast import literal_eval + try: + self.sitename = config.get(sectionName, 'sitename') + self.databaseFilename = config.get(sectionName, 'data-filename') + self.homographyFilename = config.get(sectionName, 'homography-filename') + self.calibrationFilename = config.get(sectionName, 'calibration-filename') + self.videoFilename = config.get(sectionName, 'video-filename') + self.frameRate = config.getfloat(sectionName, 'framerate') + self.date = datetime.strptime(config.get(sectionName, 'date'), datetimeFormat) # 2011-06-22 11:00:39 + self.translation = literal_eval(config.get(sectionName, 'translation')) # = [0.0, 0.0] + self.rotation = config.getfloat(sectionName, 'rotation') + self.duration = config.getint(sectionName, 'duration') + except NoOptionError as e: + print(e) + print('Not a section for scene meta-data') + + @staticmethod + def loadConfigFile(filename): + from configparser import ConfigParser + config = ConfigParser() + config.readfp(openCheck(filename)) + configDict = dict() + for sectionName in config.sections(): + configDict[sectionName] = SceneParameters(config, sectionName) + return configDict + + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/storage.txt') + unittest.TextTestRunner().run(suite) +# #doctest.testmod() +# #doctest.testfile("example.txt")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/sumo.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,53 @@ +#! /usr/bin/env python +'''Libraries for the SUMO traffic simulation software +http://sumo.dlr.de +''' + +#import csv + +def loadTazEdges(inFilename): + '''Converts list of OSM edges per OSM edge and groups per TAZ + format is csv with first two columns the OSM id and TAZ id, then the list of SUMO edge id + + Returns the list of SUMO edge per TAZ''' + data = [] + tazs = {} + with open(inFilename,'r') as f: + f.readline() # skip the headers + for r in f: + tmp = r.strip().split(',') + tazID = tmp[1] + for edge in tmp[2:]: + if len(edge) > 0: + if tazID in tazs: + if edge not in tazs[tazID]: + tazs[tazID].append(edge) + else: + tazs[tazID] = [edge] + return tazs + +def edge2Taz(tazs): + '''Returns the associative array of the TAZ of each SUMO edge''' + edge2Tazs = {} + for taz, edges in tazs.items(): + for edge in edges: + if edge in edge2Tazs: + print('error for edge: {} (taz {}/{})'.format(edge, edge2Tazs[edge], taz)) + edge2Tazs[edge] = taz + return edge2Tazs + +def saveTazEdges(outFilename, tazs): + with open(outFilename,'w') as out: + out.write('<tazs>\n') + for tazID in tazs: + out.write('<taz id="{}" edges="'.format(tazID)+' '.join(tazs[tazID])+'"/>\n') + out.write('</tazs>\n') + +# TODO add utils from process-cyber.py? + +# if __name__ == "__main__": +# import doctest +# import unittest +# suite = doctest.DocFileSuite('tests/sumo.txt') +# #suite = doctest.DocTestSuite() +# unittest.TextTestRunner().run(suite)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/cvutils.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,46 @@ +>>> import cv2, cvutils +>>> from numpy import array, round, ones, dot, linalg, absolute +>>> img = cv2.imread("../samples/val-dor-117-111.png") +>>> width = img.shape[1] +>>> height = img.shape[0] +>>> intrinsicCameraMatrix = array([[ 377.42, 0. , 639.12], [ 0. , 378.43, 490.2 ], [ 0. , 0. , 1. ]]) +>>> distortionCoefficients = array([-0.11759321, 0.0148536, 0.00030756, -0.00020578, -0.00091816])# distortionCoefficients = array([-0.11759321, 0., 0., 0., 0.]) +>>> multiplicationFactor = 1.31 +>>> [map1, map2], tmp = cvutils.computeUndistortMaps(width, height, multiplicationFactor, intrinsicCameraMatrix, distortionCoefficients) +>>> undistorted = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR) +>>> (undistorted.shape == array([int(round(height*multiplicationFactor)), int(round(width*multiplicationFactor)), 3])).all() +True +>>> imgPoints = array([[[150.,170.],[220.,340.],[340.,440.],[401.,521.]]]) +>>> newCameraMatrix = cv2.getDefaultNewCameraMatrix(intrinsicCameraMatrix, (int(round(width*multiplicationFactor)), int(round(height*multiplicationFactor))), True) +>>> undistortedPoints = cv2.undistortPoints(imgPoints, intrinsicCameraMatrix, distortionCoefficients, P = newCameraMatrix).reshape(-1, 2) # undistort and project as if seen by new camera +>>> invNewCameraMatrix = linalg.inv(newCameraMatrix) +>>> tmp = ones((imgPoints[0].shape[0], 3)) +>>> tmp[:,:2] = undistortedPoints +>>> reducedPoints = dot(invNewCameraMatrix, tmp.T).T +>>> origPoints = cv2.projectPoints(reducedPoints, (0.,0.,0.), (0.,0.,0.), intrinsicCameraMatrix, distortionCoefficients)[0].reshape(-1,2) +>>> (round(origPoints[1:,:]) == imgPoints[0][1:,:]).all() +True +>>> (absolute(origPoints[0,:]-imgPoints[0][0,:])).max() < 6. +True +>>> reducedPoints2 = cvutils.newCameraProject(undistortedPoints.T, invNewCameraMatrix) +>>> (reducedPoints == reducedPoints).all() +True + +>>> undistortedPoints2 = cv2.undistortPoints(imgPoints, intrinsicCameraMatrix, distortionCoefficients).reshape(-1, 2) # undistort and project as if seen by new camera +>>> undistortedPoints2 = cvutils.newCameraProject(undistortedPoints2.T, newCameraMatrix) +>>> (undistortedPoints == undistortedPoints2.T).all() +True + +>>> undistortedPoints = cv2.undistortPoints(imgPoints, intrinsicCameraMatrix, distortionCoefficients).reshape(-1, 2) # undistort to ideal points +>>> origPoints = cvutils.worldToImageProject(undistortedPoints.T, intrinsicCameraMatrix, distortionCoefficients).T +>>> (round(origPoints[1:,:]) == imgPoints[0][1:,:]).all() +True +>>> (absolute(origPoints[0,:]-imgPoints[0][0,:])).max() < 6. +True + +>>> undistortedPoints = cvutils.imageToWorldProject(imgPoints[0].T, intrinsicCameraMatrix, distortionCoefficients) +>>> origPoints = cvutils.worldToImageProject(undistortedPoints, intrinsicCameraMatrix, distortionCoefficients).T +>>> (round(origPoints[1:,:]) == imgPoints[0][1:,:]).all() +True +>>> (absolute(origPoints[0,:]-imgPoints[0][0,:])).max() < 6. +True
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/events.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,34 @@ +>>> from events import * +>>> from moving import MovingObject, TimeInterval, Point +>>> from prediction import ConstantPredictionParameters + +>>> objects = [MovingObject(num = i, timeInterval = TimeInterval(0,10)) for i in range(10)] +>>> interactions = createInteractions(objects) +>>> len([i for i in interactions if len(i.roadUserNumbers) == 1]) +0 +>>> objects2 = [MovingObject(num = i, timeInterval = TimeInterval(0,10)) for i in range(100, 110)] +>>> interactions = createInteractions(objects, objects2) +>>> len([i for i in interactions if len(i.roadUserNumbers) == 1]) +0 + +>>> o1 = MovingObject.generate(1, Point(-5.,0.), Point(1.,0.), TimeInterval(0,10)) +>>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(0,10)) +>>> inter = Interaction(roadUser1 = o1, roadUser2 = o2) +>>> inter.computeIndicators() +>>> predictionParams = ConstantPredictionParameters(10.) +>>> inter.computeCrossingsCollisions(predictionParams, 0.1, 10) +>>> ttc = inter.getIndicator("Time to Collision") +>>> ttc[0] +5.0 +>>> ttc[1] +4.0 +>>> (inter.collisionPoints[0][0] - Point(0.,0.)).norm2() < 0.0001 +True +>>> (inter.collisionPoints[4][0] - Point(0.,0.)).norm2() < 0.0001 +True +>>> inter.getIndicator(Interaction.indicatorNames[1])[4] < 0.000001 # collision angle +True +>>> inter.getIndicator(Interaction.indicatorNames[1])[5] is None +True +>>> inter.getIndicator(Interaction.indicatorNames[1])[6] # doctest:+ELLIPSIS +3.1415...
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/indicators.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,34 @@ +>>> from indicators import * +>>> from moving import TimeInterval,Trajectory + +>>> indic1 = TemporalIndicator('bla', [0,3,-4], TimeInterval(4,6)) +>>> indic1.empty() +False +>>> indic1.getIthValue(1) +3 +>>> indic1.getIthValue(3) +>>> indic1[6] +-4 +>>> indic1[7] +>>> [v for v in indic1] +[0, 3, -4] +>>> indic1 = TemporalIndicator('bla', {2:0,4:3,5:-5}) +>>> indic1.getIthValue(1) +3 +>>> indic1.getIthValue(3) +>>> indic1[2] +0 + +>>> t1 = Trajectory([[0.5,1.5,2.5],[0.5,3.5,6.5]]) +>>> m = indicatorMap([1,2,3], t1, 1) +>>> m[(1.0, 3.0)] +2.0 +>>> m[(2.0, 6.0)] +3.0 +>>> m[(0.0, 0.0)] +1.0 +>>> m = indicatorMap([1,2,3], t1, 4) +>>> m[(0.0, 1.0)] +3.0 +>>> m[(0.0, 0.0)] +1.5
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/ml.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,11 @@ +>>> from math import fabs +>>> from numpy import ones +>>> from ml import prototypeCluster + +>>> nTrajectories = 7 +>>> similarityFunc = lambda x, y: 1.-fabs(x-y)/(nTrajectories-1) +>>> similarities = -ones((nTrajectories, nTrajectories)) +>>> prototypeIndices = prototypeCluster(range(nTrajectories), similarities, 1., similarityFunc, optimizeCentroid = True) # too large to be similar +>>> len(prototypeIndices) == nTrajectories +True +>>> # could use lists to have a length
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/moving.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,268 @@ +>>> from moving import * +>>> import storage +>>> import numpy as np + +>>> Interval().empty() +True +>>> Interval(0,1).empty() +False +>>> Interval(0,1) +[0, 1] +>>> Interval(0,1).length() +1.0 +>>> Interval(23.2,24.9).length() +1.6999999999999993 +>>> Interval(10,8).length() +0.0 + +>>> TimeInterval(0,1).length() +2.0 +>>> TimeInterval(10,8).length() +0.0 +>>> TimeInterval(10,8) == TimeInterval(10,8) +True +>>> TimeInterval(10,8) == TimeInterval(8,10) +True +>>> TimeInterval(11,8) == TimeInterval(10,8) +False + +>>> [i for i in TimeInterval(9,13)] +[9, 10, 11, 12, 13] + +>>> TimeInterval(2,5).equal(TimeInterval(2,5)) +True +>>> TimeInterval(2,5).equal(TimeInterval(2,4)) +False +>>> TimeInterval(2,5).equal(TimeInterval(5,2)) +False + +>>> TimeInterval(3,6).distance(TimeInterval(4,6)) +0 +>>> TimeInterval(3,6).distance(TimeInterval(6,10)) +0 +>>> TimeInterval(3,6).distance(TimeInterval(8,10)) +2 +>>> TimeInterval(20,30).distance(TimeInterval(3,15)) +5 +>>> TimeInterval.unionIntervals([TimeInterval(3,6), TimeInterval(8,10),TimeInterval(11,15)]) +[3, 15] + +>>> Point(0,3) == Point(0,3) +True +>>> Point(0,3) == Point(0,3.2) +False +>>> Point(3,4)-Point(1,7) +(2.000000,-3.000000) +>>> -Point(1,2) +(-1.000000,-2.000000) +>>> Point(1,2)*0.5 +(0.500000,1.000000) + +>>> Point(3,2).norm2Squared() +13 + +>>> Point.distanceNorm2(Point(3,4),Point(1,7)) +3.605551275463989 + +>>> Point(3,2).inPolygon(np.array([[0,0],[1,0],[1,1],[0,1]])) +False +>>> Point(3,2).inPolygon(np.array([[0,0],[4,0],[4,3],[0,3]])) +True + +>>> predictPositionNoLimit(10, Point(0,0), Point(1,1)) # doctest:+ELLIPSIS +((1.0...,1.0...), (10.0...,10.0...)) + +>>> segmentIntersection(Point(0,0), Point(0,1), Point(1,1), Point(2,3)) +>>> segmentIntersection(Point(0,1), Point(0,3), Point(1,0), Point(3,1)) +>>> segmentIntersection(Point(0.,0.), Point(2.,2.), Point(0.,2.), Point(2.,0.)) +(1.000000,1.000000) +>>> segmentIntersection(Point(0,0), Point(4,4), Point(0,4), Point(4,0)) +(2.000000,2.000000) +>>> segmentIntersection(Point(0,1), Point(1,2), Point(2,0), Point(3,2)) + +>>> t1 = Trajectory.fromPointList([(92.2, 102.9), (56.7, 69.6)]) +>>> t2 = Trajectory.fromPointList([(92.2, 102.9), (56.7, 69.6)]) +>>> t1 == t2 +True +>>> t3 = Trajectory.fromPointList([(92.24, 102.9), (56.7, 69.6)]) +>>> t1 == t3 +False +>>> t3 = Trajectory.fromPointList([(92.2, 102.9), (56.7, 69.6), (56.7, 69.6)]) +>>> t1 == t3 +False + +>>> left = Trajectory.fromPointList([(92.291666666666686, 102.99239033124439), (56.774193548387103, 69.688898836168306)]) +>>> middle = Trajectory.fromPointList([(87.211021505376351, 93.390778871978512), (59.032258064516128, 67.540286481647257)]) +>>> right = Trajectory.fromPointList([(118.82392473118281, 115.68263205013426), (63.172043010752688, 66.600268576544309)]) +>>> alignments = [left, middle, right] +>>> for a in alignments: a.computeCumulativeDistances() +>>> getSYfromXY(Point(73, 82), alignments) +[1, 0, (73.819977,81.106170), 18.172277808821125, 18.172277808821125, 1.2129694042343868] +>>> getSYfromXY(Point(78, 83), alignments, 0.5) +[1, 0, (77.033188,84.053889), 13.811799123113715, 13.811799123113715, -1.4301775140225983] + +>>> Trajectory().length() +0 +>>> t1 = Trajectory([[0.5,1.5,2.5],[0.5,3.5,6.5]]) +>>> t1.length() == 3. +True +>>> t1[1] +(1.500000,3.500000) + +>>> t1.differentiate() +(1.000000,3.000000) (1.000000,3.000000) +>>> t1.differentiate(True) +(1.000000,3.000000) (1.000000,3.000000) (1.000000,3.000000) +>>> t1 = Trajectory([[0.5,1.5,3.5],[0.5,2.5,7.5]]) +>>> t1.differentiate() +(1.000000,2.000000) (2.000000,5.000000) + +>>> t1.computeCumulativeDistances() +>>> t1.getDistance(0) +2.23606797749979 +>>> t1.getDistance(1) +5.385164807134504 +>>> t1.getDistance(2) +Index 2 beyond trajectory length 3-1 +>>> t1.getCumulativeDistance(0) +0.0 +>>> t1.getCumulativeDistance(1) +2.23606797749979 +>>> t1.getCumulativeDistance(2) +7.6212327846342935 +>>> t1.getCumulativeDistance(3) +Index 3 beyond trajectory length 3 + + +>>> from utils import LCSS +>>> lcss = LCSS(lambda x,y: Point.distanceNorm2(x,y) <= 0.1) +>>> Trajectory.lcss(t1, t1, lcss) +3 +>>> lcss = LCSS(lambda p1, p2: (p1-p2).normMax() <= 0.1) +>>> Trajectory.lcss(t1, t1, lcss) +3 + +>>> p1=Point(0,0) +>>> p2=Point(1,0) +>>> v1 = Point(0.1,0.1) +>>> v2 = Point(-0.1, 0.1) +>>> abs(Point.timeToCollision(p1, p2, v1, v2, 0.)-5.0) < 0.00001 +True +>>> abs(Point.timeToCollision(p1, p2, v1, v2, 0.1)-4.5) < 0.00001 +True +>>> p1=Point(0,1) +>>> p2=Point(1,0) +>>> v1 = Point(0,0.1) +>>> v2 = Point(0.1, 0) +>>> Point.timeToCollision(p1, p2, v1, v2, 0.) == None +True +>>> Point.timeToCollision(p2, p1, v2, v1, 0.) == None +True +>>> Point.midPoint(p1, p2) +(0.500000,0.500000) +>>> p1=Point(0.,0.) +>>> p2=Point(5.,0.) +>>> v1 = Point(2.,0.) +>>> v2 = Point(1.,0.) +>>> Point.timeToCollision(p1, p2, v1, v2, 0.) +5.0 +>>> Point.timeToCollision(p1, p2, v1, v2, 1.) +4.0 + +>>> objects = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'object') +>>> len(objects) +5 +>>> objects[0].hasFeatures() +False +>>> features = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'feature') +>>> for o in objects: o.setFeatures(features) +>>> objects[0].hasFeatures() +True + +>>> o1 = MovingObject.generate(1, Point(-5.,0.), Point(1.,0.), TimeInterval(0,10)) +>>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(0,10)) +>>> MovingObject.computePET(o1, o2, 0.1) +(0.0, 5, 5) +>>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(5,15)) +>>> MovingObject.computePET(o1, o2, 0.1) +(5.0, 5, 10) +>>> o2 = MovingObject.generate(2, Point(0.,-5.), Point(0.,1.), TimeInterval(15,30)) +>>> MovingObject.computePET(o1, o2, 0.1) +(15.0, 5, 20) + +>>> t = CurvilinearTrajectory(S = [1., 2., 3., 5.], Y = [0.5, 0.5, 0.6, 0.7], lanes = ['1']*4) +>>> t.differentiate() # doctest:+ELLIPSIS +[1.0, 0.0, '1'] [1.0, 0.099..., '1'] [2.0, 0.099..., '1'] +>>> t.differentiate(True) # doctest:+ELLIPSIS +[1.0, 0.0, '1'] [1.0, 0.099..., '1'] [2.0, 0.099..., '1'] [2.0, 0.099..., '1'] +>>> t = CurvilinearTrajectory(S = [1.], Y = [0.5], lanes = ['1']) +>>> t.differentiate().empty() +True + +>>> o1 = MovingObject.generate(1, Point(1., 2.), Point(1., 1.), TimeInterval(0,10)) +>>> o1.features = [o1] +>>> o2 = MovingObject.generate(2, Point(14., 14.), Point(1., 0.), TimeInterval(14,20)) +>>> o2.features = [o2] +>>> o3 = MovingObject.generate(3, Point(2., 2.), Point(1., 1.), TimeInterval(2,12)) +>>> o3.features = [o3] +>>> o13 = MovingObject.concatenate(o1, o3, 4) +>>> o13.getNum() +4 +>>> o13.getTimeInterval() == TimeInterval(0,12) +True +>>> t=5 +>>> o13.getPositionAtInstant(t) == (o1.getPositionAtInstant(t)+o3.getPositionAtInstant(t)).divide(2) +True +>>> len(o13.getFeatures()) +2 +>>> o12 = MovingObject.concatenate(o1, o2, 5) +>>> o12.getTimeInterval() == TimeInterval(o1.getFirstInstant(), o2.getLastInstant()) +True +>>> v = o12.getVelocityAtInstant(12) +>>> v == Point(3./4, 2./4) +True +>>> o12.getPositionAtInstant(11) == o1.getPositionAtInstant(10)+v +True +>>> len(o12.getFeatures()) +3 + +>>> o1 = MovingObject.generate(1, Point(0., 2.), Point(0., 1.), TimeInterval(0,2)) +>>> o1.classifyUserTypeSpeedMotorized(0.5, np.median) +>>> userTypeNames[o1.getUserType()] +'car' +>>> o1.classifyUserTypeSpeedMotorized(1.5, np.median) +>>> userTypeNames[o1.getUserType()] +'pedestrian' + +>>> o1 = MovingObject.generate(1, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) +>>> gt1 = BBMovingObject(1, TimeInterval(0,10), MovingObject.generate(1, Point(0.2,0.6), Point(1.,0.), TimeInterval(0,10)), MovingObject.generate(2, Point(-0.2,-0.4), Point(1.,0.), TimeInterval(0,10))) +>>> gt1.computeCentroidTrajectory() +>>> computeClearMOT([gt1], [], 0.2, 0, 10) +(None, 0.0, 11, 0, 0, 11, None, None) +>>> computeClearMOT([], [o1], 0.2, 0, 10) +(None, None, 0, 0, 11, 0, None, None) +>>> computeClearMOT([gt1], [o1], 0.2, 0, 10) # doctest:+ELLIPSIS +(0.0999..., 1.0, 0, 0, 0, 11, None, None) +>>> computeClearMOT([gt1], [o1], 0.05, 0, 10) +(None, -1.0, 11, 0, 11, 11, None, None) + +>>> o1 = MovingObject(1, TimeInterval(0,3), positions = Trajectory([list(range(4)), [0.1, 0.1, 1.1, 1.1]])) +>>> o2 = MovingObject(2, TimeInterval(0,3), positions = Trajectory([list(range(4)), [0.9, 0.9, -0.1, -0.1]])) +>>> gt1 = BBMovingObject(1, TimeInterval(0,3), MovingObject(positions = Trajectory([list(range(4)), [0.]*4])), MovingObject(positions = Trajectory([list(range(4)), [0.]*4]))) +>>> gt1.computeCentroidTrajectory() +>>> gt2 = BBMovingObject(2, TimeInterval(0,3), MovingObject(positions = Trajectory([list(range(4)), [1.]*4])), MovingObject(positions = Trajectory([list(range(4)), [1.]*4]))) +>>> gt2.computeCentroidTrajectory() +>>> computeClearMOT([gt1, gt2], [o1, o2], 0.2, 0, 3) # doctest:+ELLIPSIS +(0.1..., 0.75, 0, 2, 0, 8, None, None) +>>> computeClearMOT([gt2, gt1], [o2, o1], 0.2, 0, 3) # doctest:+ELLIPSIS +(0.1..., 0.75, 0, 2, 0, 8, None, None) +>>> computeClearMOT([gt1], [o1, o2], 0.2, 0, 3) +(0.1, -0.25, 0, 1, 4, 4, None, None) +>>> computeClearMOT([gt1], [o2, o1], 0.2, 0, 3) # symmetry +(0.1, -0.25, 0, 1, 4, 4, None, None) +>>> computeClearMOT([gt1, gt2], [o1], 0.2, 0, 3) # doctest:+ELLIPSIS +(0.100..., 0.375, 4, 1, 0, 8, None, None) +>>> computeClearMOT([gt2, gt1], [o1], 0.2, 0, 3) # doctest:+ELLIPSIS +(0.100..., 0.375, 4, 1, 0, 8, None, None) +>>> computeClearMOT([gt1, gt2], [o1, o2], 0.08, 0, 3) +(None, -1.0, 8, 0, 8, 8, None, None)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/moving_shapely.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,28 @@ +>>> from moving import * +>>> from shapely.geometry import Polygon +>>> from shapely.prepared import prep + +>>> t1 = Trajectory([[0.5,1.5,2.5],[0.5,3.5,6.5]]) +>>> poly = Polygon([[0,0],[4,0],[4,3],[0,3]]) +>>> sub1, sub2 = t1.getTrajectoryInPolygon(poly) +>>> sub1 +(0.500000,0.500000) +>>> sub1, sub2 = t1.getTrajectoryInPolygon(Polygon([[10,10],[14,10],[14,13],[10,13]])) +>>> sub1.length() +0 +>>> sub1, sub2 = t1.getTrajectoryInPolygon(prep(poly)) +>>> sub1 +(0.500000,0.500000) +>>> t2 = t1.differentiate(True) +>>> sub1, sub2 = t1.getTrajectoryInPolygon(prep(poly), t2) +>>> sub1.length() == sub2.length() +True +>>> sub1 +(0.500000,0.500000) +>>> sub2 +(1.000000,3.000000) + +>>> t1.proportionInPolygon(poly, 0.5) +False +>>> t1.proportionInPolygon(poly, 0.3) +True
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/prediction.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,71 @@ +>>> from prediction import * +>>> import moving, storage, utils +>>> from numpy import absolute, array + +>>> et = PredictedTrajectoryConstant(moving.Point(0,0), moving.Point(1,0)) +>>> et.predictPosition(4) # doctest:+ELLIPSIS +(4.0...,0.0...) +>>> et.predictPosition(1) # doctest:+ELLIPSIS +(1.0...,0.0...) + +>>> et = PredictedTrajectoryConstant(moving.Point(0,0), moving.Point(1,0), moving.NormAngle(0.1,0), maxSpeed = 2) +>>> et.predictPosition(10) # doctest:+ELLIPSIS +(15.5...,0.0...) +>>> et.predictPosition(11) # doctest:+ELLIPSIS +(17.5...,0.0...) +>>> et.predictPosition(12) # doctest:+ELLIPSIS +(19.5...,0.0...) + +>>> import random +>>> acceleration = lambda: random.uniform(-0.5,0.5) +>>> steering = lambda: random.uniform(-0.1,0.1) +>>> et = PredictedTrajectoryRandomControl(moving.Point(0,0),moving.Point(1,1), acceleration, steering, maxSpeed = 2) +>>> p = et.predictPosition(500) +>>> from numpy import max +>>> max(et.getPredictedSpeeds()) <= 2. +True + +>>> p = moving.Point(3,4) +>>> sp = SafetyPoint(p, 0.1, 0) +>>> print(sp) +3 4 0.1 0 + +>>> et1 = PredictedTrajectoryConstant(moving.Point(-5.,0.), moving.Point(1.,0.)) +>>> et2 = PredictedTrajectoryConstant(moving.Point(0.,-5.), moving.Point(0.,1.)) +>>> collision, t, cp1, cp2 = computeCollisionTime(et1, et2, 0.1, 10) +>>> collision +True +>>> t +5 +>>> collision, t, cp1, cp2 = computeCollisionTime(et1, et2, 0.1, 5) +>>> collision +True +>>> t +5 +>>> collision, t, cp1, cp2 = computeCollisionTime(et1, et2, 0.1, 4) +>>> collision +False + +>>> proto = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'feature', [1204])[0] +>>> proto.getPositions().computeCumulativeDistances() +>>> et = PredictedTrajectoryPrototype(proto.getPositionAt(10)+moving.Point(0.5, 0.5), proto.getVelocityAt(10)*0.9, proto, True) +>>> absolute(et.initialSpeed - proto.getVelocityAt(10).norm2()*0.9) < 1e-5 +True +>>> for t in range(int(proto.length())): x=et.predictPosition(t) +>>> traj = et.getPredictedTrajectory() +>>> traj.computeCumulativeDistances() +>>> absolute(array(traj.distances).mean() - et.initialSpeed < 1e-3) +True + +>>> et = PredictedTrajectoryPrototype(proto.getPositionAt(10)+moving.Point(0.6, 0.6), proto.getVelocityAt(10)*0.7, proto, False) +>>> absolute(et.initialSpeed - proto.getVelocityAt(10).norm2()*0.7) < 1e-5 +True +>>> proto = moving.MovingObject.generate(1, moving.Point(-5.,0.), moving.Point(1.,0.), moving.TimeInterval(0,10)) +>>> et = PredictedTrajectoryPrototype(proto.getPositionAt(0)+moving.Point(0., 1.), proto.getVelocityAt(0)*0.5, proto, False) +>>> for t in range(int(proto.length()/0.5)): x=et.predictPosition(t) +>>> et.predictPosition(10) # doctest:+ELLIPSIS +(0.0...,1.0...) +>>> et.predictPosition(12) # doctest:+ELLIPSIS +(1.0...,1.0...) + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/storage.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,128 @@ +>>> from storage import * +>>> from io import StringIO +>>> from moving import MovingObject, Point, TimeInterval, Trajectory, prepareSplines + +>>> f = openCheck('non_existant_file.txt') +File non_existant_file.txt could not be opened. + +>>> nonexistentFilename = "nonexistent" +>>> loadTrajectoriesFromSqlite(nonexistentFilename, 'feature') +DB Error: no such table: positions +DB Error: no such table: velocities +[] +>>> from os import remove +>>> remove(nonexistentFilename) + +>>> o1 = MovingObject.generate(2, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) +>>> o2 = MovingObject.generate(3, Point(1.,1.), Point(-0.5,-0.2), TimeInterval(0,9)) +>>> saveTrajectoriesToSqlite('test.sqlite', [o1, o2], 'feature') +>>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'feature') +>>> objects[0].getNum() == o1.num +True +>>> objects[1].getNum() == o2.num +True +>>> o1.getTimeInterval() == objects[0].getTimeInterval() +True +>>> o2.getTimeInterval() == objects[1].getTimeInterval() +True +>>> o1.getVelocities().length() == objects[0].getVelocities().length() +True +>>> o2.getVelocities().length() == objects[1].getVelocities().length() +True +>>> o1.getVelocities() == objects[0].getVelocities() +True +>>> o2.getVelocities() == objects[1].getVelocities() +True +>>> o1.getPositions() == objects[0].getPositions() +True +>>> o2.getPositions() == objects[1].getPositions() +True +>>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'feature', timeStep = 2) +>>> objects[0].positions.length() +6 +>>> objects[1].positions.length() +5 +>>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'feature', timeStep = 3) +>>> objects[0].positions.length() +4 +>>> objects[1].positions.length() +4 +>>> align1 = Trajectory.fromPointList([Point(-1, 0), Point(20, 0)]) +>>> align2 = Trajectory.fromPointList([Point(-9, -3), Point(6, 3)]) +>>> align1.computeCumulativeDistances() +>>> align2.computeCumulativeDistances() +>>> prepareSplines([align1, align2]) +>>> o1.projectCurvilinear([align1, align2]) +>>> o2.projectCurvilinear([align1, align2]) +>>> saveTrajectoriesToSqlite('test.sqlite', [o1, o2], 'curvilinear') +>>> addCurvilinearTrajectoriesFromSqlite('test.sqlite', {o.num: o for o in objects}) +>>> o1.curvilinearPositions[3][:2] == objects[0].curvilinearPositions[3][:2] +True +>>> o1.curvilinearPositions[7][:2] == objects[0].curvilinearPositions[7][:2] +True +>>> [str(l) for l in o1.curvilinearPositions.getLanes()] == objects[0].curvilinearPositions.getLanes() +True +>>> o2.curvilinearPositions[2][:2] == objects[1].curvilinearPositions[2][:2] +True +>>> o2.curvilinearPositions[6][:2] == objects[1].curvilinearPositions[6][:2] +True +>>> [str(l) for l in o2.curvilinearPositions.getLanes()] == objects[1].curvilinearPositions.getLanes() +True +>>> remove('test.sqlite') + +>>> f1 = MovingObject.generate(3, Point(0.,0.), Point(1.,0.), TimeInterval(0,10)) +>>> f2 = MovingObject.generate(4, Point(1.,1.), Point(-0.5,-0.2), TimeInterval(0,9)) +>>> o1 = MovingObject(num = 1, userType = 1) +>>> o1.features = [f1, f2] +>>> saveTrajectoriesToSqlite('test.sqlite', [o1], 'object') +>>> objects = loadTrajectoriesFromSqlite('test.sqlite', 'object', withFeatures = True) +>>> len(objects) +1 +>>> reloaded1 = objects[0] +>>> reloaded1.getNum() == o1.getNum() +True +>>> reloaded1.getUserType() == o1.getUserType() +True +>>> len(reloaded1.featureNumbers) +2 +>>> len(reloaded1.features) +2 +>>> reloaded1.getPositionAt(0) == Point.midPoint(f1.getPositionAt(0), f2.getPositionAt(0)) +True +>>> reloaded1.getPositionAt(5) == Point.midPoint(f1.getPositionAt(5), f2.getPositionAt(5)) +True +>>> reloaded1.getPositionAt(10) == f1.getPositionAt(10) +True +>>> set(reloaded1.featureNumbers) == set([f1.num, f2.num]) +True +>>> remove('test.sqlite') + +>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') +>>> readline(strio) +'sadlkfjsdlakjf' +>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') +>>> readline(strio, ['#']) +'sadlkfjsdlakjf' +>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') +>>> readline(strio, ['%']) +'# asdlfjasdlkj0' +>>> strio = StringIO('# asdlfjasdlkj0\nsadlkfjsdlakjf') +>>> readline(strio, '%*$') +'# asdlfjasdlkj0' +>>> readline(strio, '%#') +'sadlkfjsdlakjf' + +>>> from sklearn.mixture import GaussianMixture +>>> from numpy.random import random_sample +>>> nPoints = 50 +>>> points = random_sample(nPoints*2).reshape(nPoints,2) +>>> gmm = GaussianMixture(4, covariance_type = 'full') +>>> tmp = gmm.fit(points) +>>> gmmId = 0 +>>> savePOIsToSqlite('pois-tmp.sqlite', gmm, 'end', gmmId) +>>> reloadedGmm = loadPOIsFromSqlite('pois-tmp.sqlite') +>>> sum(gmm.predict(points) == reloadedGmm[gmmId].predict(points)) == nPoints +True +>>> reloadedGmm[gmmId].gmmTypes[0] == 'end' +True +>>> remove('pois-tmp.sqlite')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/tutorials.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,39 @@ +import unittest + +class TestNGSIM(unittest.TestCase): + 'Tutorial example for NGSIM data' + + def test_ex1(self): + from trafficintelligence import storage + objects = storage.loadTrajectoriesFromNgsimFile('../samples/trajectories-0400-0415.txt',100) + for o in objects: o.plot() + +class TestTrajectoryLoading(unittest.TestCase): + 'Tutorial example for NGSIM data' + + def test_ex1(self): + from trafficintelligence import storage + objects = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'object') + + speed = objects[0].getVelocityAtInstant(10).norm2() + timeInterval = objects[0].getTimeInterval() + speeds = [objects[0].getVelocityAtInstant(t).norm2() for t in range(timeInterval.first, timeInterval.last)] + speeds = [v.norm2() for v in objects[0].getVelocities()] + + from matplotlib.pyplot import plot, close, axis + plot(range(timeInterval.first, timeInterval.last+1), speeds) + + close('all') + objects[0].plot() + axis('equal') + + features = storage.loadTrajectoriesFromSqlite('../samples/laurier.sqlite', 'feature') + objects[0].setFeatures(features) + + for f in objects[0].features: + f.plot() + axis('equal') + + +if __name__ == '__main__': + unittest.main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/tests/utils.txt Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,144 @@ +>>> from utils import * +>>> from moving import Point + +>>> upperCaseFirstLetter('mmmm... donuts') +'Mmmm... Donuts' +>>> s = upperCaseFirstLetter('much ado about nothing') +>>> s == 'Much Ado About Nothing' +True +>>> upperCaseFirstLetter(s) == s +True + +>>> computeChi2([],[]) +0 +>>> computeChi2(list(range(1,10)),list(range(1,10))) +0.0 +>>> computeChi2(list(range(1,9)),list(range(1,10))) +0.0 + +>>> ceilDecimals(1.23, 0) +2.0 +>>> ceilDecimals(1.23, 1) +1.3 + +>>> inBetween(1,2,1.5) +True +>>> inBetween(2.1,1,1.5) +True +>>> inBetween(1,2,0) +False + +>>> removeExtension('test-adfasdf.asdfa.txt') +'test-adfasdf.asdfa' +>>> removeExtension('test-adfasdf') +'test-adfasdf' + +>>> values = line2Ints('1 2 3 5 6') +>>> values[0] +1 +>>> values[-1] +6 +>>> values = line2Floats('1.3 2.45 7.158e+01 5 6') +>>> values[0] +1.3 +>>> values[2] #doctest: +ELLIPSIS +71.5... +>>> values[-1] +6.0 + +>>> stepPlot([3, 5, 7, 8], 1, 10, 0) +([1, 3, 3, 5, 5, 7, 7, 8, 8, 10], [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]) + +>>> mostCommon(['a','b','c','b']) +'b' +>>> mostCommon(['a','b','c','b', 'c']) +'b' +>>> mostCommon(list(range(10))+[1]) +1 +>>> mostCommon([list(range(2)), list(range(4)), list(range(2))]) +[0, 1] + +>>> res = sortByLength([list(range(3)), list(range(4)), list(range(1))]) +>>> [len(r) for r in res] +[1, 3, 4] +>>> res = sortByLength([list(range(3)), list(range(4)), list(range(1)), list(range(5))], reverse = True) +>>> [len(r) for r in res] +[5, 4, 3, 1] + +>>> lcss = LCSS(similarityFunc = lambda x,y: abs(x-y) <= 0.1) +>>> lcss.compute(list(range(5)), list(range(5))) +5 +>>> lcss.compute(list(range(1,5)), list(range(5))) +4 +>>> lcss.compute(list(range(5,10)), list(range(5))) +0 +>>> lcss.compute(list(range(5)), list(range(10))) +5 +>>> lcss.similarityFunc = lambda x,y: x == y +>>> lcss.compute(['a','b','c'], ['a','b','c', 'd']) +3 +>>> lcss.computeNormalized(['a','b','c'], ['a','b','c', 'd']) #doctest: +ELLIPSIS +1.0 +>>> lcss.computeNormalized(['a','b','c','x'], ['a','b','c', 'd']) #doctest: +ELLIPSIS +0.75 +>>> lcss.compute(['a','b','c'], ['a','b','c', 'd']) +3 +>>> lcss.compute(['a','x','b','c'], ['a','b','c','d','x']) +3 +>>> lcss.compute(['a','b','c','x','d'], ['a','b','c','d','x']) +4 +>>> lcss.delta = 1 +>>> lcss.compute(['a','b','c'], ['a','b','x','x','c']) +2 + +>>> lcss.delta = float('inf') +>>> lcss.compute(['a','b','c'], ['a','b','c', 'd'], computeSubSequence = True) +3 +>>> lcss.subSequenceIndices +[(0, 0), (1, 1), (2, 2)] +>>> lcss.compute(['a','b','c'], ['x','a','b','c'], computeSubSequence = True) +3 +>>> lcss.subSequenceIndices +[(0, 1), (1, 2), (2, 3)] +>>> lcss.compute(['a','g','b','c'], ['a','b','c', 'd'], computeSubSequence = True) +3 +>>> lcss.subSequenceIndices +[(0, 0), (2, 1), (3, 2)] + +>>> alignedLcss = LCSS(lambda x,y:(abs(x-y) <= 0.1), delta = 2, aligned = True) +>>> alignedLcss.compute(list(range(5)), list(range(5))) +5 +>>> alignedLcss.compute(list(range(1,5)), list(range(5))) +4 + +>>> alignedLcss.compute(list(range(5,10)), list(range(10))) +5 + +>>> lcss.delta = 2 +>>> lcss.compute(list(range(5,10)), list(range(10))) +0 +>>> alignedLcss.delta = 6 +>>> alignedLcss.compute(list(range(5)), list(range(5))) +5 +>>> alignedLcss.compute(list(range(5)), list(range(6))) +5 +>>> lcss.delta = 10 +>>> alignedLcss.compute(list(range(1,7)), list(range(6))) +5 +>>> lcss = LCSS(lambda x,y: x == y, delta = 2, aligned = True) +>>> lcss.compute(list(range(20)), [2,4,6,7,8,9,11,13], True) +8 +>>> lcss.subSequenceIndices +[(2, 0), (4, 1), (6, 2), (7, 3), (8, 4), (9, 5), (11, 6), (13, 7)] + +>>> lcss = LCSS(metric = 'cityblock', epsilon = 0.1) +>>> lcss.compute([[i] for i in range(5)], [[i] for i in range(5)]) +5 +>>> lcss.compute([[i] for i in range(1,5)], [[i] for i in range(5)]) +4 +>>> lcss.compute([[i] for i in range(5,10)], [[i] for i in range(5)]) +0 +>>> lcss.compute([[i] for i in range(5)], [[i] for i in range(10)]) +5 + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/traffic_engineering.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,337 @@ +#! /usr/bin/env python +''' Traffic Engineering Tools and Examples''' + +from trafficintelligence import prediction + +from math import ceil + + +######################### +# Simulation +######################### + +def generateTimeHeadways(meanTimeHeadway, simulationTime): + '''Generates the time headways between arrivals + given the meanTimeHeadway and the negative exponential distribution + over a time interval of length simulationTime (assumed to be in same time unit as headway''' + from random import expovariate + headways = [] + totalTime = 0 + flow = 1/meanTimeHeadway + while totalTime < simulationTime: + h = expovariate(flow) + headways.append(h) + totalTime += h + return headways + +class RoadUser(object): + '''Simple example of inheritance to plot different road users ''' + def __init__(self, position, velocity): + 'Both fields are 2D numpy arrays' + self.position = position.astype(float) + self.velocity = velocity.astype(float) + + def move(self, deltaT): + self.position += deltaT*self.velocity + + def draw(self, init = False): + from matplotlib.pyplot import plot + if init: + self.plotLine = plot(self.position[0], self.position[1], self.getDescriptor())[0] + else: + self.plotLine.set_data(self.position[0], self.position[1]) + + +class PassengerVehicle(RoadUser): + def getDescriptor(self): + return 'dr' + +class Pedestrian(RoadUser): + def getDescriptor(self): + return 'xb' + +class Cyclist(RoadUser): + def getDescriptor(self): + return 'og' + +######################### +# queueing models +######################### + +class CapacityReduction(object): + def __init__(self, beta, reductionDuration, demandCapacityRatio = None, demand = None, capacity = None): + '''reduction duration should be positive + demandCapacityRatio is demand/capacity (q/s)''' + if demandCapacityRatio is None and demand is None and capacity is None: + print('Missing too much information (demand, capacity and ratio)') + import sys + sys.exit() + if 0 <= beta < 1: + self.beta = beta + self.reductionDuration = reductionDuration + + if demandCapacityRatio is not None: + self.demandCapacityRatio = demandCapacityRatio + if demand is not None: + self.demand = demand + if capacity is not None: + self.capacity = capacity + if capacity is not None and demand is not None: + self.demandCapacityRatio = float(self.demand)/self.capacity + if demand <= beta*capacity: + print('There is no queueing as the demand {} is inferior to the reduced capacity {}'.format(demand, beta*capacity)) + else: + print('reduction coefficient (beta={}) is not in [0, 1['.format(beta)) + + def queueingDuration(self): + return self.reductionDuration*(1-self.beta)/(1-self.demandCapacityRatio) + + def nArrived(self, t): + if self.demand is None: + print('Missing demand field') + return None + return self.demand*t + + def nServed(self, t): + if self.capacity is None: + print('Missing capacity field') + return None + if 0<=t<=self.reductionDuration: + return self.beta*self.capacity*t + elif self.reductionDuration < t <= self.queueingDuration(): + return self.beta*self.capacity*self.reductionDuration+self.capacity*(t-self.reductionDuration) + + def nQueued(self, t): + return self.nArrived(t)-self.nServed(t) + + def maxNQueued(self): + return self.nQueued(self.reductionDuration) + + def totalDelay(self): + if self.capacity is None: + print('Missing capacity field') + return None + return self.capacity*self.reductionDuration**2*(1-self.beta)*(self.demandCapacityRatio-self.beta)/(2*(1-self.demandCapacityRatio)) + + def averageDelay(self): + return self.reductionDuration*(self.demandCapacityRatio-self.beta)/(2*self.demandCapacityRatio) + + def averageNQueued(self): + return self.totalDelay()/self.queueingDuration() + + +######################### +# fundamental diagram +######################### + +class FundamentalDiagram(object): + ''' ''' + def __init__(self, name): + self.name = name + + def q(self, k): + return k*self.v(k) + + @staticmethod + def meanHeadway(k): + return 1/k + + @staticmethod + def meanSpacing(q): + return 1/q + + def plotVK(self, language='fr', units={}): + from numpy import arange + from matplotlib.pyplot import figure,plot,xlabel,ylabel + densities = [k for k in arange(1, self.kj+1)] + figure() + plot(densities, [self.v(k) for k in densities]) + xlabel('Densite (veh/km)') # todo other languages and adapt to units + ylabel('Vitesse (km/h)') + + def plotQK(self, language='fr', units={}): + from numpy import arange + from matplotlib.pyplot import figure,plot,xlabel,ylabel + densities = [k for k in arange(1, self.kj+1)] + figure() + plot(densities, [self.q(k) for k in densities]) + xlabel('Densite (veh/km)') # todo other languages and adapt to units + ylabel('Debit (km/h)') + +class GreenbergFD(FundamentalDiagram): + '''Speed is the logarithm of density''' + def __init__(self, vc, kj): + FundamentalDiagram.__init__(self,'Greenberg') + self.vc=vc + self.kj=kj + + def v(self,k): + from numpy import log + return self.vc*log(self.kj/k) + + def criticalDensity(self): + from numpy import e + self.kc = self.kj/e + return self.kc + + def capacity(self): + self.qmax = self.kc*self.vc + return self.qmax + +######################### +# intersection +######################### + +class FourWayIntersection(object): + '''Simple class for simple intersection outline''' + def __init__(self, dimension, coordX, coordY): + self.dimension = dimension + self.coordX = coordX + self.coordY = coordY + + def plot(self, options = 'k'): + from matplotlib.pyplot import plot, axis + + minX = min(self.dimension[0]) + maxX = max(self.dimension[0]) + minY = min(self.dimension[1]) + maxY = max(self.dimension[1]) + + plot([minX, self.coordX[0], self.coordX[0]], [self.coordY[0], self.coordY[0], minY],options) + plot([self.coordX[1], self.coordX[1], maxX], [minY, self.coordY[0], self.coordY[0]],options) + plot([minX, self.coordX[0], self.coordX[0]], [self.coordY[1], self.coordY[1], maxY],options) + plot([self.coordX[1], self.coordX[1], maxX], [maxY, self.coordY[1], self.coordY[1]],options) + axis('equal') + +######################### +# traffic signals +######################### + +class Volume(object): + '''Class to represent volumes with varied vehicule types ''' + def __init__(self, volume, types = ['pc'], proportions = [1], equivalents = [1], nLanes = 1): + '''mvtEquivalent is the equivalent if the movement is right of left turn''' + + # check the sizes of the lists + if sum(proportions) == 1: + self.volume = volume + self.types = types + self.proportions = proportions + self.equivalents = equivalents + self.nLanes = nLanes + else: + print('Proportions do not sum to 1') + pass + + def checkProtected(self, opposedThroughMvt): + '''Checks if this left movement should be protected, + ie if one of the main two conditions on left turn is verified''' + return self.volume >= 200 or self.volume*opposedThroughMvt.volume/opposedThroughMvt.nLanes > 50000 + + def getPCUVolume(self): + '''Returns the passenger-car equivalent for the input volume''' + v = 0 + for p, e in zip(self.proportions, self.equivalents): + v += p*e + return v*self.volume + +class IntersectionMovement(object): + '''Represents an intersection movement + with a volume, a type (through, left or right) + and an equivalent for movement type''' + def __init__(self, volume, mvtEquivalent = 1): + self.volume = volume + self.mvtEquivalent = mvtEquivalent + + def getTVUVolume(self): + return self.mvtEquivalent*self.volume.getPCUVolume() + +class LaneGroup(object): + '''Class that represents a group of mouvements''' + + def __init__(self, movements, nLanes): + self.movements = movements + self.nLanes = nLanes + + def getTVUVolume(self): + return sum([mvt.getTVUVolume() for mvt in self.movements]) + + def getCharge(self, saturationVolume): + return self.getTVUVolume()/(self.nLanes*saturationVolume) + +def optimalCycle(lostTime, criticalCharge): + return (1.5*lostTime+5)/(1-criticalCharge) + +def minimumCycle(lostTime, criticalCharge, degreeSaturation=1.): + 'degree of saturation can be used as the peak hour factor too' + return lostTime/(1-criticalCharge/degreeSaturation) + +class Cycle(object): + '''Class to compute optimal cycle and the split of effective green times''' + def __init__(self, phases, lostTime, saturationVolume): + '''phases is a list of phases + a phase is a list of lanegroups''' + self.phases = phases + self.lostTime = lostTime + self.saturationVolume = saturationVolume + + def computeCriticalCharges(self): + self.criticalCharges = [max([lg.getCharge(self.saturationVolume) for lg in phase]) for phase in self.phases] + self.criticalCharge = sum(self.criticalCharges) + + def computeOptimalCycle(self): + self.computeCriticalCharges() + self.C = optimalCycle(self.lostTime, self.criticalCharge) + return self.C + + def computeMinimumCycle(self, degreeSaturation=1.): + self.computeCriticalCharges() + self.C = minimumCycle(self.lostTime, self.criticalCharge, degreeSaturation) + return self.C + + def computeEffectiveGreen(self): + #from numpy import round + #self.computeCycle() # in case it was not done before + effectiveGreenTime = self.C-self.lostTime + self.effectiveGreens = [round(c*effectiveGreenTime/self.criticalCharge,1) for c in self.criticalCharges] + return self.effectiveGreens + + +def computeInterGreen(perceptionReactionTime, initialSpeed, intersectionLength, vehicleAverageLength = 6, deceleration = 3): + '''Computes the intergreen time (yellow/amber plus all red time) + Deceleration is positive + All variables should be in the same units''' + if deceleration > 0: + return [perceptionReactionTime+float(initialSpeed)/(2*deceleration), float(intersectionLength+vehicleAverageLength)/initialSpeed] + else: + print('Issue deceleration should be strictly positive') + return None + +def uniformDelay(cycleLength, effectiveGreen, saturationDegree): + '''Computes the uniform delay''' + return 0.5*cycleLength*(1-float(effectiveGreen)/cycleLength)**2/(1-float(effectiveGreen*saturationDegree)/cycleLength) + +def randomDelay(volume, saturationDegree): + '''Computes the random delay = queueing time for M/D/1''' + return saturationDegree**2/(2*volume*(1-saturationDegree)) + +def incrementalDelay(T, X, c, k=0.5, I=1): + '''Computes the incremental delay (HCM) + T in hours + c capacity of the lane group + k default for fixed time signal + I=1 for isolated intersection (Poisson arrival)''' + from math import sqrt + return 900*T*(X - 1 + sqrt((X - 1)**2 + 8*k*I*X/(c*T))) + +######################### +# misc +######################### + +def timeChangingSpeed(v0, vf, a, TPR): + 'for decelerations, a < 0' + return TPR-(vf-v0)/a + +def distanceChangingSpeed(v0, vf, a, TPR): + 'for decelerations, a < 0' + return TPR*v0-(vf**2-v0**2)/(2*a)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/ubc_utils.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,226 @@ +#! /usr/bin/env python +'''Various utilities to load data saved by the UBC tool(s)''' + +from trafficintelligence import utils, events, storage +from trafficintelligence.moving import MovingObject, TimeInterval, Trajectory + + +fileTypeNames = ['feature', + 'object', + 'prototype', + 'contoursequence'] + +severityIndicatorNames = ['Distance', + 'Collision Course Cosine', + 'Velocity Cosine', + 'Speed Differential', + 'Collision Probability', + 'Severity Index', + 'Time to Collision'] + +userTypeNames = ['car', + 'pedestrian', + 'twowheels', + 'bus', + 'truck'] + +# severityIndicator = {'Distance': 0, +# 'Cosine': 1, +# 'Velocity Cosine': 2, +# 'Speed Differential': 3, +# 'Collision Probability': 4, +# 'Severity Index': 5, +# 'TTC': 6} + +mostSevereIsMax = [False, + False, + True, + True, + True, + True, + False] + +ignoredValue = [None, None, None, None, None, None, -1] + +def getFileType(s): + 'Finds the type in fileTypeNames' + for fileType in fileTypeNames: + if s.find(fileType)>0: + return fileType + return '' + +def isFileType(s, fileType): + return (s.find(fileType)>0) + +def saveTrajectoryUserTypes(inFilename, outFilename, objects): + '''The program saves the objects, + by just copying the corresponding trajectory and velocity data + from the inFilename, and saving the characteristics in objects (first line) + into outFilename''' + infile = storage.openCheck(inFilename) + outfile = storage.openCheck(outFilename,'w') + + if (inFilename.find('features') >= 0) or (not infile) or (not outfile): + return + + lines = storage.getLines(infile) + objNum = 0 # in inFilename + while lines != []: + # find object in objects (index i) + i = 0 + while (i<len(objects)) and (objects[i].num != objNum): + i+=1 + + if i<len(objects): + l = lines[0].split(' ') + l[3] = str(objects[i].userType) + outfile.write(' '.join(l)+'\n') + for l in lines[1:]: + outfile.write(l+'\n') + outfile.write(utils.delimiterChar+'\n') + # next object + objNum += 1 + lines = storage.getLines(infile) + + print('read {0} objects'.format(objNum)) + +def modifyTrajectoryFile(modifyLines, filenameIn, filenameOut): + '''Reads filenameIn, replaces the lines with the result of modifyLines and writes the result in filenameOut''' + fileIn = storage.openCheck(filenameIn, 'r', True) + fileOut = storage.openCheck(filenameOut, "w", True) + + lines = storage.getLines(fileIn) + trajNum = 0 + while (lines != []): + modifiedLines = modifyLines(trajNum, lines) + if modifiedLines: + for l in modifiedLines: + fileOut.write(l+"\n") + fileOut.write(utils.delimiterChar+"\n") + lines = storage.getLines(fileIn) + trajNum += 1 + + fileIn.close() + fileOut.close() + +def copyTrajectoryFile(keepTrajectory, filenameIn, filenameOut): + '''Reads filenameIn, keeps the trajectories for which the function keepTrajectory(trajNum, lines) is True + and writes the result in filenameOut''' + fileIn = storage.openCheck(filenameIn, 'r', True) + fileOut = storage.openCheck(filenameOut, "w", True) + + lines = storage.getLines(fileIn) + trajNum = 0 + while (lines != []): + if keepTrajectory(trajNum, lines): + for l in lines: + fileOut.write(l+"\n") + fileOut.write(utils.delimiterChar+"\n") + lines = storage.getLines(fileIn) + trajNum += 1 + + fileIn.close() + fileOut.close() + +def loadTrajectories(filename, nObjects = -1): + '''Loads trajectories''' + + file = storage.openCheck(filename) + if (not file): + return [] + + objects = [] + objNum = 0 + objectType = getFileType(filename) + lines = storage.getLines(file) + while (lines != []) and ((nObjects<0) or (objNum<nObjects)): + l = lines[0].split(' ') + parsedLine = [int(n) for n in l[:4]] + obj = MovingObject(num = objNum, timeInterval = TimeInterval(parsedLine[1],parsedLine[2])) + #add = True + if len(lines) >= 3: + obj.positions = Trajectory.load(lines[1], lines[2]) + if len(lines) >= 5: + obj.velocities = Trajectory.load(lines[3], lines[4]) + if objectType == 'object': + obj.userType = parsedLine[3] + obj.nObjects = float(l[4]) + obj.featureNumbers = [int(n) for n in l[5:]] + + # load contour data if available + if len(lines) >= 6: + obj.contourType = utils.line2Floats(lines[6]) + obj.contourOrigins = Trajectory.load(lines[7], lines[8]) + obj.contourSizes = Trajectory.load(lines[9], lines[10]) + elif objectType == 'prototype': + obj.userType = parsedLine[3] + obj.nMatchings = int(l[4]) + + if len(lines) != 2: + objects.append(obj) + objNum+=1 + else: + print("Error two lines of data for feature {}".format(f.num)) + + lines = storage.getLines(file) + + file.close() + return objects + +def getFeatureNumbers(objects): + featureNumbers=[] + for o in objects: + featureNumbers += o.featureNumbers + return featureNumbers + +def loadInteractions(filename, nInteractions = -1): + 'Loads interactions from the old UBC traffic event format' + from events import Interaction + from indicators import SeverityIndicator + file = storage.openCheck(filename) + if (not file): + return [] + + interactions = [] + interactionNum = 0 + lines = storage.getLines(file) + while (lines != []) and ((nInteractions<0) or (interactionNum<nInteractions)): + parsedLine = [int(n) for n in lines[0].split(' ')] + inter = Interaction(interactionNum, TimeInterval(parsedLine[1],parsedLine[2]), parsedLine[3], parsedLine[4], categoryNum = parsedLine[5]) + + indicatorFrameNums = [int(n) for n in lines[1].split(' ')] + for indicatorNum,line in enumerate(lines[2:]): + values = {} + for i,v in enumerate([float(n) for n in line.split(' ')]): + if not ignoredValue[indicatorNum] or v != ignoredValue[indicatorNum]: + values[indicatorFrameNums[i]] = v + inter.addIndicator(SeverityIndicator(severityIndicatorNames[indicatorNum], values, None, mostSevereIsMax[indicatorNum])) + + interactions.append(inter) + interactionNum+=1 + lines = storage.getLines(file) + + file.close() + return interactions + +def loadCollisionPoints(filename, nPoints = -1): + '''Loads collision points and returns a dict + with keys as a pair of the numbers of the two interacting objects''' + file = storage.openCheck(filename) + if (not file): + return [] + + points = {} + num = 0 + lines = storage.getLines(file) + while (lines != []) and ((nPoints<0) or (num<nPoints)): + parsedLine = [int(n) for n in lines[0].split(' ')] + protagonistNums = (parsedLine[0], parsedLine[1]) + points[protagonistNums] = [[float(n) for n in lines[1].split(' ')], + [float(n) for n in lines[2].split(' ')]] + + num+=1 + lines = storage.getLines(file) + + file.close() + return points
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trafficintelligence/utils.py Fri Jun 15 11:19:10 2018 -0400 @@ -0,0 +1,1090 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +''' Generic utilities.''' + +import matplotlib.pyplot as plt +from datetime import time, datetime +from argparse import ArgumentTypeError +from pathlib import Path +from math import sqrt, ceil, floor +from scipy.stats import rv_continuous, kruskal, shapiro, lognorm +from scipy.spatial import distance +from scipy.sparse import dok_matrix +from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, mean, median, percentile, isnan, ones, convolve, dtype, isnan, NaN, ma, isinf, savez, load as npload, log + + +datetimeFormat = "%Y-%m-%d %H:%M:%S" + +sjcamDatetimeFormat = "%Y_%m%d_%H%M%S"#2017_0626_143720 + +######################### +# Strings +######################### + +def upperCaseFirstLetter(s): + words = s.split(' ') + lowerWords = [w[0].upper()+w[1:].lower() for w in words] + return ' '.join(lowerWords) + +class TimeConverter: + def __init__(self, datetimeFormat = datetimeFormat): + self.datetimeFormat = datetimeFormat + + def convert(self, s): + try: + return datetime.strptime(s, self.datetimeFormat) + except ValueError: + msg = "Not a valid date: '{0}'.".format(s) + raise ArgumentTypeError(msg) + +######################### +# Enumerations +######################### + +def inverseEnumeration(l): + 'Returns the dictionary that provides for each element in the input list its index in the input list' + result = {} + for i,x in enumerate(l): + result[x] = i + return result + +######################### +# Simple statistics +######################### + +def logNormalMeanVar(loc, scale): + '''location and scale are respectively the mean and standard deviation of the normal in the log-normal distribution + https://en.wikipedia.org/wiki/Log-normal_distribution + + same as lognorm.stats(scale, 0, exp(loc))''' + mean = exp(loc+(scale**2)/2) + var = (exp(scale**2)-1)*exp(2*loc+scale**2) + return mean, var + +def fitLogNormal(x): + 'returns the fitted location and scale of the lognormal (general definition)' + shape, loc, scale = lognorm.fit(x, floc=0.) + return log(scale), shape + +def sampleSize(stdev, tolerance, percentConfidence, nRoundingDigits = None, printLatex = False): + from scipy.stats.distributions import norm + if nRoundingDigits is None: + k = round(norm.ppf(0.5+percentConfidence/200., 0, 1), 2) # 1.-(100-percentConfidence)/200. + else: + k = round(norm.ppf(0.5+percentConfidence/200., 0, 1), nRoundingDigits) + stdev = round(stdev, nRoundingDigits) + tolerance = round(tolerance, nRoundingDigits) + if printLatex: + print('$z_{{{}}}^2\\frac{{s^2}}{{e^2}}={}^2\\frac{{{}^2}}{{{}^2}}$'.format(0.5+percentConfidence/200.,k, stdev, tolerance)) + return (k*stdev/tolerance)**2 + +def confidenceInterval(mean, stdev, nSamples, percentConfidence, trueStd = True, printLatex = False): + '''if trueStd, use normal distribution, otherwise, Student + + Use otherwise t.interval or norm.interval for the boundaries + ex: norm.interval(0.95) + t.interval(0.95, nSamples-1)''' + from scipy.stats.distributions import norm, t + if trueStd: + k = round(norm.ppf(0.5+percentConfidence/200., 0, 1), 2) + else: # use Student + k = round(t.ppf(0.5+percentConfidence/200., nSamples-1), 2) + e = k*stdev/sqrt(nSamples) + if printLatex: + print('${0} \pm {1}\\frac{{{2}}}{{\sqrt{{{3}}}}}$'.format(mean, k, stdev, nSamples)) + return mean-e, mean+e + +def computeChi2(expected, observed): + '''Returns the Chi2 statistics''' + return sum([((e-o)*(e-o))/float(e) for e, o in zip(expected, observed)]) + +class generateDistribution(rv_continuous): + def __init__(self, values, probabilities): + '''The values (and corresponding probabilities) are supposed to be sorted by value + for v, p in zip(values, probabilities): P(X<=v)=p''' + assert probabilities[0]==0 + self.values = values + self.probabilities = probabilities + + def _cdf(self, x): + if x < self.values[0]: + return self.probabilities[0] + else: + i=0 + while i+1<len(self.values) and self.values[i+1] < x: + i += 1 + if i == len(self.values)-1: + return self.probabilities[-1] + else: + return (self.probabilities[i+1]-self.probabilities[i])/(self.values[i+1]-self.values[i]) + +class DistributionSample(object): + def nSamples(self): + return sum(self.counts) + +def cumulativeDensityFunction(sample, normalized = False): + '''Returns the cumulative density function of the sample of a random variable''' + xaxis = sorted(sample) + counts = arange(1,len(sample)+1) # dtype = float + if normalized: + counts /= float(len(sample)) + return xaxis, counts + +class DiscreteDistributionSample(DistributionSample): + '''Class to represent a sample of a distribution for a discrete random variable''' + def __init__(self, categories, counts): + self.categories = categories + self.counts = counts + + def mean(self): + result = [float(x*y) for x,y in zip(self.categories, self.counts)] + return npsum(result)/self.nSamples() + + def var(self, mean = None): + if not mean: + m = self.mean() + else: + m = mean + result = 0. + squares = [float((x-m)*(x-m)*y) for x,y in zip(self.categories, self.counts)] + return npsum(squares)/(self.nSamples()-1) + + def referenceCounts(self, probability): + '''probability is a function that returns the probability of the random variable for the category values''' + refProba = [probability(c) for c in self.categories] + refProba[-1] = 1-npsum(refProba[:-1]) + refCounts = [r*self.nSamples() for r in refProba] + return refCounts, refProba + +class ContinuousDistributionSample(DistributionSample): + '''Class to represent a sample of a distribution for a continuous random variable + with the number of observations for each interval + intervals (categories variable) are defined by their left limits, the last one being the right limit + categories contain therefore one more element than the counts''' + def __init__(self, categories, counts): + # todo add samples for initialization and everything to None? (or setSamples?) + self.categories = categories + self.counts = counts + + @staticmethod + def generate(sample, categories): + if min(sample) < min(categories): + print('Sample has lower min than proposed categories ({}, {})'.format(min(sample), min(categories))) + if max(sample) > max(categories): + print('Sample has higher max than proposed categories ({}, {})'.format(max(sample), max(categories))) + dist = ContinuousDistributionSample(sorted(categories), [0]*(len(categories)-1)) + for s in sample: + i = 0 + while i<len(dist.categories) and dist.categories[i] <= s: + i += 1 + if i <= len(dist.counts): + dist.counts[i-1] += 1 + #print('{} in {} {}'.format(s, dist.categories[i-1], dist.categories[i])) + else: + print('Element {} is not in the categories'.format(s)) + return dist + + def mean(self): + result = 0. + for i in range(len(self.counts)-1): + result += self.counts[i]*(self.categories[i]+self.categories[i+1])/2 + return result/self.nSamples() + + def var(self, mean = None): + if not mean: + m = self.mean() + else: + m = mean + result = 0. + for i in range(len(self.counts)-1): + mid = (self.categories[i]+self.categories[i+1])/2 + result += self.counts[i]*(mid - m)*(mid - m) + return result/(self.nSamples()-1) + + def referenceCounts(self, cdf): + '''cdf is a cumulative distribution function + returning the probability of the variable being less that x''' + # refCumulativeCounts = [0]#[cdf(self.categories[0][0])] +# for inter in self.categories: +# refCumulativeCounts.append(cdf(inter[1])) + refCumulativeCounts = [cdf(x) for x in self.categories[1:-1]] + + refProba = [refCumulativeCounts[0]] + for i in xrange(1,len(refCumulativeCounts)): + refProba.append(refCumulativeCounts[i]-refCumulativeCounts[i-1]) + refProba.append(1-refCumulativeCounts[-1]) + refCounts = [p*self.nSamples() for p in refProba] + + return refCounts, refProba + + def printReferenceCounts(self, refCounts=None): + if refCounts: + ref = refCounts + else: + ref = self.referenceCounts + for i in xrange(len(ref[0])): + print('{0}-{1} & {2:0.3} & {3:0.3} \\\\'.format(self.categories[i],self.categories[i+1],ref[1][i], ref[0][i])) + + +######################### +# maths section +######################### + +# def kernelSmoothing(sampleX, X, Y, weightFunc, halfwidth): +# '''Returns a smoothed weighted version of Y at the predefined values of sampleX +# Sum_x weight(sample_x,x) * y(x)''' +# from numpy import zeros, array +# smoothed = zeros(len(sampleX)) +# for i,x in enumerate(sampleX): +# weights = array([weightFunc(x,xx, halfwidth) for xx in X]) +# if sum(weights)>0: +# smoothed[i] = sum(weights*Y)/sum(weights) +# else: +# smoothed[i] = 0 +# return smoothed + +def kernelSmoothing(x, X, Y, weightFunc, halfwidth): + '''Returns the smoothed estimate of (X,Y) at x + Sum_x weight(sample_x,x) * y(x)''' + weights = array([weightFunc(x,observedx, halfwidth) for observedx in X]) + if sum(weights)>0: + return sum(weights*Y)/sum(weights) + else: + return 0 + +def uniform(center, x, halfwidth): + if abs(center-x)<halfwidth: + return 1. + else: + return 0. + +def gaussian(center, x, halfwidth): + return exp(-((center-x)/halfwidth)**2/2) + +def epanechnikov(center, x, halfwidth): + diff = abs(center-x) + if diff<halfwidth: + return 1.-(diff/halfwidth)**2 + else: + return 0. + +def triangular(center, x, halfwidth): + diff = abs(center-x) + if diff<halfwidth: + return 1.-abs(diff/halfwidth) + else: + return 0. + +def medianSmoothing(x, X, Y, halfwidth): + '''Returns the media of Y's corresponding to X's in the interval [x-halfwidth, x+halfwidth]''' + return median([y for observedx, y in zip(X,Y) if abs(x-observedx)<halfwidth]) + +def argmaxDict(d): + return max(d, key=d.get) + +def deltaFrames(t1, t2, frameRate): + '''Returns the number of frames between t1 and t2 + positive if t1<=t2, negative otherwise''' + if t1 > t2: + return -(t1-t2).seconds*frameRate + else: + return (t2-t1).seconds*frameRate + +def framesToTime(nFrames, frameRate, initialTime = time()): + '''returns a datetime.time for the time in hour, minutes and seconds + initialTime is a datetime.time''' + seconds = int(floor(float(nFrames)/float(frameRate))+initialTime.hour*3600+initialTime.minute*60+initialTime.second) + h = int(floor(seconds/3600.)) + seconds = seconds - h*3600 + m = int(floor(seconds/60)) + seconds = seconds - m*60 + return time(h, m, seconds) + +def timeToFrames(t, frameRate): + return frameRate*(t.hour*3600+t.minute*60+t.second) + +def sortXY(X,Y): + 'returns the sorted (x, Y(x)) sorted on X' + D = {} + for x, y in zip(X,Y): + D[x]=y + xsorted = sorted(D.keys()) + return xsorted, [D[x] for x in xsorted] + +def compareLengthForSort(i, j): + if len(i) < len(j): + return -1 + elif len(i) == len(j): + return 0 + else: + return 1 + +def sortByLength(instances, reverse = False): + '''Returns a new list with the instances sorted by length (method __len__) + reverse is passed to sorted''' + return sorted(instances, key = len, reverse = reverse) + +def ceilDecimals(v, nDecimals): + '''Rounds the number at the nth decimal + eg 1.23 at 0 decimal is 2, at 1 decimal is 1.3''' + tens = 10**nDecimals + return ceil(v*tens)/tens + +def inBetween(bound1, bound2, x): + 'useful if one does not know the order of bound1/bound2' + return bound1 <= x <= bound2 or bound2 <= x <= bound1 + +def pointDistanceL2(x1,y1,x2,y2): + ''' Compute point-to-point distance (L2 norm, ie Euclidean distance)''' + return sqrt((x2-x1)**2+(y2-y1)**2) + +def crossProduct(l1, l2): + return l1[0]*l2[1]-l1[1]*l2[0] + +def cat_mvgavg(cat_list, halfWidth): + ''' Return a list of categories/values smoothed according to a window. + halfWidth is the search radius on either side''' + from copy import deepcopy + smoothed = deepcopy(cat_list) + for point in range(len(cat_list)): + lower_bound_check = max(0,point-halfWidth) + upper_bound_check = min(len(cat_list)-1,point+halfWidth+1) + window_values = cat_list[lower_bound_check:upper_bound_check] + smoothed[point] = max(set(window_values), key=window_values.count) + return smoothed + +def filterMovingWindow(inputSignal, halfWidth): + '''Returns an array obtained after the smoothing of the input by a moving average + The first and last points are copied from the original.''' + width = float(halfWidth*2+1) + win = ones(width,'d') + result = convolve(win/width,array(inputSignal),'same') + result[:halfWidth] = inputSignal[:halfWidth] + result[-halfWidth:] = inputSignal[-halfWidth:] + return result + +def linearRegression(x, y, deg = 1, plotData = False): + '''returns the least square estimation of the linear regression of y = ax+b + as well as the plot''' + from numpy.lib.polynomial import polyfit + from numpy.core.multiarray import arange + coef = polyfit(x, y, deg) + if plotData: + def poly(x): + result = 0 + for i in range(len(coef)): + result += coef[i]*x**(len(coef)-i-1) + return result + plt.plot(x, y, 'x') + xx = arange(min(x), max(x),(max(x)-min(x))/1000) + plt.plot(xx, [poly(z) for z in xx]) + return coef + +def correlation(data, correlationMethod = 'pearson', plotFigure = False, displayNames = None, figureFilename = None): + '''Computes (and displays) the correlation matrix for a pandas DataFrame''' + columns = data.columns.tolist() + for var in data.columns: + uniqueValues = data[var].unique() + if len(uniqueValues) == 1 or data.dtypes[var] == dtype('O') or (len(uniqueValues) == 2 and len(data.loc[~isnan(data[var]), var].unique()) == 1): # last condition: only one other value than nan + columns.remove(var) + c=data[columns].corr(correlationMethod) + if plotFigure: + fig = plt.figure(figsize=(4+0.4*c.shape[0], 0.4*c.shape[0])) + fig.add_subplot(1,1,1) + #plt.imshow(np.fabs(c), interpolation='none') + plt.imshow(c, vmin=-1., vmax = 1., interpolation='none', cmap = 'RdYlBu_r') # coolwarm + if displayNames is not None: + colnames = [displayNames.get(s.strip(), s.strip()) for s in columns] + else: + colnames = columns + #correlation.plot_corr(c, xnames = colnames, normcolor=True, title = filename) + plt.xticks(range(len(colnames)), colnames, rotation=90) + plt.yticks(range(len(colnames)), colnames) + plt.tick_params('both', length=0) + plt.subplots_adjust(bottom = 0.29) + plt.colorbar() + plt.title('Correlation ({})'.format(correlationMethod)) + plt.tight_layout() + if len(colnames) > 50: + plt.subplots_adjust(left=.06) + if figureFilename is not None: + plt.savefig(figureFilename, dpi = 150, transparent = True) + return c + +def addDummies(data, variables, allVariables = True): + '''Add binary dummy variables for each value of a nominal variable + in a pandas DataFrame''' + newVariables = [] + for var in variables: + if var in data.columns and data.dtypes[var] == dtype('O') and len(data[var].unique()) > 2: + values = data[var].unique() + if not allVariables: + values = values[:-1] + for val in values: + if val is not NaN: + newVariable = (var+'_{}'.format(val)).replace('.','').replace(' ','').replace('-','') + data[newVariable] = (data[var] == val) + newVariables.append(newVariable) + return newVariables + +def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, filenamePrefix = None, figureFileType = 'pdf', saveLatex = False, renameVariables = lambda s: s, kwCaption = ''): + '''Studies the influence of (nominal) independent variable over the dependent variable + + Makes tests if the conditional distributions are normal + using the Shapiro-Wilk test (in which case ANOVA could be used) + Implements uses the non-parametric Kruskal Wallis test''' + tmp = data[data[independentVariable].notnull()] + independentVariableValues = sorted(tmp[independentVariable].unique().tolist()) + if len(independentVariableValues) >= 2: + if saveLatex: + from storage import openCheck + out = openCheck(filenamePrefix+'-{}-{}.tex'.format(dependentVariable, independentVariable), 'w') + for x in independentVariableValues: + print('Shapiro-Wilk normality test for {} when {}={}: {} obs'.format(dependentVariable,independentVariable, x, len(tmp.loc[tmp[independentVariable] == x, dependentVariable]))) + if len(tmp.loc[tmp[independentVariable] == x, dependentVariable]) >= 3: + print(shapiro(tmp.loc[tmp[independentVariable] == x, dependentVariable])) + if plotFigure: + plt.figure() + plt.boxplot([tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues]) + plt.xticks(range(1,len(independentVariableValues)+1), independentVariableValues) + plt.title('{} vs {}'.format(dependentVariable, independentVariable)) + if filenamePrefix is not None: + plt.savefig(filenamePrefix+'-{}-{}.{}'.format(dependentVariable, independentVariable, figureFileType)) + table = tmp.groupby([independentVariable])[dependentVariable].describe().unstack().sort(['50%'], ascending = False) + table['count'] = table['count'].astype(int) + testResult = kruskal(*[tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues]) + if saveLatex: + out.write('\\begin{minipage}{\\linewidth}\n' + +'\\centering\n' + +'\\captionof{table}{'+(kwCaption.format(dependentVariable, independentVariable, *testResult))+'}\n' + +table.to_latex(float_format = lambda x: '{:.3f}'.format(x)).encode('ascii')+'\n' + +'\\end{minipage}\n' + +'\\ \\vspace{0.5cm}\n') + else: + print(table) + return testResult + else: + return None + +def prepareRegression(data, dependentVariable, independentVariables, maxCorrelationThreshold, correlations, maxCorrelationP, correlationFunc, stdoutText = ['Removing {} (constant: {})', 'Removing {} (correlation {} with {})', 'Removing {} (no correlation: {}, p={})'], saveFiles = False, filenamePrefix = None, latexHeader = '', latexTable = None, latexFooter=''): + '''Removes variables from candidate independent variables if + - if two independent variables are correlated (> maxCorrelationThreshold), one is removed + - if an independent variable is not correlated with the dependent variable (p>maxCorrelationP) + Returns the remaining non-correlated variables, correlated with the dependent variable + + correlationFunc is spearmanr or pearsonr from scipy.stats + text is the template to display for the two types of printout (see default): 3 elements if no saving to latex file, 8 otherwise + + TODO: pass the dummies for nominal variables and remove if all dummies are correlated, or none is correlated with the dependentvariable''' + from copy import copy + from pandas import DataFrame + result = copy(independentVariables) + table1 = '' + table2 = {} + # constant variables + for var in independentVariables: + uniqueValues = data[var].unique() + if (len(uniqueValues) == 1) or (len(uniqueValues) == 2 and uniqueValues.dtype != dtype('O') and len(data.loc[~isnan(data[var]), var].unique()) == 1): + print(stdoutText[0].format(var, uniqueValues)) + if saveFiles: + table1 += latexTable[0].format(var, *uniqueValues) + result.remove(var) + # correlated variables + for v1 in copy(result): + if v1 in correlations.index: + for v2 in copy(result): + if v2 != v1 and v2 in correlations.index: + if abs(correlations.loc[v1, v2]) > maxCorrelationThreshold: + if v1 in result and v2 in result: + if saveFiles: + table1 += latexTable[1].format(v2, v1, correlations.loc[v1, v2]) + print(stdoutText[1].format(v2, v1, correlations.loc[v1, v2])) + result.remove(v2) + # not correlated with dependent variable + table2['Correlations'] = [] + table2['Valeurs p'] = [] + for var in copy(result): + if data.dtypes[var] != dtype('O'): + cor, p = correlationFunc(data[dependentVariable], data[var]) + if p > maxCorrelationP: + if saveFiles: + table1 += latexTable[2].format(var, cor, p) + print(stdoutText[2].format(var, cor, p)) + result.remove(var) + else: + table2['Correlations'].append(cor) + table2['Valeurs p'].append(p) + + if saveFiles: + from storage import openCheck + out = openCheck(filenamePrefix+'-removed-variables.tex', 'w') + out.write(latexHeader) + out.write(table1) + out.write(latexFooter) + out.close() + out = openCheck(filenamePrefix+'-correlations.html', 'w') + table2['Variables'] = [var for var in result if data.dtypes[var] != dtype('O')] + out.write(DataFrame(table2)[['Variables', 'Correlations', 'Valeurs p']].to_html(formatters = {'Correlations': lambda x: '{:.2f}'.format(x), 'Valeurs p': lambda x: '{:.3f}'.format(x)}, index = False)) + out.close() + return result + +def saveDokMatrix(filename, m, lowerTriangle = False): + 'Saves a dok_matrix using savez' + if lowerTriangle: + keys = [k for k in m if k[0] > k[1]] + savez(filename, shape = m.shape, keys = keys, values = [m[k[0],k[1]] for k in keys]) + else: + savez(filename, shape = m.shape, keys = list(m.keys()), values = list(m.values())) + +def loadDokMatrix(filename): + 'Loads a dok_matrix saved using the above saveDokMatrix' + data = npload(filename) + m = dok_matrix(tuple(data['shape'])) + for k, v in zip(data['keys'], data['values']): + m[tuple(k)] = v + return m + +def aggregationFunction(funcStr, centile = 50): + '''return the numpy function corresponding to funcStr + centile can be a list of centiles to compute at once, eg [25, 50, 75] for the 3 quartiles''' + if funcStr == 'median': + return median + elif funcStr == 'mean': + return mean + elif funcStr == 'centile': + return lambda x: percentile(x, centile) + elif funcStr == '85centile': + return lambda x: percentile(x, 85) + else: + print('Unknown aggregation method: {}'.format(funcStr)) + return None + +######################### +# regression analysis using statsmodels (and pandas) +######################### + +# TODO make class for experiments? +# TODO add tests with public dataset downloaded from Internet (IRIS et al) +def modelString(experiment, dependentVariable, independentVariables): + return dependentVariable+' ~ '+' + '.join([independentVariable for independentVariable in independentVariables if experiment[independentVariable]]) + +def runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols'): + import statsmodels.formula.api as smf + modelStr = modelString(experiment, dependentVariable, independentVariables) + if regressionType == 'ols': + model = smf.ols(modelStr, data = data) + elif regressionType == 'gls': + model = smf.gls(modelStr, data = data) + elif regressionType == 'rlm': + model = smf.rlm(modelStr, data = data) + else: + print('Unknown regression type {}. Exiting'.format(regressionType)) + import sys + sys.exit() + return model.fit() + +def runModels(experiments, data, dependentVariable, independentVariables, regressionType = 'ols'): + '''Runs several models and stores 3 statistics + adjusted R2, condition number (should be small, eg < 1000) + and p-value for Shapiro-Wilk test of residual normality''' + for i,experiment in experiments.iterrows(): + if experiment[independentVariables].any(): + results = runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols') + experiments.loc[i,'r2adj'] = results.rsquared_adj + experiments.loc[i,'condNum'] = results.condition_number + experiments.loc[i, 'shapiroP'] = shapiro(results.resid)[1] + experiments.loc[i,'nobs'] = int(results.nobs) + return experiments + +def generateExperiments(independentVariables): + '''Generates all possible models for including or not each independent variable''' + from pandas import DataFrame + experiments = {} + nIndependentVariables = len(independentVariables) + if nIndependentVariables != len(set(independentVariables)): + print("Duplicate variables. Exiting") + import sys + sys.exit() + nModels = 2**nIndependentVariables + for i,var in enumerate(independentVariables): + pattern = [False]*(2**i)+[True]*(2**i) + experiments[var] = pattern*(2**(nIndependentVariables-i-1)) + experiments = DataFrame(experiments) + experiments['r2adj'] = 0. + experiments['condNum'] = NaN + experiments['shapiroP'] = -1 + experiments['nobs'] = -1 + return experiments + +def findBestModel(data, dependentVariable, independentVariables, regressionType = 'ols', nProcesses = 1): + '''Generates all possible model with the independentVariables + and runs them, saving the results in experiments + with multiprocess option''' + from pandas import concat + from multiprocessing import Pool + experiments = generateExperiments(independentVariables) + nModels = len(experiments) + print("Running {} models with {} processes".format(nModels, nProcesses)) + print("IndependentVariables: {}".format(independentVariables)) + if nProcesses == 1: + return runModels(experiments, data, dependentVariable, independentVariables, regressionType) + else: + pool = Pool(processes = nProcesses) + chunkSize = int(ceil(nModels/nProcesses)) + jobs = [pool.apply_async(runModels, args = (experiments[i*chunkSize:(i+1)*chunkSize], data, dependentVariable, independentVariables, regressionType)) for i in range(nProcesses)] + return concat([job.get() for job in jobs]) + +def findBestModelFwd(data, dependentVariable, independentVariables, modelFunc, experiments = None): + '''Forward search for best model (based on adjusted R2) + Randomly starting with one variable and adding randomly variables + if they improve the model + + The results are added to experiments if provided as argument + Storing in experiment relies on the index being the number equal + to the binary code derived from the independent variables''' + from numpy.random import permutation as nppermutation + if experiments is None: + experiments = generateExperiments(independentVariables) + nIndependentVariables = len(independentVariables) + permutation = nppermutation(list(range(nIndependentVariables))) + variableMapping = {j: independentVariables[i] for i,j in enumerate(permutation)} + print('Tested variables '+', '.join([variableMapping[i] for i in range(nIndependentVariables)])) + bestModel = [False]*nIndependentVariables + currentVarNum = 0 + currentR2Adj = 0. + for currentVarNum in range(nIndependentVariables): + currentModel = [i for i in bestModel] + currentModel[currentVarNum] = True + rowIdx = sum([0]+[2**i for i in range(nIndependentVariables) if currentModel[permutation[i]]]) + #print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in range(nIndependentVariables) if currentModel[permutation[i]]]) + if experiments.loc[rowIdx, 'shapiroP'] < 0: + modelStr = modelString(experiments.loc[rowIdx], dependentVariable, independentVariables) + model = modelFunc(modelStr, data = data) + results = model.fit() + experiments.loc[rowIdx, 'r2adj'] = results.rsquared_adj + experiments.loc[rowIdx, 'condNum'] = results.condition_number + experiments.loc[rowIdx, 'shapiroP'] = shapiro(results.resid)[1] + experiments.loc[rowIdx, 'nobs'] = int(results.nobs) + if currentR2Adj < experiments.loc[rowIdx, 'r2adj']: + currentR2Adj = experiments.loc[rowIdx, 'r2adj'] + bestModel[currentVarNum] = True + return experiments + +def displayModelResults(results, model = None, plotFigures = True, filenamePrefix = None, figureFileType = 'pdf', text = {'title-shapiro': 'Shapiro-Wilk normality test for residuals: {:.2f} (p={:.3f})', 'true-predicted.xlabel': 'Predicted values', 'true-predicted.ylabel': 'True values', 'residuals-predicted.xlabel': 'Predicted values', 'residuals-predicted.ylabel': 'Residuals'}): + import statsmodels.api as sm + '''Displays some model results + + 3 graphics, true-predicted, residuals-predicted, ''' + print(results.summary()) + shapiroResult = shapiro(results.resid) + print(shapiroResult) + if plotFigures: + fig = plt.figure(figsize=(7,6.3*(2+int(model is not None)))) + if model is not None: + ax = fig.add_subplot(3,1,1) + plt.plot(results.predict(), model.endog, 'x') + x=plt.xlim() + y=plt.ylim() + plt.plot([max(x[0], y[0]), min(x[1], y[1])], [max(x[0], y[0]), min(x[1], y[1])], 'r') + #plt.axis('equal') + if text is not None: + plt.title(text['title-shapiro'].format(*shapiroResult)) + #plt.title(text['true-predicted.title']) + plt.xlabel(text['true-predicted.xlabel']) + plt.ylabel(text['true-predicted.ylabel']) + fig.add_subplot(3,1,2, sharex = ax) + plt.plot(results.predict(), results.resid, 'x') + nextSubplotNum = 3 + else: + fig.add_subplot(2,1,1) + plt.plot(results.predict(), results.resid, 'x') + nextSubplotNum = 2 + if text is not None: + if model is None: + plt.title(text['title-shapiro'].format(*shapiroResult)) + plt.xlabel(text['residuals-predicted.xlabel']) + plt.ylabel(text['residuals-predicted.ylabel']) + qqAx = fig.add_subplot(nextSubplotNum,1,nextSubplotNum) + sm.qqplot(results.resid, fit = True, line = '45', ax = qqAx) + plt.axis('equal') + if text is not None and 'qqplot.xlabel' in text: + plt.xlabel(text['qqplot.xlabel']) + plt.ylabel(text['qqplot.ylabel']) + plt.tight_layout() + if filenamePrefix is not None: + from storage import openCheck + out = openCheck(filenamePrefix+'-coefficients.html', 'w') + out.write(results.summary().as_html()) + plt.savefig(filenamePrefix+'-model-results.'+figureFileType) + +######################### +# iterable section +######################### + +def mostCommon(L): + '''Returns the most frequent element in a iterable + + taken from http://stackoverflow.com/questions/1518522/python-most-common-element-in-a-list''' + from itertools import groupby + from operator import itemgetter + # get an iterable of (item, iterable) pairs + SL = sorted((x, i) for i, x in enumerate(L)) + # print 'SL:', SL + groups = groupby(SL, key=itemgetter(0)) + # auxiliary function to get "quality" for an item + def _auxfun(g): + item, iterable = g + count = 0 + min_index = len(L) + for _, where in iterable: + count += 1 + min_index = min(min_index, where) + # print 'item %r, count %r, minind %r' % (item, count, min_index) + return count, -min_index + # pick the highest-count/earliest item + return max(groups, key=_auxfun)[0] + +######################### +# sequence section +######################### + +class LCSS(object): + '''Class that keeps the LCSS parameters + and puts together the various computations + + the methods with names starting with _ are not to be shadowed + in child classes, who will shadow the other methods, + ie compute and computeXX methods''' + def __init__(self, similarityFunc = None, metric = None, epsilon = None, delta = float('inf'), aligned = False, lengthFunc = min): + '''One should provide either a similarity function + that indicates (return bool) whether elements in the compares lists are similar + + eg distance(p1, p2) < epsilon + + or a type of metric usable in scipy.spatial.distance.cdist with an epsilon''' + if similarityFunc is None and metric is None: + print("No way to compute LCSS, similarityFunc and metric are None. Exiting") + import sys + sys.exit() + elif metric is not None and epsilon is None: + print("Please provide a value for epsilon if using a cdist metric. Exiting") + import sys + sys.exit() + else: + if similarityFunc is None and metric is not None and not isinf(delta): + print('Warning: you are using a cdist metric and a finite delta, which will make probably computation slower than using the equivalent similarityFunc (since all pairwise distances will be computed by cdist).') + self.similarityFunc = similarityFunc + self.metric = metric + self.epsilon = epsilon + self.aligned = aligned + self.delta = delta + self.lengthFunc = lengthFunc + self.subSequenceIndices = [(0,0)] + + def similarities(self, l1, l2, jshift=0): + n1 = len(l1) + n2 = len(l2) + self.similarityTable = zeros((n1+1,n2+1), dtype = npint) + if self.similarityFunc is not None: + for i in range(1,n1+1): + for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): + if self.similarityFunc(l1[i-1], l2[j-1]): + self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 + else: + self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1]) + elif self.metric is not None: + similarElements = distance.cdist(l1, l2, self.metric) <= self.epsilon + for i in range(1,n1+1): + for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): + if similarElements[i-1, j-1]: + self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 + else: + self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1]) + + + def subSequence(self, i, j): + '''Returns the subsequence of two sequences + http://en.wikipedia.org/wiki/Longest_common_subsequence_problem''' + if i == 0 or j == 0: + return [] + elif self.similarityTable[i][j] == self.similarityTable[i][j-1]: + return self.subSequence(i, j-1) + elif self.similarityTable[i][j] == self.similarityTable[i-1][j]: + return self.subSequence(i-1, j) + else: + return self.subSequence(i-1, j-1) + [(i-1,j-1)] + + def _compute(self, _l1, _l2, computeSubSequence = False): + '''returns the longest common subsequence similarity + l1 and l2 should be the right format + eg list of tuple points for cdist + or elements that can be compare using similarityFunc + + if aligned, returns the best matching if using a finite delta by shifting the series alignments + ''' + if len(_l2) < len(_l1): # l1 is the shortest + l1 = _l2 + l2 = _l1 + revertIndices = True + else: + l1 = _l1 + l2 = _l2 + revertIndices = False + n1 = len(l1) + n2 = len(l2) + + if self.aligned: + lcssValues = {} + similarityTables = {} + for i in range(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2 + self.similarities(l1, l2, i) + lcssValues[i] = self.similarityTable.max() + similarityTables[i] = self.similarityTable + #print self.similarityTable + alignmentShift = argmaxDict(lcssValues) # ideally get the medium alignment shift, the one that minimizes distance + self.similarityTable = similarityTables[alignmentShift] + else: + alignmentShift = 0 + self.similarities(l1, l2) + + # threshold values for the useful part of the similarity table are n2-n1-delta and n1-n2-delta + self.similarityTable = self.similarityTable[:min(n1, n2+alignmentShift+self.delta)+1, :min(n2, n1-alignmentShift+self.delta)+1] + + if computeSubSequence: + self.subSequenceIndices = self.subSequence(self.similarityTable.shape[0]-1, self.similarityTable.shape[1]-1) + if revertIndices: + self.subSequenceIndices = [(j,i) for i,j in self.subSequenceIndices] + return self.similarityTable[-1,-1] + + def compute(self, l1, l2, computeSubSequence = False): + '''get methods are to be shadowed in child classes ''' + return self._compute(l1, l2, computeSubSequence) + + def computeAlignment(self): + return mean([j-i for i,j in self.subSequenceIndices]) + + def _computeNormalized(self, l1, l2, computeSubSequence = False): + ''' compute the normalized LCSS + ie, the LCSS divided by the min or mean of the indicator lengths (using lengthFunc) + lengthFunc = lambda x,y:float(x,y)/2''' + return float(self._compute(l1, l2, computeSubSequence))/self.lengthFunc(len(l1), len(l2)) + + def computeNormalized(self, l1, l2, computeSubSequence = False): + return self._computeNormalized(l1, l2, computeSubSequence) + + def _computeDistance(self, l1, l2, computeSubSequence = False): + ''' compute the LCSS distance''' + return 1-self._computeNormalized(l1, l2, computeSubSequence) + + def computeDistance(self, l1, l2, computeSubSequence = False): + return self._computeDistance(l1, l2, computeSubSequence) + +######################### +# plotting section +######################### + +def plotPolygon(poly, options = '', **kwargs): + 'Plots shapely polygon poly' + from matplotlib.pyplot import plot + x,y = poly.exterior.xy + plot(x, y, options, **kwargs) + +def stepPlot(X, firstX, lastX, initialCount = 0, increment = 1): + '''for each value in X, increment by increment the initial count + returns the lists that can be plotted + to obtain a step plot increasing by one for each value in x, from first to last value + firstX and lastX should be respectively smaller and larger than all elements in X''' + + sortedX = [] + counts = [initialCount] + for x in sorted(X): + sortedX += [x,x] + counts.append(counts[-1]) + counts.append(counts[-1]+increment) + counts.append(counts[-1]) + return [firstX]+sortedX+[lastX], counts + +class PlottingPropertyValues(object): + def __init__(self, values): + self.values = values + + def __getitem__(self, i): + return self.values[i%len(self.values)] + +markers = PlottingPropertyValues(['+', '*', ',', '.', 'x', 'D', 's', 'o']) +scatterMarkers = PlottingPropertyValues(['s','o','^','>','v','<','d','p','h','8','+','x']) + +linestyles = PlottingPropertyValues(['-', '--', '-.', ':']) + +colors = PlottingPropertyValues('brgmyck') # 'w' + +def monochromeCycler(withMarker = False): + from cycler import cycler + if withMarker: + monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', ':', '-.']) * cycler('marker', ['^',',', '.'])) + else: + monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', ':', '-.'])) + plt.rc('axes', prop_cycle=monochrome) + +def plotIndicatorMap(indicatorMap, squareSize, masked = True, defaultValue=-1): + from matplotlib.pyplot import pcolor + coords = array(list(indicatorMap.keys())) + minX = min(coords[:,0]) + minY = min(coords[:,1]) + X = arange(minX, max(coords[:,0])+1.1)*squareSize + Y = arange(minY, max(coords[:,1])+1.1)*squareSize + C = defaultValue*ones((len(Y), len(X))) + for k,v in indicatorMap.items(): + C[k[1]-minY,k[0]-minX] = v + if masked: + pcolor(X, Y, ma.masked_where(C==defaultValue,C)) + else: + pcolor(X, Y, C) + +######################### +# Data download +######################### + +def downloadECWeather(stationID, years, months = [], outputDirectoryname = '.', english = True): + '''Downloads monthly weather data from Environment Canada + If month is provided (number 1 to 12), it means hourly data for the whole month + Otherwise, means the data for each day, for the whole year + + Example: MONTREAL MCTAVISH 10761 + MONTREALPIERRE ELLIOTT TRUDEAU INTL A 5415 + see ftp://client_climate@ftp.tor.ec.gc.ca/Pub/Get_More_Data_Plus_de_donnees/Station%20Inventory%20EN.csv + + To get daily data for 2010 and 2011, downloadECWeather(10761, [2010,2011], [], '/tmp') + To get hourly data for 2009 and 2012, January, March and October, downloadECWeather(10761, [2009,2012], [1,3,10], '/tmp') + + for annee in `seq 2016 2017`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&timeframe=2&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done + for annee in `seq 2016 2017`;do for mois in `seq 1 12`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&Month=${mois}&timeframe=1&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done;done + ''' + import urllib.request + if english: + language = 'e' + else: + language = 'f' + if len(months) == 0: + timeFrame = 2 + months = [1] + else: + timeFrame = 1 + + for year in years: + for month in months: + outFilename = '{}/{}-{}'.format(outputDirectoryname, stationID, year) + if timeFrame == 1: + outFilename += '-{}-hourly'.format(month) + else: + outFilename += '-daily' + outFilename += '.csv' + url = urllib.request.urlretrieve('http://climate.weather.gc.ca/climate_data/bulk_data_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=Download+Data'.format(language, stationID, year, month, timeFrame), outFilename) + +######################### +# File I/O +######################### + +def removeExtension(filename, delimiter = '.'): + '''Returns the filename minus the extension (all characters after last .)''' + i = filename.rfind(delimiter) + if i>0: + return filename[:i] + else: + return filename + +def getExtension(filename, delimiter = '.'): + '''Returns the filename minus the extension (all characters after last .)''' + i = filename.rfind(delimiter) + if i>0: + return filename[i+1:] + else: + return '' + +def cleanFilename(s): + 'cleans filenames obtained when contatenating figure characteristics' + return s.replace(' ','-').replace('.','').replace('/','-').replace(',','') + +def getRelativeFilename(parentPath, filename): + 'Returns filename if absolute, otherwise parentPath/filename as string' + filePath = Path(filename) + if filePath.is_absolute(): + return filename + else: + return str(parentPath/filePath) + +def listfiles(dirname, extension, remove = False): + '''Returns the list of files with the extension in the directory dirname + If remove is True, the filenames are stripped from the extension''' + d = Path(dirname) + if d.is_dir(): + tmp = [str(f) for f in d.glob('*.extension')] + if remove: + return [removeExtension(f, extension) for f in tmp] + else: + return tmp + else: + print(dirname+' is not a directory') + return [] + +def mkdir(dirname): + 'Creates a directory if it does not exist' + p = Path(dirname) + if not p.exists(): + p.mkdir() + else: + print(dirname+' already exists') + +def removeFile(filename): + '''Deletes the file while avoiding raising an error + if the file does not exist''' + f = Path(filename) + if (f.exists()): + f.unlink() + else: + print(filename+' does not exist') + +def line2Floats(l, separator=' '): + '''Returns the list of floats corresponding to the string''' + return [float(x) for x in l.split(separator)] + +def line2Ints(l, separator=' '): + '''Returns the list of ints corresponding to the string''' + return [int(x) for x in l.split(separator)] + +######################### +# Profiling +######################### + +def analyzeProfile(profileFilename, stripDirs = True): + '''Analyze the file produced by cProfile + + obtained by for example: + - call in script (for main() function in script) + import cProfile, os + cProfile.run('main()', os.path.join(os.getcwd(),'main.profile')) + + - or on the command line: + python -m cProfile [-o profile.bin] [-s sort] scriptfile [arg]''' + import pstats, os + p = pstats.Stats(os.path.join(os.pardir, profileFilename)) + if stripDirs: + p.strip_dirs() + p.sort_stats('time') + p.print_stats(.2) + #p.sort_stats('time') + # p.print_callees(.1, 'int_prediction.py:') + return p + +######################### +# running tests +######################### + +if __name__ == "__main__": + import doctest + import unittest + suite = doctest.DocFileSuite('tests/utils.txt') + #suite = doctest.DocTestSuite() + unittest.TextTestRunner().run(suite) + #doctest.testmod() + #doctest.testfile("example.txt")