--- a/classifier.cfg	Mon Feb 05 14:14:14 2024 -0500
+++ b/classifier.cfg	Mon Feb 05 17:06:01 2024 -0500
@@ -2,6 +2,8 @@
 pbv-svm-filename = modelPBV.xml
 # filename of the cyc/veh SVM classifier
 bv-svm-filename = modelBV.xml
+# filename of a Ultralytics-compatible model, eg Yolov8
+dl-filename = 
 # percent increase of the max of width and height of the bounding box of features extracted for classification
 percent-increase-crop = 0.2
 # min number of pixels in cropped image to classify by SVM

--- a/scripts/classify-objects.py	Mon Feb 05 14:14:14 2024 -0500
+++ b/scripts/classify-objects.py	Mon Feb 05 17:06:01 2024 -0500
@@ -5,6 +5,15 @@
 import numpy as np
 import cv2
 from scipy.stats import norm, lognorm
+from pathlib import Path
+
+try:
+    from ultralytics import YOLO
+    ultralyticsAvailable = True
+except ImportError:
+    #print('OpenCV library could not be loaded (video replay functions will not be available)') # TODO change to logging module
+    ultralyticsAvailable = False
+
 
 from trafficintelligence import cvutils, moving, ml, storage, utils
 
@@ -29,8 +38,15 @@
 if speedAggregationFunc is None:
     sys.exit()
 
-pedBikeCarSVM = ml.SVM_load(classifierParams.pedBikeCarSVMFilename)
-bikeCarSVM = ml.SVM_load(classifierParams.bikeCarSVMFilename)
+if ultralyticsAvailable and Path(classifierParams.dlFilename).is_file(): # use Yolo
+    pedBikeCarSVM = None
+    bikeCarSVM = None
+    yolo = YOLO(classifierParams.dlFilename, task='detect')
+    useYolo = True
+else:
+    useYolo = False
+    pedBikeCarSVM = ml.SVM_load(classifierParams.pedBikeCarSVMFilename)
+    bikeCarSVM = ml.SVM_load(classifierParams.bikeCarSVMFilename)
 
 # log logistic for ped and bik otherwise ((pedBeta/pedAlfa)*((sMean/pedAlfa)**(pedBeta-1)))/((1+(sMean/pedAlfa)**pedBeta)**2.)
 carNorm = norm(classifierParams.meanVehicleSpeed, classifierParams.stdVehicleSpeed)
@@ -61,8 +77,6 @@
 
 objects = storage.loadTrajectoriesFromSqlite(databaseFilename, 'object', args.nObjects, withFeatures = True)
 timeInterval = moving.TimeInterval.unionIntervals([obj.getTimeInterval() for obj in objects])
-if args.startFrame0:
-    timeInterval.first = 0
 
 capture = cv2.VideoCapture(videoFilename)
 width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
@@ -81,8 +95,7 @@
 if capture.isOpened():
     ret = True
     frameNum = timeInterval.first
-    if not args.startFrame0:
-        capture.set(cv2.CAP_PROP_POS_FRAMES, frameNum)
+    capture.set(cv2.CAP_PROP_POS_FRAMES, frameNum)
     lastFrameNum = timeInterval.last
 
     while ret and frameNum <= lastFrameNum:
@@ -91,7 +104,10 @@
             if frameNum%50 == 0:
                 print('frame number: {}'.format(frameNum))
             #if undistort:
-            #    img = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR)            
+            #    img = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR)
+            if useYolo:
+                results = yolo.predict(img, classes=list(moving.cocoTypeNames.keys()), verbose=False)
+
             for obj in objects[:]:
                 if obj.getFirstInstant() <= frameNum: # if images are skipped
                     obj.initClassifyUserTypeHoGSVM(speedAggregationFunc, pedBikeCarSVM, bikeCarSVM, classifierParams.maxPedestrianSpeed, classifierParams.maxCyclistSpeed, classifierParams.nFramesIgnoreAtEnds, invHomography, intrinsicCameraMatrix, distortionCoefficients)
@@ -99,12 +115,18 @@
                     objects.remove(obj)
 
             for obj in currentObjects[:]:
-                if obj.getLastInstant() <= frameNum:  # if images are skipped
+                if obj.getLastInstant() <= frameNum:
                     obj.classifyUserTypeHoGSVM(minSpeedEquiprobable = classifierParams.minSpeedEquiprobable, speedProbabilities = speedProbabilities, maxPercentUnknown = classifierParams.maxPercentUnknown)
                     pastObjects.append(obj)
                     currentObjects.remove(obj)
                 else:
-                    obj.classifyUserTypeHoGSVMAtInstant(img, frameNum, width, height, classifierParams.percentIncreaseCrop, classifierParams.percentIncreaseCrop, classifierParams.minNPixels, classifierParams.hogRescaleSize, classifierParams.hogNOrientations, classifierParams.hogNPixelsPerCell, classifierParams.hogNCellsPerBlock, classifierParams.hogBlockNorm)
+                    if useYolo:
+                        # if one feature falls in bike, it's a bike
+                        # could one count all hits in various objects, or one takes majority at the instant?
+                        # obj.classifyUserTypeYoloAtInstant(img, frameNum, width, height, classifierParams.percentIncreaseCrop, classifierParams.percentIncreaseCrop, results[0].boxes)
+                        pass
+                    else:
+                        obj.classifyUserTypeHoGSVMAtInstant(img, frameNum, width, height, classifierParams.percentIncreaseCrop, classifierParams.percentIncreaseCrop, classifierParams.minNPixels, classifierParams.hogRescaleSize, classifierParams.hogNOrientations, classifierParams.hogNPixelsPerCell, classifierParams.hogNCellsPerBlock, classifierParams.hogBlockNorm)
                     if args.verbose:
                         print('obj {}@{}: {}'.format(obj.getNum(), frameNum, moving.userTypeNames[obj.userTypes[frameNum]]))
         frameNum += 1

--- a/trafficintelligence/ml.py	Mon Feb 05 14:14:14 2024 -0500
+++ b/trafficintelligence/ml.py	Mon Feb 05 17:06:01 2024 -0500
@@ -70,6 +70,7 @@
             return svm
         else:
             print('Provided filename {} does not exist: model not loaded!'.format(filename))
+            return None
         
 #####################
 # Clustering

--- a/trafficintelligence/moving.py	Mon Feb 05 14:14:14 2024 -0500
+++ b/trafficintelligence/moving.py	Mon Feb 05 17:06:01 2024 -0500
@@ -305,6 +305,9 @@
         projected = cvutils.homographyProject(array([[self.x], [self.y]]), homography)
         return Point(projected[0], projected[1])
 
+    def inRectangle(self, xmin, xmax, ymin, ymax):
+        return (xmin <= p.x <= xmax) and (ymin <= p.y <= ymax)
+    
     def inPolygon(self, polygon):
         '''Indicates if the point x, y is inside the polygon
         (array of Nx2 coordinates of the polygon vertices)
@@ -2051,12 +2054,12 @@
         self.setUserType(utils.argmaxDict(userTypeProbabilities))
         return userTypeProbabilities
 
-    def initClassifyUserTypeHoGSVM(self, aggregationFunc, pedBikeCarSVM, bikeCarSVM = None, pedBikeSpeedTreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), nInstantsIgnoredAtEnds = 0, homography = None, intrinsicCameraMatrix = None, distortionCoefficients = None):
+    def initClassifyUserTypeHoGSVM(self, aggregationFunc, pedBikeCarSVM, bikeCarSVM = None, pedBikeSpeedThreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), nInstantsIgnoredAtEnds = 0, homography = None, intrinsicCameraMatrix = None, distortionCoefficients = None):
         '''Initializes the data structures for classification
 
         TODO? compute speed for longest feature?'''
         self.aggregatedSpeed = aggregationFunc(self.getSpeeds(nInstantsIgnoredAtEnds))
-        if self.aggregatedSpeed < pedBikeSpeedTreshold or bikeCarSVM is None:
+        if self.aggregatedSpeed < pedBikeSpeedThreshold or bikeCarSVM is None:
             self.appearanceClassifier = pedBikeCarSVM
         elif self.aggregatedSpeed < bikeCarSpeedThreshold:
             self.appearanceClassifier = bikeCarSVM
@@ -2082,7 +2085,24 @@
         else:
             self.userTypes[instant] = userType2Num['unknown']
 
-    def classifyUserTypeHoGSVM(self, pedBikeCarSVM = None, width = 0, height = 0, homography = None, images = None, bikeCarSVM = None, pedBikeSpeedTreshold = float('Inf'), bikeCarSpeedThreshold = float('Inf'), minSpeedEquiprobable = -1, speedProbabilities = None, aggregationFunc = median, maxPercentUnknown = 0.5, nInstantsIgnoredAtEnds = 0, px = 0.2, py = 0.2, minNPixels = 800, rescaleSize = (64, 64), orientations = 9, pixelsPerCell = (8,8), cellsPerBlock = (2,2)):
+    def classifyUserTypeYoloAtInstant(self, img, instant, width, height, px, py, minNPixels, bboxes):
+        '''Finds the user type based on where the feature fall in detected bboxes'''
+        userTypes = []
+        if self.hasFeatures():
+            for f in self.getFeatures():
+                if f.existsAtInstant(frameNum):
+                    p = f.getPositionAtInstant(frameNum)
+                    for box in bboxes:
+                        if box.id is not None:
+                            xyxy = box.xyxy[0].tolist()
+                            if p.inRectangle(xyxy[0], xyxy[1], xyxy[2], xyxy[3]):
+                                userTypes.append(moving.coco2Types[int(box.cls.item())])
+        if len(userTypes) > 0:
+            pass
+        else:
+            self.userTypes[instant] = userType2Num['unknown']
+
+    def classifyUserTypeHoGSVM(self, minSpeedEquiprobable = -1, speedProbabilities = None, aggregationFunc = median, maxPercentUnknown = 0.5):
         '''Agregates SVM detections in each image and returns probability
         (proportion of instants with classification in each category)
 
@@ -2090,14 +2110,14 @@
         With default parameters, the general (ped-bike-car) classifier will be used
 
         Considered categories are the keys of speedProbabilities'''
-        if not hasattr(self, 'aggregatedSpeed') or not hasattr(self, 'userTypes'):
-            print('Initializing the data structures for classification by HoG-SVM')
-            self.initClassifyUserTypeHoGSVM(aggregationFunc, pedBikeCarSVM, bikeCarSVM, pedBikeSpeedTreshold, bikeCarSpeedThreshold, nInstantsIgnoredAtEnds)
+        #if not hasattr(self, 'aggregatedSpeed') or not hasattr(self, 'userTypes'):
+        #    print('Initializing the data structures for classification by HoG-SVM')
+        #    self.initClassifyUserTypeHoGSVM(aggregationFunc, pedBikeCarSVM, bikeCarSVM, pedBikeSpeedTreshold, bikeCarSpeedThreshold, nInstantsIgnoredAtEnds)
 
-        if len(self.userTypes) != self.length() and images is not None: # if classification has not been done previously
-            for t in self.getTimeInterval():
-                if t not in self.userTypes:
-                    self.classifyUserTypeHoGSVMAtInstant(images[t], t, homography, width, height, px, py, minNPixels, rescaleSize, orientations, pixelsPerCell, cellsPerBlock)
+        # if len(self.userTypes) != self.length() and images is not None: # if classification has not been done previously
+        #     for t in self.getTimeInterval():
+        #         if t not in self.userTypes:
+        #             self.classifyUserTypeHoGSVMAtInstant(images[t], t, homography, width, height, px, py, minNPixels, rescaleSize, orientations, pixelsPerCell, cellsPerBlock)
         # compute P(Speed|Class)
         if speedProbabilities is None or self.aggregatedSpeed < minSpeedEquiprobable: # equiprobable information from speed
             userTypeProbabilities = {userType2Num['car']: 1., userType2Num['pedestrian']: 1., userType2Num['bicycle']: 1.}

--- a/trafficintelligence/storage.py	Mon Feb 05 14:14:14 2024 -0500
+++ b/trafficintelligence/storage.py	Mon Feb 05 17:06:01 2024 -0500
@@ -1509,6 +1509,7 @@
 
         self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename'))
         self.bikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'bv-svm-filename'))
+        self.dlFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'dl-filename'))
         self.percentIncreaseCrop = config.getfloat(self.sectionHeader, 'percent-increase-crop')
         self.minNPixels = config.getint(self.sectionHeader, 'min-npixels-crop')
         x = config.getint(self.sectionHeader, 'hog-rescale-size')