Mercurial Hosting > traffic-intelligence

comparison scripts/process.py @ 1072:c67f8c36ebc7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
interaction extraction
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Thu, 19 Jul 2018 01:44:11 -0400
parents 58994b08be42
children b123fa0e5440
comparison
equal deleted inserted replaced
1071:58994b08be42 1072:c67f8c36ebc7
58 # override other tracking config, erase sqlite? 58 # override other tracking config, erase sqlite?
59 59
60 60
61 # analysis options 61 # analysis options
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) 62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
63 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1) 63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float)
64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15) 64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15)
65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) 65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) 66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float) 67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float)
68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data') 68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data')
257 headers = ['site', 'date', 'time', 'user_type'] 257 headers = ['site', 'date', 'time', 'user_type']
258 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) 258 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
259 headers.extend(tmpheaders) 259 headers.extend(tmpheaders)
260 if args.nProcesses == 1: 260 if args.nProcesses == 1:
261 for vs in videoSequences: 261 for vs in videoSequences:
262 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) 262 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles))
263 else: 263 else:
264 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences] 264 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences]
265 for job in jobs: 265 for job in jobs:
266 data.extend(job.get()) 266 data.extend(job.get())
267 pool.close() 267 pool.close()
268 data = pd.DataFrame(data, columns = headers) 268 data = pd.DataFrame(data, columns = headers)
269 if args.output == 'figure': 269 if args.output == 'figure':
278 data.to_csv(args.eventFilename, index = False) 278 data.to_csv(args.eventFilename, index = False)
279 279
280 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data 280 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data
281 indicatorIds = [2,5,7,10] 281 indicatorIds = [2,5,7,10]
282 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} 282 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
283 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} 283 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
284 data = [] # list of observation per site-user with time
285 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types?
286 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
287 for i in indicatorIds[:3]:
288 for h in tmpheaders:
289 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h)
284 indicators = {} 290 indicators = {}
285 interactions = {} 291 interactions = {}
286 for vs in videoSequences: 292 for vs in videoSequences:
287 if not vs.cameraView.siteIdx in interactions: 293 print('Extracting SMoS from '+vs.getDatabaseFilename())
288 interactions[vs.cameraView.siteIdx] = [] 294 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
289 indicators[vs.cameraView.siteIdx] = {} 295 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration
290 for i in indicatorIds: 296 for inter in interactions:
291 indicators[vs.cameraView.siteIdx][i] = [] 297 if inter.length() > minDuration:
292 interactions[vs.cameraView.siteIdx] += storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename())) 298 d = vs.startTime.date()
293 print(vs.getDatabaseFilename(), len(interactions[vs.cameraView.siteIdx])) 299 t = vs.startTime.time()
294 for inter in interactions[vs.cameraView.siteIdx]: 300 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)]
295 for i in indicatorIds: 301 pet = inter.getIndicator('Post Encroachment Time')
296 indic = inter.getIndicator(events.Interaction.indicatorNames[i]) 302 if pet is None:
297 if indic is not None: 303 row.append(None)
298 v = indic.getMostSevereValue()*conversionFactors[i] 304 else:
299 if v < maxIndicatorValue[i]: 305 row.append(conversionFactors[10]*pet.getValues()[0])
300 indicators[vs.cameraView.siteIdx][i].append(v) 306 for i in indicatorIds[:3]:
301 307 indic = inter.getIndicator(events.Interaction.indicatorNames[i])
302 for i in indicatorIds: 308 if indic is not None:
303 tmp = [indicators[siteId][i] for siteId in indicators] 309 #v = indic.getMostSevereValue()*
304 plt.ioff() 310 tmp = list(indic.values.values())
305 plt.figure() 311 for method,func in aggFunctions.items():
306 plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators]) 312 agg = conversionFactors[i]*func(tmp)
307 plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')') 313 if method == 'centile':
308 plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150) 314 row.extend(agg.tolist())
309 plt.close() 315 else:
316 row.append(agg)
317 else:
318 row.extend([None]*len(aggFunctions))
319 data.append(row)
320 data = pd.DataFrame(data, columns = headers)
321 if args.output == 'figure':
322 for i in indicatorIds:
323 pass # tmp = [indicators[siteId][i] for siteId in indicators]
324 # plt.ioff()
325 # plt.figure()
326 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
327 # plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')')
328 # plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150)
329 # plt.close()
330 elif args.output == 'event':
331 data.to_csv(args.eventFilename, index = False)
310 332
311 if args.analyze == 'event': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds 333 if args.analyze == 'event': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
312 data = pd.read_csv(args.eventFilename, parse_dates = [2]) 334 data = pd.read_csv(args.eventFilename, parse_dates = [2])
313 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000) 335 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000)
314 # create time for end of each 15 min, then group by, using the agg method for each data column 336 # create time for end of each 15 min, then group by, using the agg method for each data column