Mercurial Hosting > traffic-intelligence
comparison scripts/process.py @ 1074:b123fa0e5440
merged
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Fri, 20 Jul 2018 16:24:30 -0400 |
parents | 8ab92ee3cbef c67f8c36ebc7 |
children | 706034a4c6cd |
comparison
equal
deleted
inserted
replaced
1073:8ab92ee3cbef | 1074:b123fa0e5440 |
---|---|
58 # override other tracking config, erase sqlite? | 58 # override other tracking config, erase sqlite? |
59 | 59 |
60 | 60 |
61 # analysis options | 61 # analysis options |
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) | 62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) |
63 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1) | 63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float) |
64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15) | 64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15) |
65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) | 65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) |
66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) | 66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) |
67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float) | 67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float) |
68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data') | 68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data') |
257 headers = ['site', 'date', 'time', 'user_type'] | 257 headers = ['site', 'date', 'time', 'user_type'] |
258 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) | 258 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) |
259 headers.extend(tmpheaders) | 259 headers.extend(tmpheaders) |
260 if args.nProcesses == 1: | 260 if args.nProcesses == 1: |
261 for vs in videoSequences: | 261 for vs in videoSequences: |
262 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) | 262 data.extend(processing.extractVideoSequenceSpeeds(str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) |
263 else: | 263 else: |
264 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, args.minUserDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences] | 264 jobs = [pool.apply_async(processing.extractVideoSequenceSpeeds, args = (str(parentPath/vs.getDatabaseFilename()), vs.cameraView.site.name, args.nObjects, vs.startTime, vs.cameraView.cameraType.frameRate, vs.cameraView.cameraType.frameRate*args.minDuration, args.aggMethods, args.aggCentiles)) for vs in videoSequences] |
265 for job in jobs: | 265 for job in jobs: |
266 data.extend(job.get()) | 266 data.extend(job.get()) |
267 pool.close() | 267 pool.close() |
268 data = pd.DataFrame(data, columns = headers) | 268 data = pd.DataFrame(data, columns = headers) |
269 if args.output == 'figure': | 269 if args.output == 'figure': |
278 data.to_csv(args.eventFilename, index = False) | 278 data.to_csv(args.eventFilename, index = False) |
279 | 279 |
280 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data | 280 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data |
281 indicatorIds = [2,5,7,10] | 281 indicatorIds = [2,5,7,10] |
282 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} | 282 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} |
283 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} | 283 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} |
284 data = [] # list of observation per site-user with time | |
285 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types? | |
286 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) | |
287 for i in indicatorIds[:3]: | |
288 for h in tmpheaders: | |
289 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h) | |
284 indicators = {} | 290 indicators = {} |
285 interactions = {} | 291 interactions = {} |
286 for vs in videoSequences: | 292 for vs in videoSequences: |
287 if not vs.cameraView.siteIdx in interactions: | 293 print('Extracting SMoS from '+vs.getDatabaseFilename()) |
288 interactions[vs.cameraView.siteIdx] = [] | 294 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename())) |
289 indicators[vs.cameraView.siteIdx] = {} | 295 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration |
290 for i in indicatorIds: | 296 for inter in interactions: |
291 indicators[vs.cameraView.siteIdx][i] = [] | 297 if inter.length() > minDuration: |
292 interactions[vs.cameraView.siteIdx] += storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename())) | 298 d = vs.startTime.date() |
293 print(vs.getDatabaseFilename(), len(interactions[vs.cameraView.siteIdx])) | 299 t = vs.startTime.time() |
294 for inter in interactions[vs.cameraView.siteIdx]: | 300 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)] |
295 for i in indicatorIds: | 301 pet = inter.getIndicator('Post Encroachment Time') |
296 indic = inter.getIndicator(events.Interaction.indicatorNames[i]) | 302 if pet is None: |
297 if indic is not None: | 303 row.append(None) |
298 v = indic.getMostSevereValue()*conversionFactors[i] | 304 else: |
299 if v < maxIndicatorValue[i]: | 305 row.append(conversionFactors[10]*pet.getValues()[0]) |
300 indicators[vs.cameraView.siteIdx][i].append(v) | 306 for i in indicatorIds[:3]: |
301 | 307 indic = inter.getIndicator(events.Interaction.indicatorNames[i]) |
302 for i in indicatorIds: | 308 if indic is not None: |
303 tmp = [indicators[siteId][i] for siteId in indicators] | 309 #v = indic.getMostSevereValue()* |
304 plt.ioff() | 310 tmp = list(indic.values.values()) |
305 plt.figure() | 311 for method,func in aggFunctions.items(): |
306 plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators]) | 312 agg = conversionFactors[i]*func(tmp) |
307 plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')') | 313 if method == 'centile': |
308 plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150) | 314 row.extend(agg.tolist()) |
309 plt.close() | 315 else: |
316 row.append(agg) | |
317 else: | |
318 row.extend([None]*len(aggFunctions)) | |
319 data.append(row) | |
320 data = pd.DataFrame(data, columns = headers) | |
321 if args.output == 'figure': | |
322 for i in indicatorIds: | |
323 pass # tmp = [indicators[siteId][i] for siteId in indicators] | |
324 # plt.ioff() | |
325 # plt.figure() | |
326 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators]) | |
327 # plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')') | |
328 # plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150) | |
329 # plt.close() | |
330 elif args.output == 'event': | |
331 data.to_csv(args.eventFilename, index = False) | |
310 | 332 |
311 if args.analyze == 'event': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds | 333 if args.analyze == 'event': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds |
312 data = pd.read_csv(args.eventFilename, parse_dates = [2]) | 334 data = pd.read_csv(args.eventFilename, parse_dates = [2]) |
313 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000) | 335 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000) |
314 # create time for end of each 15 min, then group by, using the agg method for each data column | 336 # create time for end of each 15 min, then group by, using the agg method for each data column |