repo/traffic-intelligence: python/utils.py comparison

work on processing large datasets (generate speed data)

author	Nicolas Saunier <nicolas.saunier@polymtl.ca>
date	Wed, 06 Jun 2018 16:51:15 -0400
parents	b7689372c0ec
children

comparison

equal deleted inserted replaced

-:b7689372c0ec
+:a13f47c8931d
 from pathlib import Path
 from math import sqrt, ceil, floor
 from scipy.stats import kruskal, shapiro, lognorm
 from scipy.spatial import distance
 from scipy.sparse import dok_matrix
-from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, median, isnan, ones, convolve,  dtype, isnan, NaN, mean, ma, isinf, savez, load as npload, log
+from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, mean, median, percentile, isnan, ones, convolve,  dtype, isnan, NaN, ma, isinf, savez, load as npload, log
 datetimeFormat = "%Y-%m-%d %H:%M:%S"
 sjcamDatetimeFormat = "%Y_%m%d_%H%M%S"#2017_0626_143720
 data = npload(filename)
 m = dok_matrix(tuple(data['shape']))
 for k, v in zip(data['keys'], data['values']):
 m[tuple(k)] = v
 return m
+def aggregationFunction(funcStr, centile = 50):
+'''return the numpy function corresponding to funcStr
+centile can be a list of centiles to compute at once, eg [25, 50, 75] for the 3 quartiles'''
+if funcStr == 'median':
+return median
+elif funcStr == 'mean':
+return mean
+elif funcStr == 'centile':
+return lambda x: percentile(x, centile)
+elif funcStr == '85centile':
+return lambda x: percentile(x, 85)
+else:
+print('Unknown aggregation method: {}'.format(funcStr))
+return None
 #########################
 # regression analysis using statsmodels (and pandas)
 #########################

Mercurial Hosting > traffic-intelligence