Mercurial Hosting > traffic-intelligence
diff python/utils.py @ 997:4f3387a242a1
updated utils to python 3
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Fri, 25 May 2018 18:15:18 -0400 |
parents | 94bee7b604eb |
children | 16932cefabc1 |
line wrap: on
line diff
--- a/python/utils.py Fri May 25 14:23:56 2018 -0400 +++ b/python/utils.py Fri May 25 18:15:18 2018 -0400 @@ -301,7 +301,7 @@ def sortByLength(instances, reverse = False): '''Returns a new list with the instances sorted by length (method __len__) reverse is passed to sorted''' - return sorted(instances, cmp = compareLengthForSort, reverse = reverse) + return sorted(instances, key = len, reverse = reverse) def ceilDecimals(v, nDecimals): '''Rounds the number at the nth decimal @@ -406,7 +406,7 @@ newVariables.append(newVariable) return newVariables -def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, filenamePrefix = None, figureFileType = 'pdf', saveLatex = False, renameVariables = lambda s: s, kwCaption = u''): +def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, filenamePrefix = None, figureFileType = 'pdf', saveLatex = False, renameVariables = lambda s: s, kwCaption = ''): '''Studies the influence of (nominal) independent variable over the dependent variable Makes tests if the conditional distributions are normal @@ -512,10 +512,10 @@ def saveDokMatrix(filename, m, lowerTriangle = False): 'Saves a dok_matrix using savez' if lowerTriangle: - keys = [k for k in m.keys() if k[0] > k[1]] + keys = [k for k in m if k[0] > k[1]] savez(filename, shape = m.shape, keys = keys, values = [m[k[0],k[1]] for k in keys]) else: - savez(filename, shape = m.shape, keys = m.keys(), values = m.values()) + savez(filename, shape = m.shape, keys = list(m.keys()), values = list(m.values())) def loadDokMatrix(filename): 'Loads a dok_matrix saved using the above saveDokMatrix' @@ -612,17 +612,17 @@ if experiments is None: experiments = generateExperiments(independentVariables) nIndependentVariables = len(independentVariables) - permutation = nppermutation(range(nIndependentVariables)).tolist() + permutation = nppermutation(list(range(nIndependentVariables))) variableMapping = {j: independentVariables[i] for i,j in enumerate(permutation)} - print('Tested variables '+', '.join([variableMapping[i] for i in xrange(nIndependentVariables)])) + print('Tested variables '+', '.join([variableMapping[i] for i in range(nIndependentVariables)])) bestModel = [False]*nIndependentVariables currentVarNum = 0 currentR2Adj = 0. - for currentVarNum in xrange(nIndependentVariables): + for currentVarNum in range(nIndependentVariables): currentModel = [i for i in bestModel] currentModel[currentVarNum] = True - rowIdx = sum([0]+[2**i for i in xrange(nIndependentVariables) if currentModel[permutation[i]]]) - #print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in xrange(nIndependentVariables) if currentModel[permutation[i]]]) + rowIdx = sum([0]+[2**i for i in range(nIndependentVariables) if currentModel[permutation[i]]]) + #print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in range(nIndependentVariables) if currentModel[permutation[i]]]) if experiments.loc[rowIdx, 'shapiroP'] < 0: modelStr = modelString(experiments.loc[rowIdx], dependentVariable, independentVariables) model = modelFunc(modelStr, data = data) @@ -752,16 +752,16 @@ n2 = len(l2) self.similarityTable = zeros((n1+1,n2+1), dtype = npint) if self.similarityFunc is not None: - for i in xrange(1,n1+1): - for j in xrange(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): + for i in range(1,n1+1): + for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): if self.similarityFunc(l1[i-1], l2[j-1]): self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 else: self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1]) elif self.metric is not None: similarElements = distance.cdist(l1, l2, self.metric) <= self.epsilon - for i in xrange(1,n1+1): - for j in xrange(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): + for i in range(1,n1+1): + for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): if similarElements[i-1, j-1]: self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 else: @@ -802,7 +802,7 @@ if self.aligned: lcssValues = {} similarityTables = {} - for i in xrange(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2 + for i in range(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2 self.similarities(l1, l2, i) lcssValues[i] = self.similarityTable.max() similarityTables[i] = self.similarityTable @@ -894,13 +894,13 @@ def plotIndicatorMap(indicatorMap, squareSize, masked = True, defaultValue=-1): from matplotlib.pyplot import pcolor - coords = array(indicatorMap.keys()) + coords = array(list(indicatorMap.keys())) minX = min(coords[:,0]) minY = min(coords[:,1]) X = arange(minX, max(coords[:,0])+1.1)*squareSize Y = arange(minY, max(coords[:,1])+1.1)*squareSize C = defaultValue*ones((len(Y), len(X))) - for k,v in indicatorMap.iteritems(): + for k,v in indicatorMap.items(): C[k[1]-minY,k[0]-minX] = v if masked: pcolor(X, Y, ma.masked_where(C==defaultValue,C)) @@ -926,7 +926,7 @@ for annee in `seq 2016 2017`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&timeframe=2&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done for annee in `seq 2016 2017`;do for mois in `seq 1 12`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&Month=${mois}&timeframe=1&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done;done ''' - import urllib2 + import urllib.request if english: language = 'e' else: @@ -939,18 +939,13 @@ for year in years: for month in months: - url = urllib2.urlopen('http://climate.weather.gc.ca/climate_data/bulk_data_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit= Download+Data'.format(language, stationID, year, month, timeFrame)) - #http://climat.meteo.gc.ca/climateData/bulkdata_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es - data = url.read() outFilename = '{}/{}-{}'.format(outputDirectoryname, stationID, year) if timeFrame == 1: outFilename += '-{}-hourly'.format(month) else: outFilename += '-daily' outFilename += '.csv' - out = open(outFilename, 'w') - out.write(data) - out.close() + url = urllib.request.urlretrieve('http://climate.weather.gc.ca/climate_data/bulk_data_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=Download+Data'.format(language, stationID, year, month, timeFrame), outFilename) ######################### # File I/O @@ -1013,40 +1008,6 @@ return [int(x) for x in l.split(separator)] ######################### -# CLI utils -######################### - -def parseCLIOptions(helpMessage, options, cliArgs, optionalOptions=[]): - ''' Simple function to handle similar argument parsing - Returns the dictionary of options and their values - - * cliArgs are most likely directly sys.argv - (only the elements after the first one are considered) - - * options should be a list of strings for getopt options, - eg ['frame=','correspondences=','video='] - A value must be provided for each option, or the program quits''' - import sys, getopt - from numpy.core.fromnumeric import all - optionValues, args = getopt.getopt(cliArgs[1:], 'h', ['help']+options+optionalOptions) - optionValues = dict(optionValues) - - if '--help' in optionValues.keys() or '-h' in optionValues.keys(): - print(helpMessage+ - '\n - Compulsory options: '+' '.join([opt.replace('=','') for opt in options])+ - '\n - Non-compulsory options: '+' '.join([opt.replace('=','') for opt in optionalOptions])) - sys.exit() - - missingArgument = [('--'+opt.replace('=','') in optionValues.keys()) for opt in options] - if not all(missingArgument): - print('Missing argument') - print(optionValues) - sys.exit() - - return optionValues - - -######################### # Profiling #########################