Mercurial Hosting > traffic-intelligence

annotate python/utils.py @ 671:849f5f8bf4b9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
corrected other missing imports
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Tue, 26 May 2015 11:40:32 +0200
parents f72ed51c6b65
children 5473b7460375
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
1 #! /usr/bin/env python
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
2 ''' Generic utilities.'''
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
3
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
4 import matplotlib.pyplot as plt
397
b36b00dd27c3 added function to read scene metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 395
diff changeset
5 from datetime import time, datetime
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
6 from math import sqrt, ceil, floor
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
7 from scipy.stats import kruskal, shapiro
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
8
421
4fce27946c60 first example of video metadata using sqlalchemy
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 405
diff changeset
9 datetimeFormat = "%Y-%m-%d %H:%M:%S"
4fce27946c60 first example of video metadata using sqlalchemy
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 405
diff changeset
10
185
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
11 #########################
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
12 # Enumerations
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
13 #########################
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
14
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
15 def inverseEnumeration(l):
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
16 'Returns the dictionary that provides for each element in the input list its index in the input list'
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
17 result = {}
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
18 for i,x in enumerate(l):
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
19 result[x] = i
c06379f25ab8 utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 181
diff changeset
20 return result
155
f03fe3d6d0c8 added functions to parse options
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 152
diff changeset
21
f03fe3d6d0c8 added functions to parse options
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 152
diff changeset
22 #########################
637
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
23 # Simple statistics
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
24 #########################
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
25
423
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
26 def sampleSize(stdev, tolerance, percentConfidence, printLatex = False):
301
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
27 from scipy.stats.distributions import norm
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
28 k = round(norm.ppf(0.5+percentConfidence/200., 0, 1)*100)/100. # 1.-(100-percentConfidence)/200.
423
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
29 if printLatex:
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
30 print('${0}^2\\frac{{{1}^2}}{{{2}^2}}$'.format(k, stdev, tolerance))
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
31 return (k*stdev/tolerance)**2
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
32
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
33 def confidenceInterval(mean, stdev, nSamples, percentConfidence, trueStd = True, printLatex = False):
499
0a93afea8243 alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 491
diff changeset
34 '''if trueStd, use normal distribution, otherwise, Student
0a93afea8243 alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 491
diff changeset
35
0a93afea8243 alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 491
diff changeset
36 Use otherwise t.interval or norm.interval
0a93afea8243 alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 491
diff changeset
37 ex: norm.interval(0.95, loc = 0., scale = 2.3/sqrt(11))
0a93afea8243 alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 491
diff changeset
38 t.interval(0.95, 10, loc=1.2, scale = 2.3/sqrt(nSamples))
0a93afea8243 alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 491
diff changeset
39 loc is mean, scale is sigma/sqrt(n) (for Student, 10 is df)'''
423
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
40 from scipy.stats.distributions import norm, t
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
41 if trueStd:
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
42 k = round(norm.ppf(0.5+percentConfidence/200., 0, 1)*100)/100. # 1.-(100-percentConfidence)/200.
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
43 else: # use Student
f738fa1b69f0 added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 421
diff changeset
44 k = round(t.ppf(0.5+percentConfidence/200., nSamples-1)*100)/100.
301
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
45 e = k*stdev/sqrt(nSamples)
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
46 if printLatex:
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
47 print('${0} \pm {1}\\frac{{{2}}}{{\sqrt{{{3}}}}}$'.format(mean, k, stdev, nSamples))
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
48 return mean-e, mean+e
27f06d28036d added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 297
diff changeset
49
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
50 def computeChi2(expected, observed):
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
51 '''Returns the Chi2 statistics'''
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
52 result = 0.
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
53 for e, o in zip(expected, observed):
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
54 result += ((e-o)*(e-o))/e
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
55 return result
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
56
665
15e244d2a1b5 corrected bug with circular import for VideoFilenameAddable, moved to base module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 659
diff changeset
57 class EmpiricalDistribution(object):
85
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
58 def nSamples(self):
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
59 return sum(self.counts)
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
60
588
c5406edbcf12 added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 574
diff changeset
61 def cumulativeDensityFunction(sample, normalized = False):
276
78922b4de3bf minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 266
diff changeset
62 '''Returns the cumulative density function of the sample of a random variable'''
588
c5406edbcf12 added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 574
diff changeset
63 from numpy import arange, cumsum
c5406edbcf12 added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 574
diff changeset
64 xaxis = sorted(sample)
c5406edbcf12 added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 574
diff changeset
65 counts = arange(1,len(sample)+1) # dtype = float
c5406edbcf12 added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 574
diff changeset
66 if normalized:
c5406edbcf12 added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 574
diff changeset
67 counts /= float(len(sample))
197
2788b2827670 simple cumulatie function distribution computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 185
diff changeset
68 return xaxis, counts
85
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
69
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
70 class EmpiricalDiscreteDistribution(EmpiricalDistribution):
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
71 '''Class to represent a sample of a distribution for a discrete random variable
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
72 '''
86
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
73 from numpy.core.fromnumeric import sum
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
74
85
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
75 def __init__(self, categories, counts):
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
76 self.categories = categories
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
77 self.counts = counts
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
78
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
79 def mean(self):
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
80 result = [float(x*y) for x,y in zip(self.categories, self.counts)]
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
81 return sum(result)/self.nSamples()
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
82
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
83 def var(self, mean = None):
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
84 if not mean:
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
85 m = self.mean()
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
86 else:
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
87 m = mean
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
88 result = 0.
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
89 squares = [float((x-m)*(x-m)*y) for x,y in zip(self.categories, self.counts)]
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
90 return sum(squares)/(self.nSamples()-1)
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
91
86
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
92 def referenceCounts(self, probability):
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
93 '''probability is a function that returns the probability of the random variable for the category values'''
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
94 refProba = [probability(c) for c in self.categories]
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
95 refProba[-1] = 1-sum(refProba[:-1])
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
96 refCounts = [r*self.nSamples() for r in refProba]
f03ec4697a09 corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 85
diff changeset
97 return refCounts, refProba
85
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
98
7f1e54234f96 added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 77
diff changeset
99 class EmpiricalContinuousDistribution(EmpiricalDistribution):
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
100 '''Class to represent a sample of a distribution for a continuous random variable
76
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
101 with the number of observations for each interval
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
102 intervals (categories variable) are defined by their left limits, the last one being the right limit
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
103 categories contain therefore one more element than the counts'''
35
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
104 def __init__(self, categories, counts):
276
78922b4de3bf minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 266
diff changeset
105 # todo add samples for initialization and everything to None? (or setSamples?)
35
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
106 self.categories = categories
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
107 self.counts = counts
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
108
35
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
109 def mean(self):
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
110 result = 0.
76
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
111 for i in range(len(self.counts)-1):
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
112 result += self.counts[i]*(self.categories[i]+self.categories[i+1])/2
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
113 return result/self.nSamples()
35
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
114
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
115 def var(self, mean = None):
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
116 if not mean:
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
117 m = self.mean()
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
118 else:
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
119 m = mean
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
120 result = 0.
76
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
121 for i in range(len(self.counts)-1):
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
122 mid = (self.categories[i]+self.categories[i+1])/2
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
123 result += self.counts[i]*(mid - m)*(mid - m)
35
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
124 return result/(self.nSamples()-1)
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
125
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
126 def referenceCounts(self, cdf):
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
127 '''cdf is a cumulative distribution function
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
128 returning the probability of the variable being less that x'''
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
129 # refCumulativeCounts = [0]#[cdf(self.categories[0][0])]
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
130 # for inter in self.categories:
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
131 # refCumulativeCounts.append(cdf(inter[1]))
76
64fde2b1f96d simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 75
diff changeset
132 refCumulativeCounts = [cdf(x) for x in self.categories[1:-1]]
35
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
133
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
134 refProba = [refCumulativeCounts[0]]
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
135 for i in xrange(1,len(refCumulativeCounts)):
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
136 refProba.append(refCumulativeCounts[i]-refCumulativeCounts[i-1])
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
137 refProba.append(1-refCumulativeCounts[-1])
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
138 refCounts = [p*self.nSamples() for p in refProba]
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
139
8cafee54466f forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 32
diff changeset
140 return refCounts, refProba
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
141
77
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
142 def printReferenceCounts(self, refCounts=None):
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
143 if refCounts:
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
144 ref = refCounts
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
145 else:
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
146 ref = self.referenceCounts
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
147 for i in xrange(len(ref[0])):
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
148 print('{0}-{1} & {2:0.3} & {3:0.3} \\\\'.format(self.categories[i],self.categories[i+1],ref[1][i], ref[0][i]))
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
149
5e6cd36a991c added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 76
diff changeset
150
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
151 #########################
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
152 # maths section
27
44689029a86f updated segmentIntersection and other
Nicolas Saunier <nico@confins.net>
parents: 24
diff changeset
153 #########################
24
6fb59cfb201e first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents: 19
diff changeset
154
433
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
155 # def kernelSmoothing(sampleX, X, Y, weightFunc, halfwidth):
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
156 # '''Returns a smoothed weighted version of Y at the predefined values of sampleX
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
157 # Sum_x weight(sample_x,x) * y(x)'''
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
158 # from numpy import zeros, array
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
159 # smoothed = zeros(len(sampleX))
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
160 # for i,x in enumerate(sampleX):
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
161 # weights = array([weightFunc(x,xx, halfwidth) for xx in X])
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
162 # if sum(weights)>0:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
163 # smoothed[i] = sum(weights*Y)/sum(weights)
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
164 # else:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
165 # smoothed[i] = 0
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
166 # return smoothed
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
167
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
168 def kernelSmoothing(x, X, Y, weightFunc, halfwidth):
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
169 '''Returns the smoothed estimate of (X,Y) at x
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
170 Sum_x weight(sample_x,x) * y(x)'''
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
171 from numpy import zeros, array
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
172 weights = array([weightFunc(x,observedx, halfwidth) for observedx in X])
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
173 if sum(weights)>0:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
174 return sum(weights*Y)/sum(weights)
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
175 else:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
176 return 0
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
177
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
178 def uniform(center, x, halfwidth):
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
179 if abs(center-x)<halfwidth:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
180 return 1.
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
181 else:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
182 return 0.
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
183
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
184 def gaussian(center, x, halfwidth):
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
185 from numpy import exp
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
186 return exp(-((center-x)/halfwidth)**2/2)
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
187
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
188 def epanechnikov(center, x, halfwidth):
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
189 diff = abs(center-x)
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
190 if diff<halfwidth:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
191 return 1.-(diff/halfwidth)**2
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
192 else:
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
193 return 0.
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
194
434
9a714f32fc9f small updates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 433
diff changeset
195 def triangular(center, x, halfwidth):
9a714f32fc9f small updates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 433
diff changeset
196 diff = abs(center-x)
9a714f32fc9f small updates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 433
diff changeset
197 if diff<halfwidth:
9a714f32fc9f small updates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 433
diff changeset
198 return 1.-abs(diff/halfwidth)
9a714f32fc9f small updates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 433
diff changeset
199 else:
9a714f32fc9f small updates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 433
diff changeset
200 return 0.
433
d40ad901b272 added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 423
diff changeset
201
518
0c86c73f3c09 median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 511
diff changeset
202 def medianSmoothing(x, X, Y, halfwidth):
0c86c73f3c09 median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 511
diff changeset
203 '''Returns the media of Y's corresponding to X's in the interval [x-halfwidth, x+halfwidth]'''
0c86c73f3c09 median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 511
diff changeset
204 from numpy import median
0c86c73f3c09 median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 511
diff changeset
205 return median([y for observedx, y in zip(X,Y) if abs(x-observedx)<halfwidth])
0c86c73f3c09 median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 511
diff changeset
206
521
3707eeb20f25 changed argMaxDict name to argmaxDict
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 518
diff changeset
207 def argmaxDict(d):
561
ee45c6eb6d49 added Mohamed Gomaa Mohamed function to smooth object trajectories
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 553
diff changeset
208 return max(d, key=d.get)
279
3af4c267a7bf generic simple LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 276
diff changeset
209
395
6fba1ab040f1 minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 391
diff changeset
210 def framesToTime(nFrames, frameRate, initialTime = time()):
6fba1ab040f1 minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 391
diff changeset
211 '''returns a datetime.time for the time in hour, minutes and seconds
6fba1ab040f1 minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 391
diff changeset
212 initialTime is a datetime.time'''
6fba1ab040f1 minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 391
diff changeset
213 seconds = int(floor(float(nFrames)/float(frameRate))+initialTime.hour*3600+initialTime.minute*60+initialTime.second)
261
4aa792cb0fa9 changing framesToTime to return a datetime.time
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 248
diff changeset
214 h = int(floor(seconds/3600.))
248
571ba5ed22e2 added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 241
diff changeset
215 seconds = seconds - h*3600
261
4aa792cb0fa9 changing framesToTime to return a datetime.time
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 248
diff changeset
216 m = int(floor(seconds/60))
248
571ba5ed22e2 added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 241
diff changeset
217 seconds = seconds - m*60
262
a048066bd20f correcting bug in framesToTime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 261
diff changeset
218 return time(h, m, seconds)
248
571ba5ed22e2 added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 241
diff changeset
219
381
387cc0142211 script to replay event annotations
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 376
diff changeset
220 def timeToFrames(t, frameRate):
387cc0142211 script to replay event annotations
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 376
diff changeset
221 return frameRate*(t.hour*3600+t.minute*60+t.second)
387cc0142211 script to replay event annotations
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 376
diff changeset
222
241
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
223 def sortXY(X,Y):
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
224 'returns the sorted (x, Y(x)) sorted on X'
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
225 D = {}
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
226 for x, y in zip(X,Y):
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
227 D[x]=y
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
228 xsorted = sorted(D.keys())
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
229 return xsorted, [D[x] for x in xsorted]
ee1caff48b03 added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 235
diff changeset
230
32
48e56179c39e added ceil function
Nicolas Saunier <nico@confins.net>
parents: 31
diff changeset
231 def ceilDecimals(v, nDecimals):
48e56179c39e added ceil function
Nicolas Saunier <nico@confins.net>
parents: 31
diff changeset
232 '''Rounds the number at the nth decimal
48e56179c39e added ceil function
Nicolas Saunier <nico@confins.net>
parents: 31
diff changeset
233 eg 1.23 at 0 decimal is 2, at 1 decimal is 1.3'''
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
234 tens = 10**nDecimals
32
48e56179c39e added ceil function
Nicolas Saunier <nico@confins.net>
parents: 31
diff changeset
235 return ceil(v*tens)/tens
48e56179c39e added ceil function
Nicolas Saunier <nico@confins.net>
parents: 31
diff changeset
236
152
74b1fc68d4df re-organized code to avoid cyclic python module dependencies
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 116
diff changeset
237 def inBetween(bound1, bound2, x):
569
0057c04f94d5 work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 561
diff changeset
238 return bound1 <= x <= bound2 or bound2 <= x <= bound1
0057c04f94d5 work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 561
diff changeset
239
0057c04f94d5 work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 561
diff changeset
240 def pointDistanceL2(x1,y1,x2,y2):
0057c04f94d5 work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 561
diff changeset
241 ''' Compute point-to-point distance (L2 norm, ie Euclidean distance)'''
0057c04f94d5 work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 561
diff changeset
242 return sqrt((x2-x1)**2+(y2-y1)**2)
24
6fb59cfb201e first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents: 19
diff changeset
243
6fb59cfb201e first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents: 19
diff changeset
244 def crossProduct(l1, l2):
6fb59cfb201e first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents: 19
diff changeset
245 return l1[0]*l2[1]-l1[1]*l2[0]
6fb59cfb201e first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents: 19
diff changeset
246
574
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
247 def cat_mvgavg(cat_list, halfWidth):
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
248 ''' Return a list of categories/values smoothed according to a window.
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
249 halfWidth is the search radius on either side'''
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
250 from copy import deepcopy
659
784298512b60 minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 637
diff changeset
251 smoothed = deepcopy(cat_list)
784298512b60 minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 637
diff changeset
252 for point in range(len(cat_list)):
574
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
253 lower_bound_check = max(0,point-halfWidth)
659
784298512b60 minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 637
diff changeset
254 upper_bound_check = min(len(cat_list)-1,point+halfWidth+1)
784298512b60 minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 637
diff changeset
255 window_values = cat_list[lower_bound_check:upper_bound_check]
574
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
256 smoothed[point] = max(set(window_values), key=window_values.count)
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
257 return smoothed
e24eeb244698 first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 569
diff changeset
258
547
97c5fef5b2d6 corrected bugs
MohamedGomaa
parents: 521
diff changeset
259 def filterMovingWindow(inputSignal, halfWidth):
29
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
260 '''Returns an array obtained after the smoothing of the input by a moving average
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
261 The first and last points are copied from the original.'''
547
97c5fef5b2d6 corrected bugs
MohamedGomaa
parents: 521
diff changeset
262 from numpy import ones,convolve,array
29
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
263 width = float(halfWidth*2+1)
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
264 win = ones(width,'d')
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
265 result = convolve(win/width,array(inputSignal),'same')
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
266 result[:halfWidth] = inputSignal[:halfWidth]
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
267 result[-halfWidth:] = inputSignal[-halfWidth:]
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
268 return result
ca8e716cc231 added moving average filter
Nicolas Saunier <nico@confins.net>
parents: 27
diff changeset
269
199
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
270 def linearRegression(x, y, deg = 1, plotData = False):
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
271 '''returns the least square estimation of the linear regression of y = ax+b
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
272 as well as the plot'''
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
273 from numpy.lib.polynomial import polyfit
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
274 from numpy.core.multiarray import arange
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
275 coef = polyfit(x, y, deg)
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
276 if plotData:
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
277 def poly(x):
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
278 result = 0
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
279 for i in range(len(coef)):
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
280 result += coef[i]*x**(len(coef)-i-1)
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
281 return result
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
282 plt.plot(x, y, 'x')
199
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
283 xx = arange(min(x), max(x),(max(x)-min(x))/1000)
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
284 plt.plot(xx, [poly(z) for z in xx])
199
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
285 return coef
ca9d9104afba added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 197
diff changeset
286
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
287 def correlation(data, correlationMethod = 'pearson', plotFigure = False, displayNames = None, figureFilename = None):
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
288 '''Computes (and displays) the correlation matrix for a pandas DataFrame'''
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
289 c=data.corr(correlationMethod)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
290 if plotFigure:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
291 fig = plt.figure(figsize=(2+0.4*c.shape[0], 0.4*c.shape[0]))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
292 fig.add_subplot(1,1,1)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
293 #plt.imshow(np.fabs(c), interpolation='none')
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
294 plt.imshow(c, vmin=-1., vmax = 1., interpolation='none', cmap = 'RdYlBu_r') # coolwarm
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
295 colnames = [displayNames.get(s.strip(), s.strip()) for s in c.columns.tolist()]
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
296 #correlation.plot_corr(c, xnames = colnames, normcolor=True, title = filename)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
297 plt.xticks(range(len(colnames)), colnames, rotation=90)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
298 plt.yticks(range(len(colnames)), colnames)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
299 plt.tick_params('both', length=0)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
300 plt.subplots_adjust(bottom = 0.29)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
301 plt.colorbar()
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
302 plt.title('Correlation ({})'.format(correlationMethod))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
303 plt.tight_layout()
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
304 if figureFilename is not None:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
305 plt.savefig(figureFilename, dpi = 150, transparent = True)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
306 return c
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
307
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
308 def addDummies(data, variables, allVariables = True):
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
309 '''Add binary dummy variables for each value of a nominal variable
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
310 in a pandas DataFrame'''
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
311 from numpy import NaN, dtype
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
312 newVariables = []
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
313 for var in variables:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
314 if var in data.columns and data.dtypes[var] == dtype('O') and len(data[var].unique()) > 2:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
315 values = data[var].unique()
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
316 if not allVariables:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
317 values = values[:-1]
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
318 for val in values:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
319 if val is not NaN:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
320 newVariable = (var+'_{}'.format(val)).replace('.','').replace(' ','').replace('-','')
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
321 data[newVariable] = (data[var] == val)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
322 newVariables.append(newVariable)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
323 return newVariables
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
324
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
325 def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, figureFilenamePrefix = None, figureFileType = 'pdf'):
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
326 '''Studies the influence of (nominal) independent variable over the dependent variable
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
327
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
328 Makes tests if the conditional distributions are normal
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
329 using the Shapiro-Wilk test (in which case ANOVA could be used)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
330 Implements uses the non-parametric Kruskal Wallis test'''
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
331 tmp = data[data[independentVariable].notnull()]
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
332 independentVariableValues = sorted(tmp[independentVariable].unique().tolist())
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
333 if len(independentVariableValues) >= 2:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
334 for x in independentVariableValues:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
335 print('Shapiro-Wilk normality test for {} when {}={}: {} obs'.format(dependentVariable,independentVariable, x, len(tmp.loc[tmp[independentVariable] == x, dependentVariable])))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
336 if len(tmp.loc[tmp[independentVariable] == x, dependentVariable]) >= 3:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
337 print shapiro(tmp.loc[tmp[independentVariable] == x, dependentVariable])
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
338 if plotFigure:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
339 plt.figure()
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
340 plt.boxplot([tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues])
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
341 #q25, q75 = tmp[dependentVariable].quantile([.25, .75])
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
342 #plt.ylim(ymax = q75+1.5*(q75-q25))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
343 plt.xticks(range(1,len(independentVariableValues)+1), independentVariableValues)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
344 plt.title('{} vs {}'.format(dependentVariable, independentVariable))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
345 if figureFilenamePrefix is not None:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
346 plt.savefig(figureFilenamePrefix+'{}-{}.{}'.format(dependentVariable, independentVariable, figureFileType))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
347 #else:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
348 # TODO formatter le tableau (html?)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
349 print tmp.groupby([independentVariable])[dependentVariable].describe().unstack().sort(['50%'], ascending = False)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
350 return kruskal(*[tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues])
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
351 else:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
352 return None
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
353
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
354 def prepareRegression(data, dependentVariable, independentVariables, maxCorrelationThreshold, correlations, maxCorrelationP, correlationFunc):
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
355 '''Removes variables from candidate independent variables if
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
356 - if two independent variables are correlated (> maxCorrelationThreshold), one is removed
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
357 - if an independent variable is not correlated with the dependent variable (p>maxCorrelationP)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
358 Returns the remaining non-correlated variables, correlated with the dependent variable
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
359
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
360 correlationFunc is spearmanr or pearsonr from scipy.stats
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
361
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
362 TODO: pass the dummies for nominal variables and remove if all dummies are correlated, or none is correlated with the dependentvariable'''
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
363 from numpy import dtype
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
364 from copy import copy
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
365 result = copy(independentVariables)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
366 for v1 in independentVariables:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
367 if v1 in correlations.index:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
368 for v2 in independentVariables:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
369 if v2 != v1 and v2 in correlations.index:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
370 if abs(correlations.loc[v1, v2]) > maxCorrelationThreshold:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
371 if v1 in result and v2 in result:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
372 print('Removing {} (correlation {} with {})'.format(v2, correlations.loc[v1, v2], v1))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
373 result.remove(v2)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
374 #regressionIndependentVariables = result
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
375 for var in copy(result):
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
376 if data.dtypes[var] != dtype('O'):
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
377 cor, p = correlationFunc(data[dependentVariable], data[var])
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
378 if p > maxCorrelationP:
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
379 print('Removing {} (no correlation p={})'.format(var, p))
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
380 result.remove(var)
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
381 return result
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
382
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
383
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
384 #########################
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
385 # regression analysis using statsmodels (and pandas)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
386 #########################
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
387
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
388 # TODO make class for experiments?
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
389 # TODO add tests with public dataset downloaded from Internet (IRIS et al)
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
390 def modelString(experiment, dependentVariable, independentVariables):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
391 return dependentVariable+' ~ '+' + '.join([independentVariable for independentVariable in independentVariables if experiment[independentVariable]])
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
392
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
393 def runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols'):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
394 import statsmodels.formula.api as smf
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
395 modelStr = modelString(experiment, dependentVariable, independentVariables)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
396 if regressionType == 'ols':
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
397 model = smf.ols(modelStr, data = data)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
398 elif regressionType == 'gls':
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
399 model = smf.gls(modelStr, data = data)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
400 elif regressionType == 'rlm':
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
401 model = smf.rlm(modelStr, data = data)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
402 else:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
403 print('Unknown regression type {}. Exiting'.format(regressionType))
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
404 import sys
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
405 sys.exit()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
406 return model.fit()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
407
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
408 def runModels(experiments, data, dependentVariable, independentVariables, regressionType = 'ols'):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
409 '''Runs several models and stores 3 statistics
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
410 adjusted R2, condition number (should be small, eg < 1000)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
411 and p-value for Shapiro-Wilk test of residual normality'''
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
412 for i,experiment in experiments.iterrows():
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
413 if experiment[independentVariables].any():
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
414 results = runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
415 experiments.loc[i,'r2adj'] = results.rsquared_adj
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
416 experiments.loc[i,'condNum'] = results.condition_number
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
417 experiments.loc[i, 'shapiroP'] = shapiro(results.resid)[1]
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
418 experiments.loc[i,'nobs'] = int(results.nobs)
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
419 return experiments
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
420
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
421 def generateExperiments(independentVariables):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
422 '''Generates all possible models for including or not each independent variable'''
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
423 from numpy import nan
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
424 from pandas import DataFrame
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
425 experiments = {}
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
426 nIndependentVariables = len(independentVariables)
669
df6be882f325 bug corrected
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 668
diff changeset
427 if nIndependentVariables != len(set(independentVariables)):
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
428 print("Duplicate variables. Exiting")
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
429 import sys
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
430 sys.exit()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
431 nModels = 2**nIndependentVariables
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
432 for i,var in enumerate(independentVariables):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
433 pattern = [False]*(2**i)+[True]*(2**i)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
434 experiments[var] = pattern*(2**(nIndependentVariables-i-1))
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
435 experiments = DataFrame(experiments)
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
436 experiments['r2adj'] = 0.
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
437 experiments['condNum'] = nan
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
438 experiments['shapiroP'] = -1
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
439 experiments['nobs'] = -1
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
440 return experiments
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
441
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
442 def findBestModel(data, dependentVariable, independentVariables, regressionType = 'ols', nProcesses = 1):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
443 '''Generates all possible model with the independentVariables
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
444 and runs them, saving the results in experiments
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
445 with multiprocess option'''
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
446 from pandas import concat
671
849f5f8bf4b9 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 670
diff changeset
447 from multiprocessing import Pool
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
448 experiments = generateExperiments(independentVariables)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
449 nModels = len(experiments)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
450 print("Running {} models with {} processes".format(nModels, nProcesses))
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
451 if nProcesses == 1:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
452 return runModels(experiments, data, dependentVariable, independentVariables, regressionType)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
453 else:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
454 pool = Pool(processes = nProcesses)
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
455 chunkSize = int(ceil(nModels/nProcesses))
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
456 jobs = [pool.apply_async(runModels, args = (experiments[i*chunkSize:(i+1)*chunkSize], data, dependentVariable, independentVariables, regressionType)) for i in range(nProcesses)]
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
457 return concat([job.get() for job in jobs])
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
458
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
459 def findBestModelFwd(data, dependentVariable, independentVariables, modelFunc, experiments = None):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
460 '''Forward search for best model (based on adjusted R2)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
461 Randomly starting with one variable and adding randomly variables
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
462 if they improve the model
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
463
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
464 The results are added to experiments if provided as argument
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
465 Storing in experiment relies on the index being the number equal
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
466 to the binary code derived from the independent variables'''
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
467 from numpy.random import permutation as nppermutation
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
468 if experiments is None:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
469 experiments = generateExperiments(independentVariables)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
470 nIndependentVariables = len(independentVariables)
670
f72ed51c6b65 corrected other missing imports
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 669
diff changeset
471 permutation = nppermutation(range(nIndependentVariables)).tolist()
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
472 variableMapping = {j: independentVariables[i] for i,j in enumerate(permutation)}
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
473 print('Tested variables '+', '.join([variableMapping[i] for i in xrange(nIndependentVariables)]))
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
474 bestModel = [False]*nIndependentVariables
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
475 currentVarNum = 0
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
476 currentR2Adj = 0.
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
477 for currentVarNum in xrange(nIndependentVariables):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
478 currentModel = [i for i in bestModel]
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
479 currentModel[currentVarNum] = True
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
480 rowIdx = sum([0]+[2**i for i in xrange(nIndependentVariables) if currentModel[permutation[i]]])
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
481 #print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in xrange(nIndependentVariables) if currentModel[permutation[i]]])
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
482 if experiments.loc[rowIdx, 'shapiroP'] < 0:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
483 modelStr = modelString(experiments.loc[rowIdx], dependentVariable, independentVariables)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
484 model = modelFunc(modelStr, data = data)
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
485 results = model.fit()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
486 experiments.loc[rowIdx, 'r2adj'] = results.rsquared_adj
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
487 experiments.loc[rowIdx, 'condNum'] = results.condition_number
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
488 experiments.loc[rowIdx, 'shapiroP'] = shapiro(results.resid)[1]
668
f8dcf483b296 code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 667
diff changeset
489 experiments.loc[rowIdx, 'nobs'] = int(results.nobs)
667
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
490 if currentR2Adj < experiments.loc[rowIdx, 'r2adj']:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
491 currentR2Adj = experiments.loc[rowIdx, 'r2adj']
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
492 bestModel[currentVarNum] = True
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
493 return experiments
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
494
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
495 def displayModelResults(results, model = None):
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
496 import statsmodels.api as sm
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
497 '''Displays some model results'''
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
498 print results.summary()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
499 print('Shapiro-Wilk normality test for residuals: {}'.format(shapiro(results.resid)))
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
500 if model is not None:
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
501 plt.figure()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
502 plt.plot(results.predict(), model.endog, 'x')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
503 x=plt.xlim()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
504 y=plt.ylim()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
505 plt.plot([max(x[0], y[0]), min(x[1], y[1])], [max(x[0], y[0]), min(x[1], y[1])], 'r')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
506 plt.title('true vs predicted')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
507 plt.figure()
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
508 plt.plot(results.predict(), results.resid, 'x')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
509 plt.title('residuals vs predicted')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
510 sm.qqplot(results.resid, fit = True, line = '45')
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
511
179b81faa1f8 added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 665
diff changeset
512
27
44689029a86f updated segmentIntersection and other
Nicolas Saunier <nico@confins.net>
parents: 24
diff changeset
513 #########################
455
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
514 # iterable section
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
515 #########################
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
516
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
517 def mostCommon(L):
456
825e5d49325d slight update
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 455
diff changeset
518 '''Returns the most frequent element in a iterable
825e5d49325d slight update
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 455
diff changeset
519
825e5d49325d slight update
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 455
diff changeset
520 taken from http://stackoverflow.com/questions/1518522/python-most-common-element-in-a-list'''
455
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
521 from itertools import groupby
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
522 from operator import itemgetter
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
523 # get an iterable of (item, iterable) pairs
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
524 SL = sorted((x, i) for i, x in enumerate(L))
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
525 # print 'SL:', SL
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
526 groups = groupby(SL, key=itemgetter(0))
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
527 # auxiliary function to get "quality" for an item
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
528 def _auxfun(g):
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
529 item, iterable = g
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
530 count = 0
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
531 min_index = len(L)
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
532 for _, where in iterable:
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
533 count += 1
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
534 min_index = min(min_index, where)
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
535 # print 'item %r, count %r, minind %r' % (item, count, min_index)
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
536 return count, -min_index
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
537 # pick the highest-count/earliest item
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
538 return max(groups, key=_auxfun)[0]
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
539
abe0b2347d4c added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 434
diff changeset
540 #########################
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
541 # sequence section
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
542 #########################
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
543
665
15e244d2a1b5 corrected bug with circular import for VideoFilenameAddable, moved to base module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 659
diff changeset
544 class LCSS(object):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
545 '''Class that keeps the LCSS parameters
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
546 and puts together the various computations'''
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
547 def __init__(self, similarityFunc, delta = float('inf'), aligned = False, lengthFunc = min):
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
548 self.similarityFunc = similarityFunc
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
549 self.aligned = aligned
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
550 self.delta = delta
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
551 self.lengthFunc = lengthFunc
389
6d26dcc7bba0 modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 381
diff changeset
552 self.subSequenceIndices = [(0,0)]
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
553
373
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
554 def similarities(self, l1, l2, jshift=0):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
555 from numpy import zeros, int as npint
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
556 n1 = len(l1)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
557 n2 = len(l2)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
558 self.similarityTable = zeros((n1+1,n2+1), dtype = npint)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
559 for i in xrange(1,n1+1):
374
a7af3519687e finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 373
diff changeset
560 for j in xrange(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
561 if self.similarityFunc(l1[i-1], l2[j-1]):
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
562 self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
563 else:
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
564 self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1])
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
565
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
566 def subSequence(self, i, j):
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
567 '''Returns the subsequence of two sequences
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
568 http://en.wikipedia.org/wiki/Longest_common_subsequence_problem'''
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
569 if i == 0 or j == 0:
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
570 return []
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
571 elif self.similarityTable[i][j] == self.similarityTable[i][j-1]:
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
572 return self.subSequence(i, j-1)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
573 elif self.similarityTable[i][j] == self.similarityTable[i-1][j]:
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
574 return self.subSequence(i-1, j)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
575 else:
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
576 return self.subSequence(i-1, j-1) + [(i-1,j-1)]
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
577
373
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
578 def _compute(self, _l1, _l2, computeSubSequence = False):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
579 '''returns the longest common subsequence similarity
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
580 based on the threshold on distance between two elements of lists l1, l2
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
581 similarityFunc returns True or False whether the two points are considered similar
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
582
607
84690dfe5560 add some functions for behaviour analysis
MohamedGomaa
parents: 574
diff changeset
583 if aligned, returns the best matching if using a finite delta by shifting the series alignments
373
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
584
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
585 eg distance(p1, p2) < epsilon
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
586 '''
372
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
587 if len(_l2) < len(_l1): # l1 is the shortest
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
588 l1 = _l2
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
589 l2 = _l1
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
590 revertIndices = True
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
591 else:
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
592 l1 = _l1
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
593 l2 = _l2
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
594 revertIndices = False
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
595 n1 = len(l1)
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
596 n2 = len(l2)
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
597
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
598 if self.aligned:
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
599 lcssValues = {}
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
600 similarityTables = {}
374
a7af3519687e finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 373
diff changeset
601 for i in xrange(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2
373
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
602 self.similarities(l1, l2, i)
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
603 lcssValues[i] = self.similarityTable.max()
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
604 similarityTables[i] = self.similarityTable
374
a7af3519687e finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 373
diff changeset
605 #print self.similarityTable
521
3707eeb20f25 changed argMaxDict name to argmaxDict
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 518
diff changeset
606 alignmentShift = argmaxDict(lcssValues) # ideally get the medium alignment shift, the one that minimizes distance
389
6d26dcc7bba0 modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 381
diff changeset
607 self.similarityTable = similarityTables[alignmentShift]
372
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
608 else:
389
6d26dcc7bba0 modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 381
diff changeset
609 alignmentShift = 0
372
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
610 self.similarities(l1, l2)
373
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
611
374
a7af3519687e finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 373
diff changeset
612 # threshold values for the useful part of the similarity table are n2-n1-delta and n1-n2-delta
389
6d26dcc7bba0 modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 381
diff changeset
613 self.similarityTable = self.similarityTable[:min(n1, n2+alignmentShift+self.delta)+1, :min(n2, n1-alignmentShift+self.delta)+1]
373
d0b86ed50f32 work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 372
diff changeset
614
372
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
615 if computeSubSequence:
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
616 self.subSequenceIndices = self.subSequence(self.similarityTable.shape[0]-1, self.similarityTable.shape[1]-1)
371
924e38c9f70e work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 370
diff changeset
617 if revertIndices:
374
a7af3519687e finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 373
diff changeset
618 self.subSequenceIndices = [(j,i) for i,j in self.subSequenceIndices]
372
349eb1e09f45 Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 371
diff changeset
619 return self.similarityTable[-1,-1]
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
620
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
621 def compute(self, l1, l2, computeSubSequence = False):
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
622 '''get methods are to be shadowed in child classes '''
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
623 return self._compute(l1, l2, computeSubSequence)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
624
375
2ea8584aa80a making indicator LCSS work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 374
diff changeset
625 def computeAlignment(self):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
626 from numpy import mean
374
a7af3519687e finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 373
diff changeset
627 return mean([j-i for i,j in self.subSequenceIndices])
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
628
376
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
629 def _computeNormalized(self, l1, l2, computeSubSequence = False):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
630 ''' compute the normalized LCSS
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
631 ie, the LCSS divided by the min or mean of the indicator lengths (using lengthFunc)
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
632 lengthFunc = lambda x,y:float(x,y)/2'''
376
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
633 return float(self._compute(l1, l2, computeSubSequence))/self.lengthFunc(len(l1), len(l2))
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
634
376
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
635 def computeNormalized(self, l1, l2, computeSubSequence = False):
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
636 return self._computeNormalized(l1, l2, computeSubSequence)
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
637
376
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
638 def _computeDistance(self, l1, l2, computeSubSequence = False):
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
639 ''' compute the LCSS distance'''
376
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
640 return 1-self._computeNormalized(l1, l2, computeSubSequence)
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
641
376
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
642 def computeDistance(self, l1, l2, computeSubSequence = False):
2e6b8610bcaa work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 375
diff changeset
643 return self._computeDistance(l1, l2, computeSubSequence)
370
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
644
97e8fa0ee9a1 work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 369
diff changeset
645 #########################
45
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
646 # plotting section
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
647 #########################
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
648
332
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
649 def plotPolygon(poly, options = ''):
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
650 'Plots shapely polygon poly'
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
651 from numpy.core.multiarray import array
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
652 from matplotlib.pyplot import plot
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
653 from shapely.geometry import Polygon
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
654
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
655 tmp = array(poly.exterior)
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
656 plot(tmp[:,0], tmp[:,1], options)
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
657
324
99ca91a46007 minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 323
diff changeset
658 def stepPlot(X, firstX, lastX, initialCount = 0, increment = 1):
99ca91a46007 minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 323
diff changeset
659 '''for each value in X, increment by increment the initial count
297
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
660 returns the lists that can be plotted
324
99ca91a46007 minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 323
diff changeset
661 to obtain a step plot increasing by one for each value in x, from first to last value
99ca91a46007 minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 323
diff changeset
662 firstX and lastX should be respectively smaller and larger than all elements in X'''
297
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
663
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
664 sortedX = []
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
665 counts = [initialCount]
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
666 for x in sorted(X):
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
667 sortedX += [x,x]
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
668 counts.append(counts[-1])
324
99ca91a46007 minor change
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 323
diff changeset
669 counts.append(counts[-1]+increment)
297
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
670 counts.append(counts[-1])
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
671 return [firstX]+sortedX+[lastX], counts
f6f423e25c7f adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 286
diff changeset
672
665
15e244d2a1b5 corrected bug with circular import for VideoFilenameAddable, moved to base module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 659
diff changeset
673 class PlottingPropertyValues(object):
45
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
674 def __init__(self, values):
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
675 self.values = values
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
676
116
2bf5b76320c0 moved intersection plotting and added markers for scatter plots
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 115
diff changeset
677 def __getitem__(self, i):
45
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
678 return self.values[i%len(self.values)]
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
679
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
680 markers = PlottingPropertyValues(['+', '*', ',', '.', 'x', 'D', 's', 'o'])
116
2bf5b76320c0 moved intersection plotting and added markers for scatter plots
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 115
diff changeset
681 scatterMarkers = PlottingPropertyValues(['s','o','^','>','v','<','d','p','h','8','+','x'])
45
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
682
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
683 linestyles = PlottingPropertyValues(['-', '--', '-.', ':'])
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
684
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
685 colors = PlottingPropertyValues('brgmyck') # 'w'
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
686
115
550556378466 added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 86
diff changeset
687 def plotIndicatorMap(indicatorMap, squareSize, masked = True, defaultValue=-1):
65
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
688 from numpy import array, arange, ones, ma
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
689 from matplotlib.pyplot import pcolor
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
690 coords = array(indicatorMap.keys())
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
691 minX = min(coords[:,0])
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
692 minY = min(coords[:,1])
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
693 X = arange(minX, max(coords[:,0])+1.1)*squareSize
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
694 Y = arange(minY, max(coords[:,1])+1.1)*squareSize
115
550556378466 added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 86
diff changeset
695 C = defaultValue*ones((len(Y), len(X)))
65
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
696 for k,v in indicatorMap.iteritems():
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
697 C[k[1]-minY,k[0]-minX] = v
115
550556378466 added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 86
diff changeset
698 if masked:
550556378466 added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 86
diff changeset
699 pcolor(X, Y, ma.masked_where(C==defaultValue,C))
550556378466 added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 86
diff changeset
700 else:
550556378466 added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 86
diff changeset
701 pcolor(X, Y, C)
65
75cf537b8d88 moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 48
diff changeset
702
45
74d2de078baf added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 42
diff changeset
703 #########################
637
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
704 # Data download
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
705 #########################
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
706
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
707 def downloadECWeather(stationID, years, months = [], outputDirectoryname = '.', english = True):
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
708 '''Downloads monthly weather data from Environment Canada
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
709 If month is provided (number 1 to 12), it means hourly data for the whole month
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
710 Otherwise, means the data for each day, for the whole year
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
711
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
712 Example: MONTREAL MCTAVISH 10761
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
713 MONTREALPIERRE ELLIOTT TRUDEAU INTL A 5415
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
714
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
715 To get daily data for 2010 and 2011, downloadECWeather(10761, [2010,2011], [], '/tmp')
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
716 To get hourly data for 2009 and 2012, January, March and October, downloadECWeather(10761, [2009,2012], [1,3,10], '/tmp')'''
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
717 import urllib2
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
718 if english:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
719 language = 'e'
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
720 else:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
721 language = 'f'
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
722 if len(months) == 0:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
723 timeFrame = 2
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
724 months = [1]
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
725 else:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
726 timeFrame = 1
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
727
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
728 for year in years:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
729 for month in months:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
730 url = urllib2.urlopen('http://climat.meteo.gc.ca/climateData/bulkdata_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es'.format(language, stationID, year, month, timeFrame))
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
731 data = url.read()
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
732 outFilename = '{}/{}-{}'.format(outputDirectoryname, stationID, year)
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
733 if timeFrame == 1:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
734 outFilename += '-{}-hourly'.format(month)
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
735 else:
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
736 outFilename += '-daily'
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
737 outFilename += '.csv'
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
738 out = open(outFilename, 'w')
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
739 out.write(data)
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
740 out.close()
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
741
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
742 #########################
c9a0b72979fd added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 615
diff changeset
743 # File I/O
27
44689029a86f updated segmentIntersection and other
Nicolas Saunier <nico@confins.net>
parents: 24
diff changeset
744 #########################
24
6fb59cfb201e first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents: 19
diff changeset
745
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
746 def removeExtension(filename, delimiter = '.'):
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
747 '''Returns the filename minus the extension (all characters after last .)'''
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
748 i = filename.rfind(delimiter)
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
749 if i>0:
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
750 return filename[:i]
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
751 else:
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
752 return filename
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
753
46
b5d007612e16 added filename util
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 45
diff changeset
754 def cleanFilename(s):
b5d007612e16 added filename util
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 45
diff changeset
755 'cleans filenames obtained when contatenating figure characteristics'
266
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
756 return s.replace(' ','-').replace('.','').replace('/','-')
46
b5d007612e16 added filename util
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 45
diff changeset
757
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
758 def listfiles(dirname, extension, remove = False):
14
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
759 '''Returns the list of files with the extension in the directory dirname
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
760 If remove is True, the filenames are stripped from the extension'''
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
761 from os import listdir
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
762 tmp = [f for f in listdir(dirname) if f.endswith(extension)]
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
763 tmp.sort()
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
764 if remove:
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
765 return [removeExtension(f, extension) for f in tmp]
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
766 else:
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
767 return tmp
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
768
266
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
769 def mkdir(dirname):
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
770 'Creates a directory if it does not exist'
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
771 import os
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
772 if not os.path.exists(dirname):
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
773 os.mkdir(dirname)
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
774 else:
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
775 print(dirname+' already exists')
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
776
14
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
777 def removeFile(filename):
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
778 '''Deletes the file while avoiding raising an error
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
779 if the file does not exist'''
266
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
780 import os
14
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
781 if (os.path.exists(filename)):
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
782 os.remove(filename)
266
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
783 else:
aba9711b3149 small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 262
diff changeset
784 print(filename+' does not exist')
14
e7bbe8465591 homography and other utils
Nicolas Saunier <nico@confins.net>
parents: 7
diff changeset
785
42
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
786 def line2Floats(l, separator=' '):
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
787 '''Returns the list of floats corresponding to the string'''
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
788 return [float(x) for x in l.split(separator)]
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
789
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
790 def line2Ints(l, separator=' '):
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
791 '''Returns the list of ints corresponding to the string'''
1a2ac2d4f53a added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 35
diff changeset
792 return [int(x) for x in l.split(separator)]
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
793
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
794 #########################
332
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
795 # CLI utils
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
796 #########################
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
797
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
798 def parseCLIOptions(helpMessage, options, cliArgs, optionalOptions=[]):
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
799 ''' Simple function to handle similar argument parsing
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
800 Returns the dictionary of options and their values
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
801
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
802 * cliArgs are most likely directly sys.argv
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
803 (only the elements after the first one are considered)
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
804
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
805 * options should be a list of strings for getopt options,
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
806 eg ['frame=','correspondences=','video=']
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
807 A value must be provided for each option, or the program quits'''
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
808 import sys, getopt
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
809 from numpy.core.fromnumeric import all
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
810 optionValues, args = getopt.getopt(cliArgs[1:], 'h', ['help']+options+optionalOptions)
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
811 optionValues = dict(optionValues)
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
812
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
813 if '--help' in optionValues.keys() or '-h' in optionValues.keys():
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
814 print(helpMessage+
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
815 '\n - Compulsory options: '+' '.join([opt.replace('=','') for opt in options])+
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
816 '\n - Non-compulsory options: '+' '.join([opt.replace('=','') for opt in optionalOptions]))
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
817 sys.exit()
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
818
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
819 missingArgument = [('--'+opt.replace('=','') in optionValues.keys()) for opt in options]
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
820 if not all(missingArgument):
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
821 print('Missing argument')
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
822 print(optionValues)
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
823 sys.exit()
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
824
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
825 return optionValues
a6ca86107f27 reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 324
diff changeset
826
397
b36b00dd27c3 added function to read scene metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 395
diff changeset
827
b36b00dd27c3 added function to read scene metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 395
diff changeset
828 #########################
553
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
829 # Profiling
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
830 #########################
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
831
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
832 def analyzeProfile(profileFilename, stripDirs = True):
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
833 '''Analyze the file produced by cProfile
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
834
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
835 obtained by for example:
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
836 - call in script (for main() function in script)
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
837 import cProfile, os
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
838 cProfile.run('main()', os.path.join(os.getcwd(),'main.profile'))
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
839
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
840 - or on the command line:
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
841 python -m cProfile [-o profile.bin] [-s sort] scriptfile [arg]'''
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
842 import pstats, os
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
843 p = pstats.Stats(os.path.join(os.pardir, profileFilename))
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
844 if stripDirs:
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
845 p.strip_dirs()
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
846 p.sort_stats('time')
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
847 p.print_stats(.2)
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
848 #p.sort_stats('time')
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
849 # p.print_callees(.1, 'int_prediction.py:')
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
850 return p
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
851
3622a5653ee9 added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 547
diff changeset
852 #########################
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
853 # running tests
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
854 #########################
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
855
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
856 if __name__ == "__main__":
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
857 import doctest
2
de5642925615 started implementation of TimeInterval and Spatio-temporal object
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 0
diff changeset
858 import unittest
31
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
859 suite = doctest.DocFileSuite('tests/utils.txt')
c000f37c316d moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents: 29
diff changeset
860 #suite = doctest.DocTestSuite()
2
de5642925615 started implementation of TimeInterval and Spatio-temporal object
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 0
diff changeset
861 unittest.TextTestRunner().run(suite)
de5642925615 started implementation of TimeInterval and Spatio-temporal object
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents: 0
diff changeset
862 #doctest.testmod()
0
aed8eb63cdde initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff changeset
863 #doctest.testfile("example.txt")