Mercurial Hosting > traffic-intelligence
annotate python/utils.py @ 669:df6be882f325
bug corrected
author | Nicolas Saunier <nicolas.saunier@polymtl.ca> |
---|---|
date | Tue, 26 May 2015 11:35:24 +0200 |
parents | f8dcf483b296 |
children | f72ed51c6b65 |
rev | line source |
---|---|
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
1 #! /usr/bin/env python |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
2 ''' Generic utilities.''' |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
3 |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
4 import matplotlib.pyplot as plt |
397
b36b00dd27c3
added function to read scene metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
395
diff
changeset
|
5 from datetime import time, datetime |
569
0057c04f94d5
work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
561
diff
changeset
|
6 from math import sqrt |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
7 from scipy.stats import kruskal, shapiro |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
8 |
421
4fce27946c60
first example of video metadata using sqlalchemy
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
405
diff
changeset
|
9 datetimeFormat = "%Y-%m-%d %H:%M:%S" |
4fce27946c60
first example of video metadata using sqlalchemy
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
405
diff
changeset
|
10 |
185
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
11 ######################### |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
12 # Enumerations |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
13 ######################### |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
14 |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
15 def inverseEnumeration(l): |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
16 'Returns the dictionary that provides for each element in the input list its index in the input list' |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
17 result = {} |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
18 for i,x in enumerate(l): |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
19 result[x] = i |
c06379f25ab8
utilities for user types
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
181
diff
changeset
|
20 return result |
155
f03fe3d6d0c8
added functions to parse options
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
152
diff
changeset
|
21 |
f03fe3d6d0c8
added functions to parse options
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
152
diff
changeset
|
22 ######################### |
637
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
23 # Simple statistics |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
24 ######################### |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
25 |
423
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
26 def sampleSize(stdev, tolerance, percentConfidence, printLatex = False): |
301
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
27 from scipy.stats.distributions import norm |
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
28 k = round(norm.ppf(0.5+percentConfidence/200., 0, 1)*100)/100. # 1.-(100-percentConfidence)/200. |
423
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
29 if printLatex: |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
30 print('${0}^2\\frac{{{1}^2}}{{{2}^2}}$'.format(k, stdev, tolerance)) |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
31 return (k*stdev/tolerance)**2 |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
32 |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
33 def confidenceInterval(mean, stdev, nSamples, percentConfidence, trueStd = True, printLatex = False): |
499
0a93afea8243
alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
491
diff
changeset
|
34 '''if trueStd, use normal distribution, otherwise, Student |
0a93afea8243
alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
491
diff
changeset
|
35 |
0a93afea8243
alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
491
diff
changeset
|
36 Use otherwise t.interval or norm.interval |
0a93afea8243
alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
491
diff
changeset
|
37 ex: norm.interval(0.95, loc = 0., scale = 2.3/sqrt(11)) |
0a93afea8243
alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
491
diff
changeset
|
38 t.interval(0.95, 10, loc=1.2, scale = 2.3/sqrt(nSamples)) |
0a93afea8243
alternative confidence interval
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
491
diff
changeset
|
39 loc is mean, scale is sigma/sqrt(n) (for Student, 10 is df)''' |
423
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
40 from math import sqrt |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
41 from scipy.stats.distributions import norm, t |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
42 if trueStd: |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
43 k = round(norm.ppf(0.5+percentConfidence/200., 0, 1)*100)/100. # 1.-(100-percentConfidence)/200. |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
44 else: # use Student |
f738fa1b69f0
added sample size and Student distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
421
diff
changeset
|
45 k = round(t.ppf(0.5+percentConfidence/200., nSamples-1)*100)/100. |
301
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
46 e = k*stdev/sqrt(nSamples) |
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
47 if printLatex: |
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
48 print('${0} \pm {1}\\frac{{{2}}}{{\sqrt{{{3}}}}}$'.format(mean, k, stdev, nSamples)) |
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
49 return mean-e, mean+e |
27f06d28036d
added simple helper for confidence intervals
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
297
diff
changeset
|
50 |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
51 def computeChi2(expected, observed): |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
52 '''Returns the Chi2 statistics''' |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
53 result = 0. |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
54 for e, o in zip(expected, observed): |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
55 result += ((e-o)*(e-o))/e |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
56 return result |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
57 |
665
15e244d2a1b5
corrected bug with circular import for VideoFilenameAddable, moved to base module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
659
diff
changeset
|
58 class EmpiricalDistribution(object): |
85
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
59 def nSamples(self): |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
60 return sum(self.counts) |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
61 |
588
c5406edbcf12
added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
574
diff
changeset
|
62 def cumulativeDensityFunction(sample, normalized = False): |
276 | 63 '''Returns the cumulative density function of the sample of a random variable''' |
588
c5406edbcf12
added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
574
diff
changeset
|
64 from numpy import arange, cumsum |
c5406edbcf12
added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
574
diff
changeset
|
65 xaxis = sorted(sample) |
c5406edbcf12
added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
574
diff
changeset
|
66 counts = arange(1,len(sample)+1) # dtype = float |
c5406edbcf12
added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
574
diff
changeset
|
67 if normalized: |
c5406edbcf12
added loading ground truth annotations (ground truth) from polytrack format
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
574
diff
changeset
|
68 counts /= float(len(sample)) |
197
2788b2827670
simple cumulatie function distribution computation
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
185
diff
changeset
|
69 return xaxis, counts |
85
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
70 |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
71 class EmpiricalDiscreteDistribution(EmpiricalDistribution): |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
72 '''Class to represent a sample of a distribution for a discrete random variable |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
73 ''' |
86
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
74 from numpy.core.fromnumeric import sum |
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
75 |
85
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
76 def __init__(self, categories, counts): |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
77 self.categories = categories |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
78 self.counts = counts |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
79 |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
80 def mean(self): |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
81 result = [float(x*y) for x,y in zip(self.categories, self.counts)] |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
82 return sum(result)/self.nSamples() |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
83 |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
84 def var(self, mean = None): |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
85 if not mean: |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
86 m = self.mean() |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
87 else: |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
88 m = mean |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
89 result = 0. |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
90 squares = [float((x-m)*(x-m)*y) for x,y in zip(self.categories, self.counts)] |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
91 return sum(squares)/(self.nSamples()-1) |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
92 |
86
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
93 def referenceCounts(self, probability): |
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
94 '''probability is a function that returns the probability of the random variable for the category values''' |
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
95 refProba = [probability(c) for c in self.categories] |
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
96 refProba[-1] = 1-sum(refProba[:-1]) |
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
97 refCounts = [r*self.nSamples() for r in refProba] |
f03ec4697a09
corrected discrete distribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
85
diff
changeset
|
98 return refCounts, refProba |
85
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
99 |
7f1e54234f96
added empirical discrete distribution, modified class organization and names
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
77
diff
changeset
|
100 class EmpiricalContinuousDistribution(EmpiricalDistribution): |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
101 '''Class to represent a sample of a distribution for a continuous random variable |
76
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
102 with the number of observations for each interval |
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
103 intervals (categories variable) are defined by their left limits, the last one being the right limit |
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
104 categories contain therefore one more element than the counts''' |
35
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
105 def __init__(self, categories, counts): |
276 | 106 # todo add samples for initialization and everything to None? (or setSamples?) |
35
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
107 self.categories = categories |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
108 self.counts = counts |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
109 |
35
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
110 def mean(self): |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
111 result = 0. |
76
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
112 for i in range(len(self.counts)-1): |
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
113 result += self.counts[i]*(self.categories[i]+self.categories[i+1])/2 |
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
114 return result/self.nSamples() |
35
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
115 |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
116 def var(self, mean = None): |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
117 if not mean: |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
118 m = self.mean() |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
119 else: |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
120 m = mean |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
121 result = 0. |
76
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
122 for i in range(len(self.counts)-1): |
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
123 mid = (self.categories[i]+self.categories[i+1])/2 |
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
124 result += self.counts[i]*(mid - m)*(mid - m) |
35
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
125 return result/(self.nSamples()-1) |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
126 |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
127 def referenceCounts(self, cdf): |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
128 '''cdf is a cumulative distribution function |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
129 returning the probability of the variable being less that x''' |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
130 # refCumulativeCounts = [0]#[cdf(self.categories[0][0])] |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
131 # for inter in self.categories: |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
132 # refCumulativeCounts.append(cdf(inter[1])) |
76
64fde2b1f96d
simplified intervales in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
75
diff
changeset
|
133 refCumulativeCounts = [cdf(x) for x in self.categories[1:-1]] |
35
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
134 |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
135 refProba = [refCumulativeCounts[0]] |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
136 for i in xrange(1,len(refCumulativeCounts)): |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
137 refProba.append(refCumulativeCounts[i]-refCumulativeCounts[i-1]) |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
138 refProba.append(1-refCumulativeCounts[-1]) |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
139 refCounts = [p*self.nSamples() for p in refProba] |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
140 |
8cafee54466f
forgotten update of histogram class
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
32
diff
changeset
|
141 return refCounts, refProba |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
142 |
77
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
143 def printReferenceCounts(self, refCounts=None): |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
144 if refCounts: |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
145 ref = refCounts |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
146 else: |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
147 ref = self.referenceCounts |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
148 for i in xrange(len(ref[0])): |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
149 print('{0}-{1} & {2:0.3} & {3:0.3} \\\\'.format(self.categories[i],self.categories[i+1],ref[1][i], ref[0][i])) |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
150 |
5e6cd36a991c
added pretty print in empiricalDistribution
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
76
diff
changeset
|
151 |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
152 ######################### |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
153 # maths section |
27
44689029a86f
updated segmentIntersection and other
Nicolas Saunier <nico@confins.net>
parents:
24
diff
changeset
|
154 ######################### |
24
6fb59cfb201e
first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents:
19
diff
changeset
|
155 |
433
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
156 # def kernelSmoothing(sampleX, X, Y, weightFunc, halfwidth): |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
157 # '''Returns a smoothed weighted version of Y at the predefined values of sampleX |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
158 # Sum_x weight(sample_x,x) * y(x)''' |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
159 # from numpy import zeros, array |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
160 # smoothed = zeros(len(sampleX)) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
161 # for i,x in enumerate(sampleX): |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
162 # weights = array([weightFunc(x,xx, halfwidth) for xx in X]) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
163 # if sum(weights)>0: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
164 # smoothed[i] = sum(weights*Y)/sum(weights) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
165 # else: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
166 # smoothed[i] = 0 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
167 # return smoothed |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
168 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
169 def kernelSmoothing(x, X, Y, weightFunc, halfwidth): |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
170 '''Returns the smoothed estimate of (X,Y) at x |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
171 Sum_x weight(sample_x,x) * y(x)''' |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
172 from numpy import zeros, array |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
173 weights = array([weightFunc(x,observedx, halfwidth) for observedx in X]) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
174 if sum(weights)>0: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
175 return sum(weights*Y)/sum(weights) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
176 else: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
177 return 0 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
178 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
179 def uniform(center, x, halfwidth): |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
180 if abs(center-x)<halfwidth: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
181 return 1. |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
182 else: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
183 return 0. |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
184 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
185 def gaussian(center, x, halfwidth): |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
186 from numpy import exp |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
187 return exp(-((center-x)/halfwidth)**2/2) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
188 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
189 def epanechnikov(center, x, halfwidth): |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
190 diff = abs(center-x) |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
191 if diff<halfwidth: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
192 return 1.-(diff/halfwidth)**2 |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
193 else: |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
194 return 0. |
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
195 |
434 | 196 def triangular(center, x, halfwidth): |
197 diff = abs(center-x) | |
198 if diff<halfwidth: | |
199 return 1.-abs(diff/halfwidth) | |
200 else: | |
201 return 0. | |
433
d40ad901b272
added kernel smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
423
diff
changeset
|
202 |
518
0c86c73f3c09
median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
511
diff
changeset
|
203 def medianSmoothing(x, X, Y, halfwidth): |
0c86c73f3c09
median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
511
diff
changeset
|
204 '''Returns the media of Y's corresponding to X's in the interval [x-halfwidth, x+halfwidth]''' |
0c86c73f3c09
median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
511
diff
changeset
|
205 from numpy import median |
0c86c73f3c09
median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
511
diff
changeset
|
206 return median([y for observedx, y in zip(X,Y) if abs(x-observedx)<halfwidth]) |
0c86c73f3c09
median smoothing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
511
diff
changeset
|
207 |
521
3707eeb20f25
changed argMaxDict name to argmaxDict
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
518
diff
changeset
|
208 def argmaxDict(d): |
561
ee45c6eb6d49
added Mohamed Gomaa Mohamed function to smooth object trajectories
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
553
diff
changeset
|
209 return max(d, key=d.get) |
279
3af4c267a7bf
generic simple LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
276
diff
changeset
|
210 |
395
6fba1ab040f1
minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
391
diff
changeset
|
211 def framesToTime(nFrames, frameRate, initialTime = time()): |
6fba1ab040f1
minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
391
diff
changeset
|
212 '''returns a datetime.time for the time in hour, minutes and seconds |
6fba1ab040f1
minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
391
diff
changeset
|
213 initialTime is a datetime.time''' |
248
571ba5ed22e2
added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
241
diff
changeset
|
214 from math import floor |
395
6fba1ab040f1
minor modification to framestotime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
391
diff
changeset
|
215 seconds = int(floor(float(nFrames)/float(frameRate))+initialTime.hour*3600+initialTime.minute*60+initialTime.second) |
261
4aa792cb0fa9
changing framesToTime to return a datetime.time
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
248
diff
changeset
|
216 h = int(floor(seconds/3600.)) |
248
571ba5ed22e2
added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
241
diff
changeset
|
217 seconds = seconds - h*3600 |
261
4aa792cb0fa9
changing framesToTime to return a datetime.time
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
248
diff
changeset
|
218 m = int(floor(seconds/60)) |
248
571ba5ed22e2
added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
241
diff
changeset
|
219 seconds = seconds - m*60 |
262
a048066bd20f
correcting bug in framesToTime
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
261
diff
changeset
|
220 return time(h, m, seconds) |
248
571ba5ed22e2
added utils for bus processing
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
241
diff
changeset
|
221 |
381
387cc0142211
script to replay event annotations
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
376
diff
changeset
|
222 def timeToFrames(t, frameRate): |
387cc0142211
script to replay event annotations
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
376
diff
changeset
|
223 return frameRate*(t.hour*3600+t.minute*60+t.second) |
387cc0142211
script to replay event annotations
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
376
diff
changeset
|
224 |
241
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
225 def sortXY(X,Y): |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
226 'returns the sorted (x, Y(x)) sorted on X' |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
227 D = {} |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
228 for x, y in zip(X,Y): |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
229 D[x]=y |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
230 xsorted = sorted(D.keys()) |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
231 return xsorted, [D[x] for x in xsorted] |
ee1caff48b03
added function to sort to list of paired data X,Y
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
235
diff
changeset
|
232 |
32 | 233 def ceilDecimals(v, nDecimals): |
234 '''Rounds the number at the nth decimal | |
235 eg 1.23 at 0 decimal is 2, at 1 decimal is 1.3''' | |
236 from math import ceil,pow | |
237 tens = pow(10,nDecimals) | |
238 return ceil(v*tens)/tens | |
239 | |
152
74b1fc68d4df
re-organized code to avoid cyclic python module dependencies
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
116
diff
changeset
|
240 def inBetween(bound1, bound2, x): |
569
0057c04f94d5
work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
561
diff
changeset
|
241 return bound1 <= x <= bound2 or bound2 <= x <= bound1 |
0057c04f94d5
work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
561
diff
changeset
|
242 |
0057c04f94d5
work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
561
diff
changeset
|
243 def pointDistanceL2(x1,y1,x2,y2): |
0057c04f94d5
work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
561
diff
changeset
|
244 ''' Compute point-to-point distance (L2 norm, ie Euclidean distance)''' |
0057c04f94d5
work in progress on intersections (for PET)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
561
diff
changeset
|
245 return sqrt((x2-x1)**2+(y2-y1)**2) |
24
6fb59cfb201e
first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents:
19
diff
changeset
|
246 |
6fb59cfb201e
first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents:
19
diff
changeset
|
247 def crossProduct(l1, l2): |
6fb59cfb201e
first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents:
19
diff
changeset
|
248 return l1[0]*l2[1]-l1[1]*l2[0] |
6fb59cfb201e
first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents:
19
diff
changeset
|
249 |
574
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
250 def cat_mvgavg(cat_list, halfWidth): |
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
251 ''' Return a list of categories/values smoothed according to a window. |
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
252 halfWidth is the search radius on either side''' |
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
253 from copy import deepcopy |
659
784298512b60
minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
637
diff
changeset
|
254 smoothed = deepcopy(cat_list) |
784298512b60
minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
637
diff
changeset
|
255 for point in range(len(cat_list)): |
574
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
256 lower_bound_check = max(0,point-halfWidth) |
659
784298512b60
minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
637
diff
changeset
|
257 upper_bound_check = min(len(cat_list)-1,point+halfWidth+1) |
784298512b60
minor modifications
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
637
diff
changeset
|
258 window_values = cat_list[lower_bound_check:upper_bound_check] |
574
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
259 smoothed[point] = max(set(window_values), key=window_values.count) |
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
260 return smoothed |
e24eeb244698
first implementation of projection to curvilinear coordinates
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
569
diff
changeset
|
261 |
547 | 262 def filterMovingWindow(inputSignal, halfWidth): |
29
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
263 '''Returns an array obtained after the smoothing of the input by a moving average |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
264 The first and last points are copied from the original.''' |
547 | 265 from numpy import ones,convolve,array |
29
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
266 width = float(halfWidth*2+1) |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
267 win = ones(width,'d') |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
268 result = convolve(win/width,array(inputSignal),'same') |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
269 result[:halfWidth] = inputSignal[:halfWidth] |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
270 result[-halfWidth:] = inputSignal[-halfWidth:] |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
271 return result |
ca8e716cc231
added moving average filter
Nicolas Saunier <nico@confins.net>
parents:
27
diff
changeset
|
272 |
199
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
273 def linearRegression(x, y, deg = 1, plotData = False): |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
274 '''returns the least square estimation of the linear regression of y = ax+b |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
275 as well as the plot''' |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
276 from numpy.lib.polynomial import polyfit |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
277 from numpy.core.multiarray import arange |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
278 coef = polyfit(x, y, deg) |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
279 if plotData: |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
280 def poly(x): |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
281 result = 0 |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
282 for i in range(len(coef)): |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
283 result += coef[i]*x**(len(coef)-i-1) |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
284 return result |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
285 plt.plot(x, y, 'x') |
199
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
286 xx = arange(min(x), max(x),(max(x)-min(x))/1000) |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
287 plt.plot(xx, [poly(z) for z in xx]) |
199
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
288 return coef |
ca9d9104afba
added utility to calibrate polynoms and plot
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
197
diff
changeset
|
289 |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
290 def correlation(data, correlationMethod = 'pearson', plotFigure = False, displayNames = None, figureFilename = None): |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
291 '''Computes (and displays) the correlation matrix for a pandas DataFrame''' |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
292 c=data.corr(correlationMethod) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
293 if plotFigure: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
294 fig = plt.figure(figsize=(2+0.4*c.shape[0], 0.4*c.shape[0])) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
295 fig.add_subplot(1,1,1) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
296 #plt.imshow(np.fabs(c), interpolation='none') |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
297 plt.imshow(c, vmin=-1., vmax = 1., interpolation='none', cmap = 'RdYlBu_r') # coolwarm |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
298 colnames = [displayNames.get(s.strip(), s.strip()) for s in c.columns.tolist()] |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
299 #correlation.plot_corr(c, xnames = colnames, normcolor=True, title = filename) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
300 plt.xticks(range(len(colnames)), colnames, rotation=90) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
301 plt.yticks(range(len(colnames)), colnames) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
302 plt.tick_params('both', length=0) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
303 plt.subplots_adjust(bottom = 0.29) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
304 plt.colorbar() |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
305 plt.title('Correlation ({})'.format(correlationMethod)) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
306 plt.tight_layout() |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
307 if figureFilename is not None: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
308 plt.savefig(figureFilename, dpi = 150, transparent = True) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
309 return c |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
310 |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
311 def addDummies(data, variables, allVariables = True): |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
312 '''Add binary dummy variables for each value of a nominal variable |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
313 in a pandas DataFrame''' |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
314 from numpy import NaN, dtype |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
315 newVariables = [] |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
316 for var in variables: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
317 if var in data.columns and data.dtypes[var] == dtype('O') and len(data[var].unique()) > 2: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
318 values = data[var].unique() |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
319 if not allVariables: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
320 values = values[:-1] |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
321 for val in values: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
322 if val is not NaN: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
323 newVariable = (var+'_{}'.format(val)).replace('.','').replace(' ','').replace('-','') |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
324 data[newVariable] = (data[var] == val) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
325 newVariables.append(newVariable) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
326 return newVariables |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
327 |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
328 def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, figureFilenamePrefix = None, figureFileType = 'pdf'): |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
329 '''Studies the influence of (nominal) independent variable over the dependent variable |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
330 |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
331 Makes tests if the conditional distributions are normal |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
332 using the Shapiro-Wilk test (in which case ANOVA could be used) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
333 Implements uses the non-parametric Kruskal Wallis test''' |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
334 tmp = data[data[independentVariable].notnull()] |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
335 independentVariableValues = sorted(tmp[independentVariable].unique().tolist()) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
336 if len(independentVariableValues) >= 2: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
337 for x in independentVariableValues: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
338 print('Shapiro-Wilk normality test for {} when {}={}: {} obs'.format(dependentVariable,independentVariable, x, len(tmp.loc[tmp[independentVariable] == x, dependentVariable]))) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
339 if len(tmp.loc[tmp[independentVariable] == x, dependentVariable]) >= 3: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
340 print shapiro(tmp.loc[tmp[independentVariable] == x, dependentVariable]) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
341 if plotFigure: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
342 plt.figure() |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
343 plt.boxplot([tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues]) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
344 #q25, q75 = tmp[dependentVariable].quantile([.25, .75]) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
345 #plt.ylim(ymax = q75+1.5*(q75-q25)) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
346 plt.xticks(range(1,len(independentVariableValues)+1), independentVariableValues) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
347 plt.title('{} vs {}'.format(dependentVariable, independentVariable)) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
348 if figureFilenamePrefix is not None: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
349 plt.savefig(figureFilenamePrefix+'{}-{}.{}'.format(dependentVariable, independentVariable, figureFileType)) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
350 #else: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
351 # TODO formatter le tableau (html?) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
352 print tmp.groupby([independentVariable])[dependentVariable].describe().unstack().sort(['50%'], ascending = False) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
353 return kruskal(*[tmp.loc[tmp[independentVariable] == x, dependentVariable] for x in independentVariableValues]) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
354 else: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
355 return None |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
356 |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
357 def prepareRegression(data, dependentVariable, independentVariables, maxCorrelationThreshold, correlations, maxCorrelationP, correlationFunc): |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
358 '''Removes variables from candidate independent variables if |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
359 - if two independent variables are correlated (> maxCorrelationThreshold), one is removed |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
360 - if an independent variable is not correlated with the dependent variable (p>maxCorrelationP) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
361 Returns the remaining non-correlated variables, correlated with the dependent variable |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
362 |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
363 correlationFunc is spearmanr or pearsonr from scipy.stats |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
364 |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
365 TODO: pass the dummies for nominal variables and remove if all dummies are correlated, or none is correlated with the dependentvariable''' |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
366 from numpy import dtype |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
367 from copy import copy |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
368 result = copy(independentVariables) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
369 for v1 in independentVariables: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
370 if v1 in correlations.index: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
371 for v2 in independentVariables: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
372 if v2 != v1 and v2 in correlations.index: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
373 if abs(correlations.loc[v1, v2]) > maxCorrelationThreshold: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
374 if v1 in result and v2 in result: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
375 print('Removing {} (correlation {} with {})'.format(v2, correlations.loc[v1, v2], v1)) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
376 result.remove(v2) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
377 #regressionIndependentVariables = result |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
378 for var in copy(result): |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
379 if data.dtypes[var] != dtype('O'): |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
380 cor, p = correlationFunc(data[dependentVariable], data[var]) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
381 if p > maxCorrelationP: |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
382 print('Removing {} (no correlation p={})'.format(var, p)) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
383 result.remove(var) |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
384 return result |
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
385 |
667
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
386 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
387 ######################### |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
388 # regression analysis using statsmodels (and pandas) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
389 ######################### |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
390 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
391 # TODO make class for experiments? |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
392 # TODO add tests with public dataset downloaded from Internet (IRIS et al) |
667
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
393 def modelString(experiment, dependentVariable, independentVariables): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
394 return dependentVariable+' ~ '+' + '.join([independentVariable for independentVariable in independentVariables if experiment[independentVariable]]) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
395 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
396 def runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols'): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
397 import statsmodels.formula.api as smf |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
398 modelStr = modelString(experiment, dependentVariable, independentVariables) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
399 if regressionType == 'ols': |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
400 model = smf.ols(modelStr, data = data) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
401 elif regressionType == 'gls': |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
402 model = smf.gls(modelStr, data = data) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
403 elif regressionType == 'rlm': |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
404 model = smf.rlm(modelStr, data = data) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
405 else: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
406 print('Unknown regression type {}. Exiting'.format(regressionType)) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
407 import sys |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
408 sys.exit() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
409 return model.fit() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
410 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
411 def runModels(experiments, data, dependentVariable, independentVariables, regressionType = 'ols'): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
412 '''Runs several models and stores 3 statistics |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
413 adjusted R2, condition number (should be small, eg < 1000) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
414 and p-value for Shapiro-Wilk test of residual normality''' |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
415 for i,experiment in experiments.iterrows(): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
416 if experiment[independentVariables].any(): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
417 results = runModel(experiment, data, dependentVariable, independentVariables, regressionType = 'ols') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
418 experiments.loc[i,'r2adj'] = results.rsquared_adj |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
419 experiments.loc[i,'condNum'] = results.condition_number |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
420 experiments.loc[i, 'shapiroP'] = shapiro(results.resid)[1] |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
421 experiments.loc[i,'nobs'] = int(results.nobs) |
667
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
422 return experiments |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
423 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
424 def generateExperiments(independentVariables): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
425 '''Generates all possible models for including or not each independent variable''' |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
426 experiments = {} |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
427 nIndependentVariables = len(independentVariables) |
669 | 428 if nIndependentVariables != len(set(independentVariables)): |
667
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
429 print("Duplicate variables. Exiting") |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
430 import sys |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
431 sys.exit() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
432 nModels = 2**nIndependentVariables |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
433 for i,var in enumerate(independentVariables): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
434 pattern = [False]*(2**i)+[True]*(2**i) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
435 experiments[var] = pattern*(2**(nIndependentVariables-i-1)) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
436 experiments = pd.DataFrame(experiments) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
437 experiments['r2adj'] = 0. |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
438 experiments['condNum'] = np.nan |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
439 experiments['shapiroP'] = -1 |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
440 experiments['nobs'] = -1 |
667
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
441 return experiments |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
442 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
443 def findBestModel(data, dependentVariable, independentVariables, regressionType = 'ols', nProcesses = 1): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
444 '''Generates all possible model with the independentVariables |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
445 and runs them, saving the results in experiments |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
446 with multiprocess option''' |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
447 experiments = generateExperiments(independentVariables) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
448 nModels = len(experiments) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
449 print("Running {} models with {} processes".format(nModels, nProcesses)) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
450 if nProcesses == 1: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
451 return runModels(experiments, data, dependentVariable, independentVariables, regressionType) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
452 else: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
453 pool = Pool(processes = nProcesses) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
454 chunkSize = int(np.ceil(nModels/nProcesses)) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
455 jobs = [pool.apply_async(runModels, args = (experiments[i*chunkSize:(i+1)*chunkSize], data, dependentVariable, independentVariables, regressionType)) for i in range(nProcesses)] |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
456 return pd.concat([job.get() for job in jobs]) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
457 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
458 def findBestModelFwd(data, dependentVariable, independentVariables, modelFunc, experiments = None): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
459 '''Forward search for best model (based on adjusted R2) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
460 Randomly starting with one variable and adding randomly variables |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
461 if they improve the model |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
462 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
463 The results are added to experiments if provided as argument |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
464 Storing in experiment relies on the index being the number equal |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
465 to the binary code derived from the independent variables''' |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
466 if experiments is None: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
467 experiments = generateExperiments(independentVariables) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
468 nIndependentVariables = len(independentVariables) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
469 permutation = np.random.permutation(range(nIndependentVariables)).tolist() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
470 variableMapping = {j: independentVariables[i] for i,j in enumerate(permutation)} |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
471 print('Tested variables '+', '.join([variableMapping[i] for i in xrange(nIndependentVariables)])) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
472 bestModel = [False]*nIndependentVariables |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
473 currentVarNum = 0 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
474 currentR2Adj = 0. |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
475 for currentVarNum in xrange(nIndependentVariables): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
476 currentModel = [i for i in bestModel] |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
477 currentModel[currentVarNum] = True |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
478 rowIdx = sum([0]+[2**i for i in xrange(nIndependentVariables) if currentModel[permutation[i]]]) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
479 #print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in xrange(nIndependentVariables) if currentModel[permutation[i]]]) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
480 if experiments.loc[rowIdx, 'shapiroP'] < 0: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
481 modelStr = modelString(experiments.loc[rowIdx], dependentVariable, independentVariables) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
482 model = modelFunc(modelStr, data = data) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
483 results = model.fit() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
484 experiments.loc[rowIdx, 'r2adj'] = results.rsquared_adj |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
485 experiments.loc[rowIdx, 'condNum'] = results.condition_number |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
486 experiments.loc[rowIdx, 'shapiroP'] = shapiro(results.resid)[1] |
668
f8dcf483b296
code to prepare regression variables (remove correlated variables) and record dataset size in experimnets
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
667
diff
changeset
|
487 experiments.loc[rowIdx, 'nobs'] = int(results.nobs) |
667
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
488 if currentR2Adj < experiments.loc[rowIdx, 'r2adj']: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
489 currentR2Adj = experiments.loc[rowIdx, 'r2adj'] |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
490 bestModel[currentVarNum] = True |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
491 return experiments |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
492 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
493 def displayModelResults(results, model = None): |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
494 import statsmodels.api as sm |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
495 '''Displays some model results''' |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
496 print results.summary() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
497 print('Shapiro-Wilk normality test for residuals: {}'.format(shapiro(results.resid))) |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
498 if model is not None: |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
499 plt.figure() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
500 plt.plot(results.predict(), model.endog, 'x') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
501 x=plt.xlim() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
502 y=plt.ylim() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
503 plt.plot([max(x[0], y[0]), min(x[1], y[1])], [max(x[0], y[0]), min(x[1], y[1])], 'r') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
504 plt.title('true vs predicted') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
505 plt.figure() |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
506 plt.plot(results.predict(), results.resid, 'x') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
507 plt.title('residuals vs predicted') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
508 sm.qqplot(results.resid, fit = True, line = '45') |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
509 |
179b81faa1f8
added regression analysis functions
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
665
diff
changeset
|
510 |
27
44689029a86f
updated segmentIntersection and other
Nicolas Saunier <nico@confins.net>
parents:
24
diff
changeset
|
511 ######################### |
455
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
512 # iterable section |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
513 ######################### |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
514 |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
515 def mostCommon(L): |
456 | 516 '''Returns the most frequent element in a iterable |
517 | |
518 taken from http://stackoverflow.com/questions/1518522/python-most-common-element-in-a-list''' | |
455
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
519 from itertools import groupby |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
520 from operator import itemgetter |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
521 # get an iterable of (item, iterable) pairs |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
522 SL = sorted((x, i) for i, x in enumerate(L)) |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
523 # print 'SL:', SL |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
524 groups = groupby(SL, key=itemgetter(0)) |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
525 # auxiliary function to get "quality" for an item |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
526 def _auxfun(g): |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
527 item, iterable = g |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
528 count = 0 |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
529 min_index = len(L) |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
530 for _, where in iterable: |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
531 count += 1 |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
532 min_index = min(min_index, where) |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
533 # print 'item %r, count %r, minind %r' % (item, count, min_index) |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
534 return count, -min_index |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
535 # pick the highest-count/earliest item |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
536 return max(groups, key=_auxfun)[0] |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
537 |
abe0b2347d4c
added most common utility function
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
434
diff
changeset
|
538 ######################### |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
539 # sequence section |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
540 ######################### |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
541 |
665
15e244d2a1b5
corrected bug with circular import for VideoFilenameAddable, moved to base module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
659
diff
changeset
|
542 class LCSS(object): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
543 '''Class that keeps the LCSS parameters |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
544 and puts together the various computations''' |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
545 def __init__(self, similarityFunc, delta = float('inf'), aligned = False, lengthFunc = min): |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
546 self.similarityFunc = similarityFunc |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
547 self.aligned = aligned |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
548 self.delta = delta |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
549 self.lengthFunc = lengthFunc |
389
6d26dcc7bba0
modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
381
diff
changeset
|
550 self.subSequenceIndices = [(0,0)] |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
551 |
373
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
552 def similarities(self, l1, l2, jshift=0): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
553 from numpy import zeros, int as npint |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
554 n1 = len(l1) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
555 n2 = len(l2) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
556 self.similarityTable = zeros((n1+1,n2+1), dtype = npint) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
557 for i in xrange(1,n1+1): |
374
a7af3519687e
finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
373
diff
changeset
|
558 for j in xrange(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
559 if self.similarityFunc(l1[i-1], l2[j-1]): |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
560 self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1 |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
561 else: |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
562 self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1]) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
563 |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
564 def subSequence(self, i, j): |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
565 '''Returns the subsequence of two sequences |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
566 http://en.wikipedia.org/wiki/Longest_common_subsequence_problem''' |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
567 if i == 0 or j == 0: |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
568 return [] |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
569 elif self.similarityTable[i][j] == self.similarityTable[i][j-1]: |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
570 return self.subSequence(i, j-1) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
571 elif self.similarityTable[i][j] == self.similarityTable[i-1][j]: |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
572 return self.subSequence(i-1, j) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
573 else: |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
574 return self.subSequence(i-1, j-1) + [(i-1,j-1)] |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
575 |
373
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
576 def _compute(self, _l1, _l2, computeSubSequence = False): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
577 '''returns the longest common subsequence similarity |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
578 based on the threshold on distance between two elements of lists l1, l2 |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
579 similarityFunc returns True or False whether the two points are considered similar |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
580 |
607 | 581 if aligned, returns the best matching if using a finite delta by shifting the series alignments |
373
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
582 |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
583 eg distance(p1, p2) < epsilon |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
584 ''' |
372
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
585 if len(_l2) < len(_l1): # l1 is the shortest |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
586 l1 = _l2 |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
587 l2 = _l1 |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
588 revertIndices = True |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
589 else: |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
590 l1 = _l1 |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
591 l2 = _l2 |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
592 revertIndices = False |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
593 n1 = len(l1) |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
594 n2 = len(l2) |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
595 |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
596 if self.aligned: |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
597 lcssValues = {} |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
598 similarityTables = {} |
374
a7af3519687e
finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
373
diff
changeset
|
599 for i in xrange(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2 |
373
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
600 self.similarities(l1, l2, i) |
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
601 lcssValues[i] = self.similarityTable.max() |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
602 similarityTables[i] = self.similarityTable |
374
a7af3519687e
finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
373
diff
changeset
|
603 #print self.similarityTable |
521
3707eeb20f25
changed argMaxDict name to argmaxDict
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
518
diff
changeset
|
604 alignmentShift = argmaxDict(lcssValues) # ideally get the medium alignment shift, the one that minimizes distance |
389
6d26dcc7bba0
modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
381
diff
changeset
|
605 self.similarityTable = similarityTables[alignmentShift] |
372
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
606 else: |
389
6d26dcc7bba0
modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
381
diff
changeset
|
607 alignmentShift = 0 |
372
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
608 self.similarities(l1, l2) |
373
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
609 |
374
a7af3519687e
finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
373
diff
changeset
|
610 # threshold values for the useful part of the similarity table are n2-n1-delta and n1-n2-delta |
389
6d26dcc7bba0
modifications to compute alignment for None indicators
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
381
diff
changeset
|
611 self.similarityTable = self.similarityTable[:min(n1, n2+alignmentShift+self.delta)+1, :min(n2, n1-alignmentShift+self.delta)+1] |
373
d0b86ed50f32
work in progress on LCSS
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
372
diff
changeset
|
612 |
372
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
613 if computeSubSequence: |
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
614 self.subSequenceIndices = self.subSequence(self.similarityTable.shape[0]-1, self.similarityTable.shape[1]-1) |
371
924e38c9f70e
work in progress
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
370
diff
changeset
|
615 if revertIndices: |
374
a7af3519687e
finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
373
diff
changeset
|
616 self.subSequenceIndices = [(j,i) for i,j in self.subSequenceIndices] |
372
349eb1e09f45
Cleaned the methods/functions indicating if a point is in a polygon
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
371
diff
changeset
|
617 return self.similarityTable[-1,-1] |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
618 |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
619 def compute(self, l1, l2, computeSubSequence = False): |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
620 '''get methods are to be shadowed in child classes ''' |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
621 return self._compute(l1, l2, computeSubSequence) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
622 |
375
2ea8584aa80a
making indicator LCSS work
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
374
diff
changeset
|
623 def computeAlignment(self): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
624 from numpy import mean |
374
a7af3519687e
finished implementation of aligned LCSS with matching sequence decoded
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
373
diff
changeset
|
625 return mean([j-i for i,j in self.subSequenceIndices]) |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
626 |
376
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
627 def _computeNormalized(self, l1, l2, computeSubSequence = False): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
628 ''' compute the normalized LCSS |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
629 ie, the LCSS divided by the min or mean of the indicator lengths (using lengthFunc) |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
630 lengthFunc = lambda x,y:float(x,y)/2''' |
376
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
631 return float(self._compute(l1, l2, computeSubSequence))/self.lengthFunc(len(l1), len(l2)) |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
632 |
376
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
633 def computeNormalized(self, l1, l2, computeSubSequence = False): |
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
634 return self._computeNormalized(l1, l2, computeSubSequence) |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
635 |
376
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
636 def _computeDistance(self, l1, l2, computeSubSequence = False): |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
637 ''' compute the LCSS distance''' |
376
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
638 return 1-self._computeNormalized(l1, l2, computeSubSequence) |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
639 |
376
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
640 def computeDistance(self, l1, l2, computeSubSequence = False): |
2e6b8610bcaa
work on indicator similarity
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
375
diff
changeset
|
641 return self._computeDistance(l1, l2, computeSubSequence) |
370
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
642 |
97e8fa0ee9a1
work in progress for complete alignment
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
369
diff
changeset
|
643 ######################### |
45
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
644 # plotting section |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
645 ######################### |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
646 |
332
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
647 def plotPolygon(poly, options = ''): |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
648 'Plots shapely polygon poly' |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
649 from numpy.core.multiarray import array |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
650 from matplotlib.pyplot import plot |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
651 from shapely.geometry import Polygon |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
652 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
653 tmp = array(poly.exterior) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
654 plot(tmp[:,0], tmp[:,1], options) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
655 |
324 | 656 def stepPlot(X, firstX, lastX, initialCount = 0, increment = 1): |
657 '''for each value in X, increment by increment the initial count | |
297
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
658 returns the lists that can be plotted |
324 | 659 to obtain a step plot increasing by one for each value in x, from first to last value |
660 firstX and lastX should be respectively smaller and larger than all elements in X''' | |
297
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
661 |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
662 sortedX = [] |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
663 counts = [initialCount] |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
664 for x in sorted(X): |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
665 sortedX += [x,x] |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
666 counts.append(counts[-1]) |
324 | 667 counts.append(counts[-1]+increment) |
297
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
668 counts.append(counts[-1]) |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
669 return [firstX]+sortedX+[lastX], counts |
f6f423e25c7f
adding function to generate step plots (for cumulative number of vehicles)
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
286
diff
changeset
|
670 |
665
15e244d2a1b5
corrected bug with circular import for VideoFilenameAddable, moved to base module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
659
diff
changeset
|
671 class PlottingPropertyValues(object): |
45
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
672 def __init__(self, values): |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
673 self.values = values |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
674 |
116
2bf5b76320c0
moved intersection plotting and added markers for scatter plots
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
115
diff
changeset
|
675 def __getitem__(self, i): |
45
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
676 return self.values[i%len(self.values)] |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
677 |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
678 markers = PlottingPropertyValues(['+', '*', ',', '.', 'x', 'D', 's', 'o']) |
116
2bf5b76320c0
moved intersection plotting and added markers for scatter plots
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
115
diff
changeset
|
679 scatterMarkers = PlottingPropertyValues(['s','o','^','>','v','<','d','p','h','8','+','x']) |
45
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
680 |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
681 linestyles = PlottingPropertyValues(['-', '--', '-.', ':']) |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
682 |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
683 colors = PlottingPropertyValues('brgmyck') # 'w' |
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
684 |
115
550556378466
added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
86
diff
changeset
|
685 def plotIndicatorMap(indicatorMap, squareSize, masked = True, defaultValue=-1): |
65
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
686 from numpy import array, arange, ones, ma |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
687 from matplotlib.pyplot import pcolor |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
688 coords = array(indicatorMap.keys()) |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
689 minX = min(coords[:,0]) |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
690 minY = min(coords[:,1]) |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
691 X = arange(minX, max(coords[:,0])+1.1)*squareSize |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
692 Y = arange(minY, max(coords[:,1])+1.1)*squareSize |
115
550556378466
added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
86
diff
changeset
|
693 C = defaultValue*ones((len(Y), len(X))) |
65
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
694 for k,v in indicatorMap.iteritems(): |
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
695 C[k[1]-minY,k[0]-minX] = v |
115
550556378466
added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
86
diff
changeset
|
696 if masked: |
550556378466
added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
86
diff
changeset
|
697 pcolor(X, Y, ma.masked_where(C==defaultValue,C)) |
550556378466
added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
86
diff
changeset
|
698 else: |
550556378466
added functionalities to indicator maps
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
86
diff
changeset
|
699 pcolor(X, Y, C) |
65
75cf537b8d88
moved and generalized map making functions to the library
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
48
diff
changeset
|
700 |
45
74d2de078baf
added colors, linestyles and markers to pick from
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
42
diff
changeset
|
701 ######################### |
637
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
702 # Data download |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
703 ######################### |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
704 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
705 def downloadECWeather(stationID, years, months = [], outputDirectoryname = '.', english = True): |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
706 '''Downloads monthly weather data from Environment Canada |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
707 If month is provided (number 1 to 12), it means hourly data for the whole month |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
708 Otherwise, means the data for each day, for the whole year |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
709 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
710 Example: MONTREAL MCTAVISH 10761 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
711 MONTREALPIERRE ELLIOTT TRUDEAU INTL A 5415 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
712 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
713 To get daily data for 2010 and 2011, downloadECWeather(10761, [2010,2011], [], '/tmp') |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
714 To get hourly data for 2009 and 2012, January, March and October, downloadECWeather(10761, [2009,2012], [1,3,10], '/tmp')''' |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
715 import urllib2 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
716 if english: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
717 language = 'e' |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
718 else: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
719 language = 'f' |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
720 if len(months) == 0: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
721 timeFrame = 2 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
722 months = [1] |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
723 else: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
724 timeFrame = 1 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
725 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
726 for year in years: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
727 for month in months: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
728 url = urllib2.urlopen('http://climat.meteo.gc.ca/climateData/bulkdata_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es'.format(language, stationID, year, month, timeFrame)) |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
729 data = url.read() |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
730 outFilename = '{}/{}-{}'.format(outputDirectoryname, stationID, year) |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
731 if timeFrame == 1: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
732 outFilename += '-{}-hourly'.format(month) |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
733 else: |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
734 outFilename += '-daily' |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
735 outFilename += '.csv' |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
736 out = open(outFilename, 'w') |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
737 out.write(data) |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
738 out.close() |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
739 |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
740 ######################### |
c9a0b72979fd
added function to get canadian public weather data
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
615
diff
changeset
|
741 # File I/O |
27
44689029a86f
updated segmentIntersection and other
Nicolas Saunier <nico@confins.net>
parents:
24
diff
changeset
|
742 ######################### |
24
6fb59cfb201e
first version of segmentIntersection
Nicolas Saunier <nico@confins.net>
parents:
19
diff
changeset
|
743 |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
744 def removeExtension(filename, delimiter = '.'): |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
745 '''Returns the filename minus the extension (all characters after last .)''' |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
746 i = filename.rfind(delimiter) |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
747 if i>0: |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
748 return filename[:i] |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
749 else: |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
750 return filename |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
751 |
46
b5d007612e16
added filename util
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
45
diff
changeset
|
752 def cleanFilename(s): |
b5d007612e16
added filename util
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
45
diff
changeset
|
753 'cleans filenames obtained when contatenating figure characteristics' |
266
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
754 return s.replace(' ','-').replace('.','').replace('/','-') |
46
b5d007612e16
added filename util
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
45
diff
changeset
|
755 |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
756 def listfiles(dirname, extension, remove = False): |
14
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
757 '''Returns the list of files with the extension in the directory dirname |
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
758 If remove is True, the filenames are stripped from the extension''' |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
759 from os import listdir |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
760 tmp = [f for f in listdir(dirname) if f.endswith(extension)] |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
761 tmp.sort() |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
762 if remove: |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
763 return [removeExtension(f, extension) for f in tmp] |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
764 else: |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
765 return tmp |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
766 |
266
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
767 def mkdir(dirname): |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
768 'Creates a directory if it does not exist' |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
769 import os |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
770 if not os.path.exists(dirname): |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
771 os.mkdir(dirname) |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
772 else: |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
773 print(dirname+' already exists') |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
774 |
14
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
775 def removeFile(filename): |
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
776 '''Deletes the file while avoiding raising an error |
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
777 if the file does not exist''' |
266
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
778 import os |
14
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
779 if (os.path.exists(filename)): |
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
780 os.remove(filename) |
266
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
781 else: |
aba9711b3149
small modificatons and reorganization
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
262
diff
changeset
|
782 print(filename+' does not exist') |
14
e7bbe8465591
homography and other utils
Nicolas Saunier <nico@confins.net>
parents:
7
diff
changeset
|
783 |
42
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
784 def line2Floats(l, separator=' '): |
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
785 '''Returns the list of floats corresponding to the string''' |
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
786 return [float(x) for x in l.split(separator)] |
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
787 |
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
788 def line2Ints(l, separator=' '): |
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
789 '''Returns the list of ints corresponding to the string''' |
1a2ac2d4f53a
added loading of the rest of the data for objects
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
35
diff
changeset
|
790 return [int(x) for x in l.split(separator)] |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
791 |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
792 ######################### |
332
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
793 # CLI utils |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
794 ######################### |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
795 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
796 def parseCLIOptions(helpMessage, options, cliArgs, optionalOptions=[]): |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
797 ''' Simple function to handle similar argument parsing |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
798 Returns the dictionary of options and their values |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
799 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
800 * cliArgs are most likely directly sys.argv |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
801 (only the elements after the first one are considered) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
802 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
803 * options should be a list of strings for getopt options, |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
804 eg ['frame=','correspondences=','video='] |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
805 A value must be provided for each option, or the program quits''' |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
806 import sys, getopt |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
807 from numpy.core.fromnumeric import all |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
808 optionValues, args = getopt.getopt(cliArgs[1:], 'h', ['help']+options+optionalOptions) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
809 optionValues = dict(optionValues) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
810 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
811 if '--help' in optionValues.keys() or '-h' in optionValues.keys(): |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
812 print(helpMessage+ |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
813 '\n - Compulsory options: '+' '.join([opt.replace('=','') for opt in options])+ |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
814 '\n - Non-compulsory options: '+' '.join([opt.replace('=','') for opt in optionalOptions])) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
815 sys.exit() |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
816 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
817 missingArgument = [('--'+opt.replace('=','') in optionValues.keys()) for opt in options] |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
818 if not all(missingArgument): |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
819 print('Missing argument') |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
820 print(optionValues) |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
821 sys.exit() |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
822 |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
823 return optionValues |
a6ca86107f27
reorganized utils module
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
324
diff
changeset
|
824 |
397
b36b00dd27c3
added function to read scene metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
395
diff
changeset
|
825 |
b36b00dd27c3
added function to read scene metadata
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
395
diff
changeset
|
826 ######################### |
553
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
827 # Profiling |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
828 ######################### |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
829 |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
830 def analyzeProfile(profileFilename, stripDirs = True): |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
831 '''Analyze the file produced by cProfile |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
832 |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
833 obtained by for example: |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
834 - call in script (for main() function in script) |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
835 import cProfile, os |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
836 cProfile.run('main()', os.path.join(os.getcwd(),'main.profile')) |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
837 |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
838 - or on the command line: |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
839 python -m cProfile [-o profile.bin] [-s sort] scriptfile [arg]''' |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
840 import pstats, os |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
841 p = pstats.Stats(os.path.join(os.pardir, profileFilename)) |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
842 if stripDirs: |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
843 p.strip_dirs() |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
844 p.sort_stats('time') |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
845 p.print_stats(.2) |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
846 #p.sort_stats('time') |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
847 # p.print_callees(.1, 'int_prediction.py:') |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
848 return p |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
849 |
3622a5653ee9
added basic info and function to profile code
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
547
diff
changeset
|
850 ######################### |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
851 # running tests |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
852 ######################### |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
853 |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
854 if __name__ == "__main__": |
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
855 import doctest |
2
de5642925615
started implementation of TimeInterval and Spatio-temporal object
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
0
diff
changeset
|
856 import unittest |
31
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
857 suite = doctest.DocFileSuite('tests/utils.txt') |
c000f37c316d
moved tests to independent file, added chi2 computation
Nicolas Saunier <nico@confins.net>
parents:
29
diff
changeset
|
858 #suite = doctest.DocTestSuite() |
2
de5642925615
started implementation of TimeInterval and Spatio-temporal object
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
0
diff
changeset
|
859 unittest.TextTestRunner().run(suite) |
de5642925615
started implementation of TimeInterval and Spatio-temporal object
Nicolas Saunier <nicolas.saunier@polymtl.ca>
parents:
0
diff
changeset
|
860 #doctest.testmod() |
0
aed8eb63cdde
initial commit with non-functional python code for NGSIM
Nicolas Saunier <nico@confins.net>
parents:
diff
changeset
|
861 #doctest.testfile("example.txt") |