1
import pandas as pd
2
import numpy as np
3
from sklearn.feature_selection import SelectKBest
4
from sklearn.feature_selection import chi2
5

6
DATA_FILE = 'sample-data/mobile-price-classification/train.csv'
7

8
uv_data = pd.read_csv(DATA_FILE)
9

10
uv_data.head()
11

12
uv_x = uv_data.iloc[:, 0:20] # Read in the first 20 columns
13
uv_y = uv_data.iloc[:, -1]   # Read in the last column
14

15
feature_count = 10                   # Number of features we want to select
16

17
scores = SelectKBest(score_func=chi2, k = 'all').fit(uv_x, uv_y).scores_
18

19
df_fit  = pd.DataFrame(scores)        # Scores as DF
20
df_cols = pd.DataFrame(uv_x.columns)  # Column names as DF
21

22
df_scores = pd.concat([df_cols, df_fit], axis=1)
23
df_scores.columns = ['Feature', 'Score']
24

25
df_scores.nlargest(feature_count, 'Score')

1
import pandas as pd
2
import numpy as np
3
from sklearn.ensemble import ExtraTreesClassifier
4
import matplotlib.pyplot as plt
5

6
fs_data = pd.read_csv(DATA_FILE)
7
fs_x = fs_data.iloc[:,0:20]
8
fs_y = fs_data.iloc[:,-1]
9

10
classifier = ExtraTreesClassifier()  # Create classifier instance
11
classifier.fit(fs_x, fs_y)           # Train the Classifier
12

13
fs_importance = classifier.feature_importances_
14

15
print(fs_importance)
16

17
df_importance = pd.Series(fs_importance, index=fs_x.columns)
18
df_importance.nlargest
19

20
df_importance.nlargest(10).plot(kind='barh')
21
plt.show()

1
import pandas as pd
2
import numpy as np
3
import seaborn as sns
4

5
cm_data = pd.read_csv(DATA_FILE)
6

7
cm_data.head()
8

9
cm_x = cm_data.iloc[:,0:20] # Extract feature columns
10
cm_y = cm_data.iloc[:,-1]   # Extract target column
11

12
correlation_matrix = cm_data.corr()
13

14
correlation_matrix
15

16
top_correlation_features = correlation_matrix.index
17
plt.figure(figsize=(20,20))
18

19
_ = sns.heatmap(cm_data[top_correlation_features].corr(), annot=True, cmap="RdYlGn")

1
def plot_df_correlation(df):
2
    plt.figure(figsize=(20,20))
3
    return sns.heatmap(df[df.corr().index].corr(), annot=True, cmap="coolwarm")

1
domain_counts = df_domain_invoice['Email Domain'].value_counts()
2

3
replace_domains = domain_counts[domain_counts < 100].index
4

5
# on-hot encoding of Domains
6
df_domain_invoice = pd.get_dummies(df_domain_invoice['Email Domain']
7
                                           .replace(replace_domains, 'other_'),
8
                                       columns = ['Email Domain'],
9
                                       drop_first=False)

1
import scipy.stats as ss
2
from collections import Counter
3
import math
4
import pandas as pd
5
import numpy as np
6
import seaborn as sns
7
from matplotlib import pyplot as plt
8
from scipy import stats
9
import numpy as np
10

11
def convert(data, to):
12
    converted = None
13
    if to == 'array':
14
        if isinstance(data, np.ndarray):
15
            converted = data
16
        elif isinstance(data, pd.Series):
17
            converted = data.values
18
        elif isinstance(data, list):
19
            converted = np.array(data)
20
        elif isinstance(data, pd.DataFrame):
21
            converted = data.as_matrix()
22
    elif to == 'list':
23
        if isinstance(data, list):
24
            converted = data
25
        elif isinstance(data, pd.Series):
26
            converted = data.values.tolist()
27
        elif isinstance(data, np.ndarray):
28
            converted = data.tolist()
29
    elif to == 'dataframe':
30
        if isinstance(data, pd.DataFrame):
31
            converted = data
32
        elif isinstance(data, np.ndarray):
33
            converted = pd.DataFrame(data)
34
    else:
35
        raise ValueError("Unknown data conversion: {}".format(to))
36
    if converted is None:
37
        raise TypeError('cannot handle data conversion of type: {} to {}'.format(type(data),to))
38
    else:
39
        return converted
40

41
def conditional_entropy(x, y):
42
    """
43
    Calculates the conditional entropy of x given y: S(x|y)
44
    Wikipedia: https://en.wikipedia.org/wiki/Conditional_entropy
45
    :param x: list / NumPy ndarray / Pandas Series
46
        A sequence of measurements
47
    :param y: list / NumPy ndarray / Pandas Series
48
        A sequence of measurements
49
    :return: float
50
    """
51
    # entropy of x given y
52
    y_counter = Counter(y)
53
    xy_counter = Counter(list(zip(x,y)))
54
    total_occurrences = sum(y_counter.values())
55
    entropy = 0.0
56
    for xy in xy_counter.keys():
57
        p_xy = xy_counter[xy] / total_occurrences
58
        p_y = y_counter[xy[1]] / total_occurrences
59
        entropy += p_xy * math.log(p_y/p_xy)
60
    return entropy
61

62
def cramers_v(x, y):
63
    confusion_matrix = pd.crosstab(x,y)
64
    chi2 = ss.chi2_contingency(confusion_matrix)[0]
65
    n = confusion_matrix.sum().sum()
66
    phi2 = chi2/n
67
    r,k = confusion_matrix.shape
68
    phi2corr = max(0, phi2-((k-1)*(r-1))/(n-1))
69
    rcorr = r-((r-1)**2)/(n-1)
70
    kcorr = k-((k-1)**2)/(n-1)
71
    return np.sqrt(phi2corr/min((kcorr-1),(rcorr-1)))
72

73
def theils_u(x, y):
74
    s_xy = conditional_entropy(x,y)
75
    x_counter = Counter(x)
76
    total_occurrences = sum(x_counter.values())
77
    p_x = list(map(lambda n: n/total_occurrences, x_counter.values()))
78
    s_x = ss.entropy(p_x)
79
    if s_x == 0:
80
        return 1
81
    else:
82
        return (s_x - s_xy) / s_x
83

84
def correlation_ratio(categories, measurements):
85
    fcat, _ = pd.factorize(categories)
86
    cat_num = np.max(fcat)+1
87
    y_avg_array = np.zeros(cat_num)
88
    n_array = np.zeros(cat_num)
89
    for i in range(0,cat_num):
90
        cat_measures = measurements[np.argwhere(fcat == i).flatten()]
91
        n_array[i] = len(cat_measures)
92
        y_avg_array[i] = np.average(cat_measures)
93
    y_total_avg = np.sum(np.multiply(y_avg_array,n_array))/np.sum(n_array)
94
    numerator = np.sum(np.multiply(n_array,np.power(np.subtract(y_avg_array,y_total_avg),2)))
95
    denominator = np.sum(np.power(np.subtract(measurements,y_total_avg),2))
96
    if numerator == 0:
97
        eta = 0.0
98
    else:
99
        eta = numerator/denominator
100
    return eta
101

102
def associations(dataset, nominal_columns=None, mark_columns=False, theil_u=False, plot=True,
103
                          return_results = False, **kwargs):
104
    """
105
    Calculate the correlation/strength-of-association of features in data-set with both categorical (eda_tools) and
106
    continuous features using:
107
     - Pearson's R for continuous-continuous cases
108
     - Correlation Ratio for categorical-continuous cases
109
     - Cramer's V or Theil's U for categorical-categorical cases
110
    :param dataset: NumPy ndarray / Pandas DataFrame
111
        The data-set for which the features' correlation is computed
112
    :param nominal_columns: string / list / NumPy ndarray
113
        Names of columns of the data-set which hold categorical values. Can also be the string 'all' to state that all
114
        columns are categorical, or None (default) to state none are categorical
115
    :param mark_columns: Boolean (default: False)
116
        if True, output's columns' names will have a suffix of '(nom)' or '(con)' based on there type (eda_tools or
117
        continuous), as provided by nominal_columns
118
    :param theil_u: Boolean (default: False)
119
        In the case of categorical-categorical feaures, use Theil's U instead of Cramer's V
120
    :param plot: Boolean (default: True)
121
        If True, plot a heat-map of the correlation matrix
122
    :param return_results: Boolean (default: False)
123
        If True, the function will return a Pandas DataFrame of the computed associations
124
    :param kwargs:
125
        Arguments to be passed to used function and methods
126
    :return: Pandas DataFrame
127
        A DataFrame of the correlation/strength-of-association between all features
128
    """
129

130
    dataset = convert(dataset, 'dataframe')
131
    columns = dataset.columns
132
    if nominal_columns is None:
133
        nominal_columns = list()
134
    elif nominal_columns == 'all':
135
        nominal_columns = columns
136
    corr = pd.DataFrame(index=columns, columns=columns)
137
    for i in range(0,len(columns)):
138
        for j in range(i,len(columns)):
139
            if i == j:
140
                corr[columns[i]][columns[j]] = 1.0
141
            else:
142
                if columns[i] in nominal_columns:
143
                    if columns[j] in nominal_columns:
144
                        if theil_u:
145
                            corr[columns[j]][columns[i]] = theils_u(dataset[columns[i]],dataset[columns[j]])
146
                            corr[columns[i]][columns[j]] = theils_u(dataset[columns[j]],dataset[columns[i]])
147
                        else:
148
                            cell = cramers_v(dataset[columns[i]],dataset[columns[j]])
149
                            corr[columns[i]][columns[j]] = cell
150
                            corr[columns[j]][columns[i]] = cell
151
                    else:
152
                        cell = correlation_ratio(dataset[columns[i]], dataset[columns[j]])
153
                        corr[columns[i]][columns[j]] = cell
154
                        corr[columns[j]][columns[i]] = cell
155
                else:
156
                    if columns[j] in nominal_columns:
157
                        cell = correlation_ratio(dataset[columns[j]], dataset[columns[i]])
158
                        corr[columns[i]][columns[j]] = cell
159
                        corr[columns[j]][columns[i]] = cell
160
                    else:
161
                        cell, _ = ss.pearsonr(dataset[columns[i]], dataset[columns[j]])
162
                        corr[columns[i]][columns[j]] = cell
163
                        corr[columns[j]][columns[i]] = cell
164
    corr.fillna(value=np.nan, inplace=True)
165
    if mark_columns:
166
        marked_columns = ['{} (nom)'.format(col) if col in nominal_columns else '{} (con)'.format(col) for col in columns]
167
        corr.columns = marked_columns
168
        corr.index = marked_columns
169
    if plot:
170
        plt.figure(figsize=(20,20))#kwargs.get('figsize',None))
171
        sns.heatmap(corr, annot=kwargs.get('annot',True), fmt=kwargs.get('fmt','.2f'), cmap='coolwarm')
172
        plt.show()
173
    if return_results:
174
        return corr