ada_boost算法实现

integr/my_ada_boost.py

@@ -0,0 +1,86 @@
+# -*- encoding:utf-8 -*-
+from sklearn.datasets import load_wine, load_iris, load_breast_cancer
+from sklearn.model_selection import train_test_split
+from sklearn.tree import DecisionTreeClassifier
+import numpy as np
+from tree import my_tree
+
+wine = load_breast_cancer()
+Xtrain, Xtest, Ytrain, Ytest = train_test_split(wine.data,wine.target,test_size=0.3)
+
+# 设置数据初始权重
+w = np.array([1/len(Ytrain) for i in range(len(Ytrain))])
+# 决策树权重
+alpha = []
+# 决策树数组
+trees = []
+pn = 1/len(Ytrain)
+
+for i in range(len(Ytrain)):
+    if Ytrain[i] == 0:
+        Ytrain[i] = -1
+for i in range(len(Ytest)):
+    if Ytest[i] == 0:
+        Ytest[i] = -1
+print(Ytest)
+
+for i in range(30):
+    # 训练决策树
+    # clf = DecisionTreeClassifier(criterion="entropy", max_features=1, max_depth=1,
+    #                              class_weight={0:w0, 1:1-w0})  # 实例化，criterion不写的话默认是基尼系数
+    nodes = my_tree.fit(Xtrain, Ytrain, None, 0, w)
+
+    # my_tree.print_width([nodes], 1)
+    # print("熵值", my_tree.calc_ent(Ytrain, w))
+    Xpredit = my_tree.predict(Xtrain, Ytrain, nodes)
+    error = 0
+    p_error = 0
+    for j in range(len(Ytrain)):
+        if Xpredit[j] != Ytrain[j]:
+            error += w[j]
+            p_error += pn
+
+    if error > 0.5:
+        continue
+    if error == 0:
+        error = 0.001
+
+    print("第", i, "轮错误率", p_error, error)
+    ab = 0.5*np.log2(1/error - 1)
+
+    # 更新权重
+    for j in range(len(Ytrain)):
+        w[j] = w[j]*np.exp(-ab*Ytrain[j]*Xpredit[j])
+    sum_w = sum(w)
+    w = w/sum_w
+
+    alpha.append(ab)
+    trees.append(nodes)
+
+predicts = []
+for tree in trees:
+    predicts.append(my_tree.predict(Xtest, None, tree))
+
+print(alpha)
+# 结果加权
+result = np.zeros(len(Xtest), float)
+for p in predicts:
+    r = 0
+    for w_alpha in alpha:
+        r += w_alpha * p
+    result = result + r
+
+print("sign前:" , result)
+result = np.sign(result)
+print("sign后:", result)
+
+# print(1- sum(np.bitwise_xor(Ytest, result))/len(result))
+# print(result == Ytest)
+print(len([i for i in result == Ytest if i])/len(result))
+# cmp = np.concatenate(([result], [Ytest]), axis=0)
+# print(cmp)
+
+
+clf = DecisionTreeClassifier(criterion="entropy", max_features=1, max_depth=1)
+clf = clf.fit(Xtrain, Ytrain)
+print(clf.score(Xtest, Ytest))

tree/my_tree.py

@@ -5,7 +5,8 @@ from sklearn.model_selection import train_test_split
 import numpy as np
 
 feature_name = ['酒精', '苹果酸', '灰', '灰的碱性', '镁', '总酚', '类黄酮',
-                '非黄烷类酚类', '花青素', '颜色强度', '色调', 'od280/od315稀释葡萄酒', '脯氨酸']
+                '非黄烷类酚类', '花青素', '颜色强度', '色调', 'od280/od315稀释葡萄酒', '脯氨酸'
+                , 'A', 'B', 'c', 'D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T']
 class_names=["琴酒", "雪莉", "贝尔摩德"]
 
 # 生成决策树的节点类型
@@ -31,9 +32,9 @@ class TreeNode(object):
         self.left = left
         self.right = right
 
-        if self.y == -1:
-            self.y = np.where(value == np.max(value))[0][0]
-            print(self.y, self.value)
+        # if self.y == -1:
+        #     self.y = np.where(value == np.max(value))[0][0]
+            # print(self.y, self.value)
 
     def __str__(self):
         if self.idx == -1:
@@ -57,14 +58,18 @@ def read_data():
     return Xtrain, Xtest, Ytrain, Ytest
 
 
-def calc_ent(x):
+def calc_ent(x, weights=None):
     """
         calculate shanno ent of x
     """
     x_value_list = set([x[i] for i in range(x.shape[0])])
     ent = 0.0
     for x_value in x_value_list:
-        p = float(x[x == x_value].shape[0]) / x.shape[0]
+        if weights is None:
+            p = float(x[x == x_value].shape[0]) / x.shape[0]
+        else:
+            weights = weights/sum(weights)
+            p = sum(sum([x == x_value]*weights))
         logp = np.log2(p)
         ent -= p * logp
 
@@ -108,31 +113,33 @@ def calc_ent1(x):
 
 
 # 计算某个属性的信息增益
-def cal_ent_attr(Xtrain, Ytrain):
-    print('sharp', Xtrain.shape)
-
+def cal_ent_attr(Xtrain, Ytrain, weights):
+    # print('sharp', Xtrain.shape)
+    weights = weights / sum(weights)
     # 对每个属性
     min_ent = 100
     min_i = 0
     min_mean = 0
-    for i in range(Xtrain.shape[1]):
+
+    for i in np.random.randint(0,Xtrain.shape[1],size=(15)):
         x_value_list = set([Xtrain[j][i] for j in range(Xtrain.shape[0])])
         mean = sum(x_value_list)/len(x_value_list)
         sum_ent = 0
         # 二叉树
         p = Ytrain[Xtrain[:, i] > mean]
-        sum_ent = sum_ent + calc_ent(p)*len(p)/len(Ytrain)
+        p0 = sum(weights[Xtrain[:, i] > mean])
+        sum_ent = sum_ent + calc_ent(p, weights[Xtrain[:, i] > mean])*p0
         p = Ytrain[Xtrain[:, i] <= mean]
-        sum_ent = sum_ent + calc_ent(p)*len(p)/len(Ytrain)
+        sum_ent = sum_ent + calc_ent(p, weights[Xtrain[:, i] <= mean])*(1-p0)
 
-        if sum_ent < min_ent:
+        if sum_ent <= min_ent:
             min_ent = sum_ent
             min_i = i
             min_mean = mean
     return min_i, min_mean, min_ent
 
 
-def cal_max_ent_attr_c45(Xtrain, Ytrain):
+def cal_max_ent_attr_c45(Xtrain, Ytrain, weights=None):
     max_ent = 0
     max_mean = 0
     h = calc_ent(Ytrain)
@@ -140,12 +147,17 @@ def cal_max_ent_attr_c45(Xtrain, Ytrain):
         left = Xtrain[:k + 1]
         right = Xtrain[k + 1:]
 
-        left_ent = calc_ent(Ytrain[:k+1])*len(left)/len(Ytrain)
-        right_ent = calc_ent(Ytrain[k + 1:])*len(right)/len(Ytrain)
+        if weights is None:
+            left_ent = calc_ent(Ytrain[:k+1])*len(left)/len(Ytrain)
+            right_ent = calc_ent(Ytrain[k + 1:])*len(right)/len(Ytrain)
+
+        else:
+            pass
 
         iv = -len(left) / len(Ytrain) * np.log2(len(left) / len(Ytrain))
         iv -= len(right) / len(Ytrain) * np.log2(len(right) / len(Ytrain))
 
+
         gain_ent = (h - left_ent - right_ent)/iv
 
         if gain_ent > max_ent:
@@ -153,6 +165,8 @@ def cal_max_ent_attr_c45(Xtrain, Ytrain):
             max_mean = left[-1]
     return  max_ent, max_mean
 
+# 样本权重
+weights = []
 
 # 计算某个属性的信息增益率
 def cal_ent_attr_c45(Xtrain, Ytrain):
@@ -174,7 +188,7 @@ def cal_ent_attr_c45(Xtrain, Ytrain):
 
 # 计算某个属性的基尼指数
 def cal_gini_attr(Xtrain, Ytrain):
-    print('sharp', Xtrain.shape)
+    # print('sharp', Xtrain.shape)
 
     # 对每个属性
     min_ent = 100
@@ -196,7 +210,7 @@ def cal_gini_attr(Xtrain, Ytrain):
             min_mean = mean
     return min_i, min_mean, min_ent
 
-MAX_T = 5
+MAX_T = 1
 
 
 def is_end(Ytrain):
@@ -206,22 +220,24 @@ def is_end(Ytrain):
         return True
 
 # 强行划分为叶子节点
-def leaf_node(Ytrain):
+def leaf_node(Ytrain, weights):
     p_set = []
+    k = 0
     for item in Ytrain:
         for i in p_set:
             if i[0] == item:
-                i[1] = i[1] + 1
+                i[1] = i[1] + weights[k]
                 break
         else:
-            i = [item, 1]
+            i = [item, weights[k]]
             p_set.append(i)
+        k = k + 1
 
     max_item = [0, 0]
     for item in p_set:
         if item[1] > max_item[1]:
             max_item = item
-    print('这个是叶子节点，value:', max_item[0])
+    # print('这个是叶子节点，value:', max_item[0])
     return TreeNode(-1, 0, 0, True, max_item[0], len(Ytrain), distrib(Ytrain))
 
 
@@ -235,18 +251,18 @@ def distrib(Ytrain):
     return d_list
 
 
-def fit(Xtrain, Ytrain, parent_node, depth):
+def fit(Xtrain, Ytrain, parent_node, depth, weights):
 
     if is_end(Ytrain):
-        print('这个是叶子节点')
+        # print('这个是叶子节点')
         return TreeNode(-1, 0, 0, True, -1, len(Ytrain), distrib(Ytrain))
 
     if depth >= MAX_T:
-        return leaf_node(Ytrain)
+        return leaf_node(Ytrain, weights)
 
-    i, mean, min_ent = cal_ent_attr_c45(Xtrain, Ytrain)
+    i, mean, min_ent = cal_ent_attr(Xtrain, Ytrain, weights)
     total_ent = calc_ent(Ytrain)
-    print("第", i, "个属性,mean:", mean)
+    # print("第", i, "个属性,mean:", mean)
     # 生成节点
     parent_node = TreeNode(i, mean, total_ent - min_ent, False, -1, len(Ytrain), distrib(Ytrain))
 
@@ -255,12 +271,12 @@ def fit(Xtrain, Ytrain, parent_node, depth):
     right_Xtrain = Xtrain[Xtrain[:, i] > mean]
     # right_Xtrain = np.delete(right_Xtrain, i, axis=1) # 这个属性还可以再被切分
 
-    right_node = fit(right_Xtrain, right_Ytrain, parent_node, depth+1)
+    right_node = fit(right_Xtrain, right_Ytrain, parent_node, depth+1, weights[Xtrain[:, i] > mean])
 
     left_Ytrain = Ytrain[Xtrain[:, i] <= mean]
     left_Xtrain = Xtrain[Xtrain[:, i] <= mean]
     # left_Xtrain = np.delete(left_Xtrain, i, axis=1)
-    left_node = fit(left_Xtrain, left_Ytrain, parent_node, depth + 1)
+    left_node = fit(left_Xtrain, left_Ytrain, parent_node, depth + 1, weights[Xtrain[:, i] <= mean])
 
     parent_node.left = left_node
     parent_node.right = right_node
@@ -286,29 +302,37 @@ def print_width(nodes, depth):
 
 def predit_one(X, Y, node):
     if node.is_leaf:
-        print(class_names[node.y], class_names[Y])
+        # print(class_names[node.y], class_names[Y])
+        if node.y == 0:
+            return -1
+        return node.y
     else:
         if X[node.idx] <= node.idx_value:
-            predit_one(X,Y,node.left)
+            return predit_one(X,Y,node.left)
         else:
-            predit_one(X, Y, node.right)
+            return predit_one(X, Y, node.right)
 
 
-def predit(Xtest, Ytest, node):
-    for i in range(Xtest.shape[1]):
-        predit_one(Xtest[i], Ytest[i], node)
+def predict(Xtest, Ytest, node):
+    result = []
+    for i in range(Xtest.shape[0]):
+        result.append(predit_one(Xtest[i], None, node))
+    return np.array(result)
 
 
 if __name__ == '__main__':
     Xtrain, Xtest, Ytrain, Ytest = read_data()
     print(calc_ent1(Ytrain))
-    print(calc_ent(Ytrain))
+
+    weights = np.ones(len(Ytrain))/Ytrain.shape[0]
+    print("熵值", calc_ent(Ytrain))
+    print("熵值", calc_ent(Ytrain, weights))
 
     print("基尼指数", cal_gini(Ytrain))
 
     print("信息增益率", cal_ent_attr_c45(Xtrain, Ytrain))
 
-    node = fit(Xtrain, Ytrain, None, 0)
+    node = fit(Xtrain, Ytrain, None, 0, weights)
     print_width([node], 1)
 
-    predit(Xtest, Ytest, node)
+    print(predict(Xtest, Ytest, node))