使用C4.5算法

integr/my_ada_boost.py

@@ -24,13 +24,13 @@ for i in range(len(Ytest)):
         Ytest[i] = -1
 print(Ytest)
 
-for i in range(30):
+for i in range(3):
     # 训练决策树
     # clf = DecisionTreeClassifier(criterion="entropy", max_features=1, max_depth=1,
     #                              class_weight={0:w0, 1:1-w0})  # 实例化，criterion不写的话默认是基尼系数
     nodes = my_tree.fit(Xtrain, Ytrain, None, 0, w)
 
-    # my_tree.print_width([nodes], 1)
+    my_tree.print_width([nodes], 1)
     # print("熵值", my_tree.calc_ent(Ytrain, w))
     Xpredit = my_tree.predict(Xtrain, Ytrain, nodes)
     error = 0
@@ -76,11 +76,23 @@ print("sign后:", result)
 
 # print(1- sum(np.bitwise_xor(Ytest, result))/len(result))
 # print(result == Ytest)
-print(len([i for i in result == Ytest if i])/len(result))
+print()
+x = len([i for i in result == Ytest if i])/len(result)
+x = x + 0.5*len([i for i in result if i==0])/len(result)
+print(x)
 # cmp = np.concatenate(([result], [Ytest]), axis=0)
 # print(cmp)
 
 
-clf = DecisionTreeClassifier(criterion="entropy", max_features=1, max_depth=1)
-clf = clf.fit(Xtrain, Ytrain)
-print(clf.score(Xtest, Ytest))
+# clf = DecisionTreeClassifier(criterion="entropy", max_features=1, max_depth=2)
+# clf = clf.fit(Xtrain, Ytrain)
+# print(clf.score(Xtest, Ytest))
+
+w = np.array([1/len(Ytrain) for i in range(len(Ytrain))])
+my_tree_0 = trees[0]
+Xpredit = my_tree.predict(Xtest, None, nodes)
+error = 0
+for j in range(len(Xtest)):
+    if Xpredit[j] != Ytest[j]:
+        error += w[j]
+print(1-error)

tree/my_tree.py

@@ -32,8 +32,8 @@ class TreeNode(object):
         self.left = left
         self.right = right
 
-        # if self.y == -1:
-        #     self.y = np.where(value == np.max(value))[0][0]
+        if self.y is None:
+            self.y = np.where(value == np.max(value))[0][0] ## TODO
             # print(self.y, self.value)
 
     def __str__(self):
@@ -121,7 +121,7 @@ def cal_ent_attr(Xtrain, Ytrain, weights):
     min_i = 0
     min_mean = 0
 
-    for i in np.random.randint(0,Xtrain.shape[1],size=(15)):
+    for i in range(Xtrain.shape[1]):
         x_value_list = set([Xtrain[j][i] for j in range(Xtrain.shape[0])])
         mean = sum(x_value_list)/len(x_value_list)
         sum_ent = 0
@@ -142,6 +142,7 @@ def cal_ent_attr(Xtrain, Ytrain, weights):
 def cal_max_ent_attr_c45(Xtrain, Ytrain, weights=None):
     max_ent = 0
     max_mean = 0
+    weights = weights / sum(weights)
     h = calc_ent(Ytrain)
     for k in range(len(Xtrain) - 1):
         left = Xtrain[:k + 1]
@@ -150,12 +151,14 @@ def cal_max_ent_attr_c45(Xtrain, Ytrain, weights=None):
         if weights is None:
             left_ent = calc_ent(Ytrain[:k+1])*len(left)/len(Ytrain)
             right_ent = calc_ent(Ytrain[k + 1:])*len(right)/len(Ytrain)
-
+            iv = -len(left) / len(Ytrain) * np.log2(len(left) / len(Ytrain))
+            iv -= len(right) / len(Ytrain) * np.log2(len(right) / len(Ytrain))
         else:
-            pass
-
-        iv = -len(left) / len(Ytrain) * np.log2(len(left) / len(Ytrain))
-        iv -= len(right) / len(Ytrain) * np.log2(len(right) / len(Ytrain))
+            p = sum(weights[:k+1])
+            left_ent = calc_ent(Ytrain[:k + 1], weights[:k+1]) * p
+            right_ent = calc_ent(Ytrain[k + 1:], weights[k+1:]) * (1-p)
+            iv = -p * np.log2(p)
+            iv -= (1-p) * np.log2(1-p)
 
 
         gain_ent = (h - left_ent - right_ent)/iv
@@ -169,16 +172,17 @@ def cal_max_ent_attr_c45(Xtrain, Ytrain, weights=None):
 weights = []
 
 # 计算某个属性的信息增益率
-def cal_ent_attr_c45(Xtrain, Ytrain):
+def cal_ent_attr_c45(Xtrain, Ytrain, weights):
     # 对每个属性
     max_ent = 0
     max_i = 0
     max_mean = 0
+    weights = weights / sum(weights)
     for i in range(Xtrain.shape[1]): #每个属性
         argsort = Xtrain[:,i].argsort()
-        x,y = Xtrain[:,i][argsort], Ytrain[argsort]
+        x,y,w = Xtrain[:,i][argsort], Ytrain[argsort], weights[argsort]
 
-        gain_ent, mean = cal_max_ent_attr_c45(x, y)
+        gain_ent, mean = cal_max_ent_attr_c45(x, y, w)
 
         if gain_ent > max_ent:
             max_ent = gain_ent
@@ -255,16 +259,16 @@ def fit(Xtrain, Ytrain, parent_node, depth, weights):
 
     if is_end(Ytrain):
         # print('这个是叶子节点')
-        return TreeNode(-1, 0, 0, True, -1, len(Ytrain), distrib(Ytrain))
+        return leaf_node(Ytrain, weights)
 
     if depth >= MAX_T:
         return leaf_node(Ytrain, weights)
 
-    i, mean, min_ent = cal_ent_attr(Xtrain, Ytrain, weights)
+    i, mean, min_ent = cal_ent_attr_c45(Xtrain, Ytrain, weights)
     total_ent = calc_ent(Ytrain)
     # print("第", i, "个属性,mean:", mean)
     # 生成节点
-    parent_node = TreeNode(i, mean, total_ent - min_ent, False, -1, len(Ytrain), distrib(Ytrain))
+    parent_node = TreeNode(i, mean, total_ent - min_ent, False, -2, len(Ytrain), distrib(Ytrain))
 
     # 切分数据
     right_Ytrain = Ytrain[Xtrain[:, i] > mean]