kmeans before train

stock/dnn_predict.py

@@ -56,10 +56,21 @@ def predict(file_path='', model_path='15min_dnn_seq.h5'):
                 up_num = up_num + 1
 
             i = i + 1
+    if up_num == 0:
+        up_num = 1
     print('DNN', up_right, up_num, up_right/up_num)
-    return win_dnn
+    return win_dnn,up_right/up_num
+
+
+def multi_predict():
+    r = 0
+    for x in [7,9,12,14,15]:
+        win_dnn, ratio = predict(file_path='D:\\data\\quantization\\kmeans\\stock2_10_' + str(x) + '_test.log', model_path='5d_dnn_seq_' + str(x) + '.h5')
+        r = r + ratio
+    print(r)
 
 
 if __name__ == '__main__':
-    predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
-    # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
+    multi_predict()

stock/dnn_train.py

@@ -7,13 +7,16 @@ import random
 from keras import regularizers
 from keras.models import load_model
 from imblearn.over_sampling import RandomOverSampler
+import joblib
 
 
 def read_data(path):
     lines = []
     with open(path) as f:
-        for x in range(30000):
-            lines.append(eval(f.readline().strip()))
+        # for x in range(30000):
+        # lines.append(eval(f.readline().strip()))
+        for line in f.readlines()[:]:
+            lines.append(eval(line.strip()))
 
     random.shuffle(lines)
     print('读取数据完毕')
@@ -36,25 +39,53 @@ def read_data(path):
     return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
 
 
+def resample(path):
+    lines = []
+    with open(path) as f:
+        for x in range(160000):
+            lines.append(eval(f.readline().strip()))
+    estimator = joblib.load('km.pkl')
+
+    x = 17
+    for line in lines:
+        v = line[1:x*10 + 1]
+        v = np.array(v)
+        v = v.reshape(10, x)
+        v = v[:,0:4]
+        v = v.reshape(1, 40)
+        # print(v)
+        r = estimator.predict(v)
+        with open('D:\\data\\quantization\\kmeans\\stock2_10_' + str(r[0]) + '.log', 'a') as f:
+            f.write(str(line) + '\n')
+
+
+def mul_train():
+    # for x in range(0, 16):
+    for x in [7,9,12,14,15]:
+        train(input_dim=176, result_class=5, file_path="D:\\data\\quantization\\kmeans\\stock2_10_" + str(x) + ".log",
+              model_name='5d_dnn_seq_' + str(x) + '.h5')
+
+
 def train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stock6.log", model_name=''):
     train_x,train_y,test_x,test_y=read_data(file_path)
 
     model = Sequential()
     model.add(Dense(units=120+input_dim, input_dim=input_dim,  activation='relu'))
-    model.add(Dense(units=120+input_dim, activation='relu',kernel_regularizer=regularizers.l2(0.001)))
+    model.add(Dense(units=120+input_dim, activation='relu',kernel_regularizer=regularizers.l1(0.001)))
     model.add(Dense(units=120+input_dim, activation='relu'))
-    model.add(Dropout(0.2))
-    model.add(Dense(units=60+input_dim, activation='selu'))
-    model.add(Dropout(0.2))
+    model.add(Dense(units=120 + input_dim, activation='relu'))
+    model.add(Dropout(0.1))
     model.add(Dense(units=60+input_dim, activation='selu'))
     # model.add(Dropout(0.2))
+    # model.add(Dense(units=60+input_dim, activation='selu'))
+    # model.add(Dropout(0.2))
     model.add(Dense(units=512, activation='relu'))
 
     model.add(Dense(units=result_class, activation='softmax'))
     model.compile(loss='categorical_crossentropy', optimizer="adam",metrics=['accuracy'])
 
     print("Starting training ")
-    model.fit(train_x, train_y, batch_size=32, epochs=56, shuffle=True)
+    model.fit(train_x, train_y, batch_size=32, epochs=117 + int(len(train_x)/900), shuffle=True)
     score = model.evaluate(test_x, test_y)
     print(score)
     print('Test score:', score[0])
@@ -62,13 +93,15 @@ def train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stoc
 
     model.save(model_name)
 
-    model=None
-    model=load_model(model_name)
-    result=model.predict(test_x)
-    print(result)
-    print(test_y)
+    # model=None
+    # model=load_model(model_name)
+    # result=model.predict(test_x)
+    # print(result)
+    # print(test_y)
 
 
 if __name__ == '__main__':
-    train(input_dim=86, result_class=5, file_path="D:\\data\\quantization\\stock6_5.log", model_name='5d_dnn_seq.h5')
-    # train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stock6.log", model_name='15m_dnn_seq.h5')
+    # train(input_dim=86, result_class=5, file_path="D:\\data\\quantization\\stock6_5.log", model_name='5d_dnn_seq.h5')
+    # train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stock6.log", model_name='15m_dnn_seq.h5')
+    # resample('D:\\data\\quantization\\stock6_5.log')
+    mul_train()

stock/kmeans.py

@@ -2,20 +2,20 @@
 from sklearn.cluster import KMeans
 import numpy as np
 from annoy import AnnoyIndex
-
+import joblib
 
 def read_data(path):
     lines = []
     with open(path) as f:
-        for x in range(160000):
+        for x in range(100000):
             line = eval(f.readline().strip())
             # if line[-1][0] == 1 or line[-1][1] == 1:
             lines.append(line)
 
     return lines
 
-length = 20
-def class_fic(file_path=''):
+length = 10
+def  class_fic(file_path=''):
     lines = read_data(file_path)
     print('读取数据完毕')
     size = len(lines[0])
@@ -24,9 +24,19 @@ def class_fic(file_path=''):
     v_x = train_x.reshape(train_x.shape[0], 4*length)
     stock_list = [s[size - 2] for s in lines]
 
+    estimator = KMeans(n_clusters=16, random_state=9)
+    estimator.fit(v_x)
+    label_pred = estimator.labels_  # 获取聚类标签
+    centroids = estimator.cluster_centers_
+    joblib.dump(estimator , 'km.pkl')
+
+    print(estimator.predict(v_x[:10]))
+
+    estimator = joblib.load('km.pkl')
+    print(estimator.predict(v_x[10:20]))
     # annoy_sim(v_x)
-    print('save数据完毕')
-    return find_annoy(train_y, stock_list)
+    # print('save数据完毕')
+    # return find_annoy(train_y, stock_list)
 
 def annoy_sim(lines):
     tree = 30
@@ -93,4 +103,4 @@ def find_annoy(lines, stock_list):
 
 
 if __name__ == '__main__':
-    class_fic(file_path="D:\\data\\quantization\\stock2_20.log")
+    class_fic(file_path="D:\\data\\quantization\\stock2_10.log")