LSTM CNN

mix/lstm_predict.py

@@ -0,0 +1,108 @@
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import load_model
+import joblib
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            if line[-2][0].startswith('0') or line[-2][0].startswith('3'):
+                lines.append(line)
+
+    size = len(lines[0])
+    train_x=[s[:size - 2] for s in lines]
+    train_y=[s[size-1] for s in lines]
+    return np.array(train_x),np.array(train_y),lines
+
+
+def _score(fact, line):
+    with open('mix_predict_dmi_18d.txt', 'a') as f:
+        f.write(str([line[-2], line[-1]]) + "\n")
+
+    up_right = 0
+    up_error = 0
+
+    if fact[0] == 1:
+        up_right = up_right + 1.12
+    elif fact[1] == 1:
+        up_right = up_right + 1.06
+    elif fact[2] == 1:
+        up_right = up_right + 1
+        up_error = up_error + 0.5
+    elif fact[3] == 1:
+        up_right = up_right + 0.94
+        up_error = up_error + 1
+    else:
+        up_error = up_error + 1
+        up_right = up_right + 0.88
+    return up_right,up_error
+
+
+def predict(file_path='', model_path='15min_dnn_seq.h5', idx=-1):
+    test_x,test_y,lines=read_data(file_path)
+    print('Load data success')
+
+    test_x_a = test_x[:,:18*16]
+    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 16)
+    test_x_b = test_x[:, 18*16:18*16+10*18]
+    test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+    test_x_c = test_x[:,18*16+10*18:]
+
+    model=load_model(model_path)
+    score = model.evaluate([test_x_c, test_x_a, test_x_b], test_y)
+    print('MIX', score)
+
+    up_num = 0
+    up_error = 0
+    up_right = 0
+    down_num = 0
+    down_error = 0
+    down_right = 0
+    i = 0
+    result=model.predict([test_x_c, test_x_a, test_x_b])
+    win_dnn = []
+    for r in result:
+        fact = test_y[i]
+
+        if idx in [-2]:
+            if r[0] > 0.5 or r[1] > 0.5:
+                pass
+        else:
+            if r[0] > 0.6 or r[1] > 0.6:
+                tmp_right,tmp_error = _score(fact, lines[i])
+                up_right = tmp_right + up_right
+                up_error = tmp_error + up_error
+                up_num = up_num + 1
+            elif r[3] > 0.7 or r[4] > 0.7:
+                if fact[0] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.12
+                elif fact[1] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.06
+                elif fact[2] == 1:
+                    down_error = down_error + 0.5
+                    down_right = down_right + 1
+                elif fact[3] == 1:
+                    down_right = down_right + 0.94
+                else:
+                    down_right = down_right + 0.88
+                down_num = down_num + 1
+
+        i = i + 1
+    if up_num == 0:
+        up_num = 1
+    if down_num == 0:
+        down_num = 1
+    print('MIX', up_right, up_num, up_right/up_num, up_error/up_num, down_right/down_num, down_error/down_num)
+    return win_dnn,up_right/up_num,down_right/down_num
+
+
+if __name__ == '__main__':
+    predict(file_path='D:\\data\\quantization\\stock19_18d_test.log', model_path='19_18d_lstm_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
+    # multi_predict(model='15_18d')
+    # predict_today(20200229, model='11_18d')

mix/lstm_train.py

@@ -0,0 +1,205 @@
+import keras
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import Sequential
+# 优化方法选用Adam(其实可选项有很多，如SGD)
+from keras.optimizers import Adam
+import random
+from keras.models import load_model
+from imblearn.over_sampling import RandomOverSampler
+from keras.utils import np_utils
+# 用于模型初始化，Conv2D模型初始化、Activation激活函数，MaxPooling2D是池化层
+# Flatten作用是将多位输入进行一维化
+# Dense是全连接层
+from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense,Dropout,Input,MaxPooling2D,BatchNormalization,concatenate
+from keras.layers import LSTM
+from keras import regularizers
+from keras.models import Model
+
+from keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(monitor='accuracy', patience=5, verbose=2)
+
+epochs= 40
+size = 80000
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        for x in range(size): #380000
+            lines.append(eval(f.readline().strip()))
+
+    random.shuffle(lines)
+    print('读取数据完毕')
+
+    d=int(0.7*len(lines))
+
+    train_x=[s[:-2] for s in lines[0:d]]
+    train_y=[s[-1] for s in lines[0:d]]
+    test_x=[s[:-2] for s in lines[d:]]
+    test_y=[s[-1] for s in lines[d:]]
+
+    print('转换数据完毕')
+
+    ros = RandomOverSampler(random_state=0)
+    X_resampled, y_resampled = ros.fit_sample(np.array(train_x), np.array(train_y))
+
+    print('数据重采样完毕')
+
+    return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
+
+
+train_x,train_y,test_x,test_y=read_data("D:\\data\\quantization\\stock19_18d_train_1.log")
+
+train_x_a = train_x[:,:18*16]
+train_x_a = train_x_a.reshape(train_x.shape[0], 18, 16)
+train_x_b = train_x[:, 18*16:18*16+10*18]
+train_x_b = train_x_b.reshape(train_x.shape[0], 18, 10, 1)
+train_x_c = train_x[:,18*16+10*18:]
+
+
+def create_mlp(dim, regress=False):
+    # define our MLP network
+    model = Sequential()
+    model.add(Dense(16, input_dim=dim, activation="relu"))
+    model.add(Dense(16, activation="relu"))
+
+    # check to see if the regression node should be added
+    if regress:
+        model.add(Dense(1, activation="linear"))
+
+    # return our model
+    return model
+
+
+def create_cnn(width, height, depth, filters=32, kernel_size=(5, 6), regress=False, output=24):
+    # initialize the input shape and channel dimension, assuming
+    # TensorFlow/channels-last ordering
+    inputShape = (width, height, 1)
+    chanDim = -1
+
+    # define the model input
+    inputs = Input(shape=inputShape)
+
+    x = inputs
+
+    # CONV => RELU => BN => POOL
+    x = Conv2D(filters, kernel_size, strides=(2,2), padding="same",
+               # data_format='channels_first'
+               )(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    # x = MaxPooling2D(pool_size=(2, 2))(x)
+    # if width > 2:
+    #     x = Conv2D(32, (10, 6), padding="same")(x)
+    #     x = Activation("relu")(x)
+    #     x = BatchNormalization(axis=chanDim)(x)
+
+    # flatten the volume, then FC => RELU => BN => DROPOUT
+    x = Flatten()(x)
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    x = Dropout(0.2)(x)
+
+    # apply another FC layer, this one to match the number of nodes
+    # coming out of the MLP
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+
+    # check to see if the regression node should be added
+    if regress:
+        x = Dense(1, activation="linear")(x)
+
+    # construct the CNN
+    model = Model(inputs, x)
+
+    # return the CNN
+    return model
+
+
+def create_lstm(sample, timesteps, input_dim):
+    inputShape = (timesteps, input_dim)
+
+    # define the model input
+    inputs = Input(shape=inputShape)
+
+    x = inputs
+
+    x = LSTM(units = 32, input_shape=(18, 16), dropout=0.2
+               )(x)
+    # x = LSTM(16*16, return_sequences=False)
+    # x = Activation("relu")(x)
+    x = Dense(64)(x)
+    x = Dropout(0.2)(x)
+    x = Activation("relu")(x)
+
+    # construct the CNN
+    model = Model(inputs, x)
+
+    # return the CNN
+    return model
+
+# create the MLP and CNN models
+mlp = create_mlp(train_x_c.shape[1], regress=False)
+cnn_0 = create_lstm(train_x_a.shape[1], 18, 16)
+cnn_1 = create_cnn(18, 10, 1, kernel_size=(3, 5), filters=32, regress=False, output=120)
+
+# create the input to our final set of layers as the *output* of both
+# the MLP and CNN
+combinedInput = concatenate([mlp.output, cnn_0.output, cnn_1.output])
+
+# our final FC layer head will have two dense layers, the final one
+# being our regression head
+x = Dense(888, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
+x = Dropout(0.2)(x)
+x = Dense(888, activation="relu")(x)
+x = Dense(888, activation="relu")(x)
+# 在建设一层
+x = Dense(5, activation="softmax")(x)
+
+# our final model will accept categorical/numerical data on the MLP
+# input and images on the CNN input, outputting a single value (the
+# predicted price of the house)
+model = Model(inputs=[mlp.input, cnn_0.input, cnn_1.input], outputs=x)
+
+
+print("Starting training ")
+# h = model.fit(train_x, train_y, batch_size=4096*2, epochs=500, shuffle=True)
+
+# compile the model using mean absolute percentage error as our loss,
+# implying that we seek to minimize the absolute percentage difference
+# between our price *predictions* and the *actual prices*
+opt = Adam(lr=1e-3, decay=1e-3 / 200)
+model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'],
+              )
+
+# train the model
+print("[INFO] training model...")
+model.fit(
+    [train_x_c, train_x_a, train_x_b], train_y,
+    # validation_data=([testAttrX, testImagesX], testY),
+    # epochs=int(3*train_x_a.shape[0]/1300),
+    epochs=epochs,
+    batch_size=2048, shuffle=True,
+    callbacks=[early_stopping]
+)
+
+test_x_a = test_x[:,:18*16]
+test_x_a = test_x_a.reshape(test_x.shape[0], 18, 16)
+test_x_b = test_x[:, 18*16:18*16+10*18]
+test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+test_x_c = test_x[:,18*16+10*18:]
+
+# make predictions on the testing data
+print("[INFO] predicting house prices...")
+score  = model.evaluate([test_x_c, test_x_a, test_x_b], test_y)
+
+print(score)
+print('Test score:', score[0])
+print('Test accuracy:', score[1])
+
+path="19_18d_lstm_seq.h5"
+model.save(path)
+model=None

mix/mix_predict.py

@@ -45,11 +45,11 @@ def predict(file_path='', model_path='15min_dnn_seq.h5', idx=-1):
     test_x,test_y,lines=read_data(file_path)
     print('Load data success')
 
-    test_x_a = test_x[:,:18*18]
-    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 18, 1)
-    test_x_b = test_x[:, 18*18:18*18+8*18]
-    test_x_b = test_x_b.reshape(test_x.shape[0], 18, 8, 1)
-    test_x_c = test_x[:,18*18+8*18:]
+    test_x_a = test_x[:,:18*16]
+    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 16, 1)
+    test_x_b = test_x[:, 18*16:18*16+10*18]
+    test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+    test_x_c = test_x[:,18*16+10*18:]
 
     model=load_model(model_path)
     score = model.evaluate([test_x_c, test_x_a, test_x_b], test_y)

mix/mix_train.py

@@ -15,9 +15,13 @@ from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense,Dropout,I
 from keras import regularizers
 from keras.models import Model
 
+from keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(monitor='accuracy', patience=5, verbose=2)
+
+epochs= 50
+size = 80000
 
-epochs= 130
-size = 380000
 
 def read_data(path):
     lines = []
@@ -45,13 +49,13 @@ def read_data(path):
     return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
 
 
-train_x,train_y,test_x,test_y=read_data("D:\\data\\quantization\\stock18_18d_train_1.log")
+train_x,train_y,test_x,test_y=read_data("D:\\data\\quantization\\stock19_18d_train_1.log")
 
-train_x_a = train_x[:,:18*18]
-train_x_a = train_x_a.reshape(train_x.shape[0], 18, 18, 1)
-train_x_b = train_x[:, 18*18:18*18+8*18]
-train_x_b = train_x_b.reshape(train_x.shape[0], 18, 8, 1)
-train_x_c = train_x[:,18*18+8*18:]
+train_x_a = train_x[:,:18*16]
+train_x_a = train_x_a.reshape(train_x.shape[0], 18, 16, 1)
+train_x_b = train_x[:, 18*16:18*16+10*18]
+train_x_b = train_x_b.reshape(train_x.shape[0], 18, 10, 1)
+train_x_c = train_x[:,18*16+10*18:]
 
 
 def create_mlp(dim, regress=False):
@@ -80,7 +84,9 @@ def create_cnn(width, height, depth, filters=32, kernel_size=(5, 6), regress=Fal
     x = inputs
 
     # CONV => RELU => BN => POOL
-    x = Conv2D(filters, kernel_size, strides=2, padding="same")(x)
+    x = Conv2D(filters, kernel_size, strides=2, padding="same",
+               # data_format='channels_first'
+               )(x)
     x = Activation("relu")(x)
     x = BatchNormalization(axis=chanDim)(x)
     # x = MaxPooling2D(pool_size=(2, 2))(x)
@@ -114,8 +120,8 @@ def create_cnn(width, height, depth, filters=32, kernel_size=(5, 6), regress=Fal
 
 # create the MLP and CNN models
 mlp = create_mlp(train_x_c.shape[1], regress=False)
-cnn_0 = create_cnn(18, 18, 1, kernel_size=(5, 6), filters=48, regress=False, output=256)
-cnn_1 = create_cnn(18, 8, 1, kernel_size=(5, 6), filters=32, regress=False, output=128)
+cnn_0 = create_cnn(18, 16, 1, kernel_size=(3, 5), filters=32, regress=False, output=150)
+cnn_1 = create_cnn(18, 10, 1, kernel_size=(3, 4), filters=32, regress=False, output=120)
 
 # create the input to our final set of layers as the *output* of both
 # the MLP and CNN
@@ -123,10 +129,9 @@ combinedInput = concatenate([mlp.output, cnn_0.output, cnn_1.output])
 
 # our final FC layer head will have two dense layers, the final one
 # being our regression head
-x = Dense(888, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
+x = Dense(1024, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
 x = Dropout(0.2)(x)
-x = Dense(888, activation="relu")(x)
-x = Dense(888, activation="relu")(x)
+x = Dense(1024, activation="relu")(x)
 # 在建设一层
 x = Dense(5, activation="softmax")(x)
 
@@ -143,7 +148,8 @@ print("Starting training ")
 # implying that we seek to minimize the absolute percentage difference
 # between our price *predictions* and the *actual prices*
 opt = Adam(lr=1e-3, decay=1e-3 / 200)
-model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'])
+model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'],
+              )
 
 # train the model
 print("[INFO] training model...")
@@ -152,13 +158,15 @@ model.fit(
     # validation_data=([testAttrX, testImagesX], testY),
     # epochs=int(3*train_x_a.shape[0]/1300),
     epochs=epochs,
-    batch_size=1024, shuffle=True)
-
-test_x_a = test_x[:,:18*18]
-test_x_a = test_x_a.reshape(test_x.shape[0], 18, 18, 1)
-test_x_b = test_x[:, 18*18:18*18+8*18]
-test_x_b = test_x_b.reshape(test_x.shape[0], 18, 8, 1)
-test_x_c = test_x[:,18*18+8*18:]
+    batch_size=2048, shuffle=True,
+    callbacks=[early_stopping]
+)
+
+test_x_a = test_x[:,:18*16]
+test_x_a = test_x_a.reshape(test_x.shape[0], 18, 16, 1)
+test_x_b = test_x[:, 18*16:18*16+10*18]
+test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+test_x_c = test_x[:,18*16+10*18:]
 
 # make predictions on the testing data
 print("[INFO] predicting house prices...")

stock/dnn_predict_by_day.py

@@ -35,13 +35,13 @@ def predict(file_path='', model_path='15min_dnn_seq'):
 
         up_num = 0
         down_num = 0
-        x = 24 # 每条数据项数
+        x = 16 # 每条数据项数
         k = 18 # 周期
         for line in lines:
-            v = line[1:x*k + 1]
+            v = line[0:x*k]
             v = np.array(v)
             v = v.reshape(k, x)
-            v = v[:,4:8]
+            v = v[:,6:10]
             v = v.reshape(1, 4*k)
             # print(v)
             r = estimator.predict(v)
@@ -51,7 +51,7 @@ def predict(file_path='', model_path='15min_dnn_seq'):
 
             if result[0][3] > 0.5 or result[0][4] > 0.5:
                 down_num = down_num + 1
-            elif result[0][1] > 0.5 or result[0][2] > 0.5:
+            elif result[0][1] > 0.5 or result[0][0] > 0.5:
                 up_num = up_num + 0.5  # 乐观调大 悲观调小
 
             # if result[0][0] > 0.5 or result[0][1] > 0.5:
@@ -76,6 +76,6 @@ if __name__ == '__main__':
     # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock9_18_20200220.log', model_path='18d_dnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock9_18_2.log', model_path='18d_dnn_seq.h5')
-    predict(file_path='D:\\data\\quantization\\stock11_18d_20200303.log', model_path='11_18d_dnn_seq')
-    # predict(file_path='D:\\data\\quantization\\stock12_18d_20200226.log', model_path='11_18d_dnn_seq')
+    predict(file_path='D:\\data\\quantization\\stock19_18d_20200309.log', model_path='19_18d_dnn_seq')
+    # predict(file_path='D:\\data\\quantization\\stock11_18d_20190103_20190604.log', model_path='14_18d_dnn_seq')
     # predict(file_path='D:\\data\\quantization\\stock9_18_4.log', model_path='18d_dnn_seq.h5')

stock/dnn_predict_by_stock.py

@@ -105,7 +105,7 @@ def predict(file_path='', model_path='15min_dnn_seq'):
                         if today_price['close'] > high_price:
                             high_price = today_price['close']
 
-        else:
+            else:
                 if buy == 1:
                     init_money = (init_money * (today_price['close'] - last_price)/last_price) + init_money
                     if init_money < 8500:

stock/dnn_predict_dmi.py

@@ -30,7 +30,9 @@ def _score(fact, line):
         up_right = up_right + 1.06
     elif fact[2] == 1:
         up_right = up_right + 1
+        up_error = up_error + 0.5
     elif fact[3] == 1:
+        up_error = up_error + 1
         up_right = up_right + 0.94
     else:
         up_error = up_error + 1
@@ -83,6 +85,7 @@ def predict(file_path='', model_path='15min_dnn_seq.h5', idx=-1):
                     down_error = down_error + 1
                     down_right = down_right + 1.12
                 elif fact[1] == 1:
+                    down_error = down_error + 1
                     down_right = down_right + 1.06
                 elif fact[2] == 1:
                     down_right = down_right + 1
@@ -101,7 +104,7 @@ def predict(file_path='', model_path='15min_dnn_seq.h5', idx=-1):
     return win_dnn,up_right/up_num,down_right/down_num
 
 
-def multi_predict():
+def multi_predict(model='14_18d'):
     r = 0;
     p = 0
     for x in range(0, 12): # 0,2,3,4,6,8,9,10,11
@@ -110,8 +113,8 @@ def multi_predict():
     # for x in [2,4,7,10]: # 2表现最好 优秀的 0,8正确的反向指标,(9错误的反向指标 样本量太少)
         print(x)
     # for x in [0,2,5,6,7]: # 5表现最好
-        win_dnn, up_ratio,down_ratio = predict(file_path='D:\\data\\quantization\\kmeans\\stock13_18d_test_' + str(x) + '.log',
-                                               model_path='13_18d_dnn_seq_' + str(x) + '.h5', idx=x)
+        win_dnn, up_ratio,down_ratio = predict(file_path='D:\\data\\quantization\\kmeans\\stock' + model + '_test_' + str(x) + '.log',
+                                               model_path=model + '_dnn_seq_' + str(x) + '.h5', idx=x)
         r = r + up_ratio
         p = p + down_ratio
     print(r, p)
@@ -198,7 +201,7 @@ def predict_today(day, model='10_18d'):
 
 
 if __name__ == '__main__':
-    # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock16_18d_test.log', model_path='16_18d_cnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
-    multi_predict()
+    multi_predict(model='19_18d')
     # predict_today(20200229, model='11_18d')