比较lstm kmeans后分类的情况

mix/lstm_kmeans_predict.py

@@ -0,0 +1,125 @@
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import load_model
+import joblib
+
+model_path = '160_18d_lstm_5D_ma5_s_seq.h5'
+data_dir = 'D:\\data\\quantization\\'
+kmeans = 'roc'
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            if line[-2][0].startswith('0') or line[-2][0].startswith('3'):
+                lines.append(line)
+
+    size = len(lines[0])
+    train_x=[s[:size - 2] for s in lines]
+    train_y=[s[size-1] for s in lines]
+    return np.array(train_x),np.array(train_y),lines
+
+
+def _score(fact, line):
+    with open('mix_predict_dmi_18d.txt', 'a') as f:
+        f.write(str([line[-2], line[-1]]) + "\n")
+
+    up_right = 0
+    up_error = 0
+
+    if fact[0] == 1:
+        up_right = up_right + 1.12
+    elif fact[1] == 1:
+        up_right = up_right + 1.06
+    elif fact[2] == 1:
+        up_right = up_right + 1
+        up_error = up_error + 0.5
+    elif fact[3] == 1:
+        up_right = up_right + 0.94
+        up_error = up_error + 1
+    else:
+        up_error = up_error + 1
+        up_right = up_right + 0.88
+    return up_right,up_error
+
+
+def mul_predict(name="10_18d"):
+    r = 0
+    p = 0
+
+    for x in range(0, 8):
+        win_dnn, up_ratio,down_ratio  = predict(data_dir + kmeans + '\\stock160_18d_train1_B_' + str(x) + ".log", x) # stock160_18d_trai_0
+
+        r = r + up_ratio
+        p = p + down_ratio
+
+    print(r, p)
+
+
+def predict(file_path='', idx=-1):
+    test_x,test_y,lines=read_data(file_path)
+    print(idx, 'Load data success')
+
+    test_x_a = test_x[:,:18*24]
+    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 24)
+    # test_x_b = test_x[:, 18*16:18*16+10*18]
+    # test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+    test_x_c = test_x[:,18*24:]
+
+    model=load_model(model_path.split('.')[0] + '_' + str(idx) + '.h5')
+    score = model.evaluate([test_x_c, test_x_a, ], test_y)
+    print('LSTM', score)
+
+    up_num = 0
+    up_error = 0
+    up_right = 0
+    down_num = 0
+    down_error = 0
+    down_right = 0
+    i = 0
+    result=model.predict([test_x_c, test_x_a, ])
+    win_dnn = []
+    for r in result:
+        fact = test_y[i]
+
+        if idx in [-2]:
+            if r[0] > 0.5 or r[1] > 0.5:
+                pass
+        else:
+            if r[0] > 0.6 or r[1] > 0.6:
+                tmp_right,tmp_error = _score(fact, lines[i])
+                up_right = tmp_right + up_right
+                up_error = tmp_error + up_error
+                up_num = up_num + 1
+            elif r[3] > 0.7 or r[4] > 0.7:
+                if fact[0] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.12
+                elif fact[1] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.06
+                elif fact[2] == 1:
+                    down_error = down_error + 0.5
+                    down_right = down_right + 1
+                elif fact[3] == 1:
+                    down_right = down_right + 0.94
+                else:
+                    down_right = down_right + 0.88
+                down_num = down_num + 1
+
+        i = i + 1
+    if up_num == 0:
+        up_num = 1
+    if down_num == 0:
+        down_num = 1
+    print('LSTM', up_right, up_num, up_right/up_num, up_error/up_num, down_right/down_num, down_error/down_num)
+    return win_dnn,up_right/up_num,down_right/down_num
+
+
+if __name__ == '__main__':
+    # predict(file_path='D:\\data\\quantization\\stock160_18d_10D_test.log', model_path='160_18d_lstm_5D_ma5_s_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
+    mul_predict(name='stock160_18d')
+    # predict_today(20200229, model='11_18d')

mix/lstm_kmeans_train.py

@@ -0,0 +1,221 @@
+import keras
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import Sequential
+# 优化方法选用Adam(其实可选项有很多，如SGD)
+from keras.optimizers import Adam
+import random
+from keras.models import load_model
+from imblearn.over_sampling import RandomOverSampler
+from keras.utils import np_utils
+# 用于模型初始化，Conv2D模型初始化、Activation激活函数，MaxPooling2D是池化层
+# Flatten作用是将多位输入进行一维化
+# Dense是全连接层
+from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense,Dropout,Input,MaxPooling2D,BatchNormalization,concatenate
+from keras.layers import LSTM
+from keras import regularizers
+from keras.models import Model
+
+from keras.callbacks import EarlyStopping
+
+early_stopping = EarlyStopping(monitor='accuracy', patience=5, verbose=2)
+
+epochs= 330
+size = 580000  # 61W
+file_path = 'D:\\data\\quantization\\stock160_18d_train.log'
+model_path = '160_18d_lstm_5D_ma5_s_seq.h5'
+data_dir = 'D:\\data\\quantization\\'
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        for line in f.readlines()[:]:
+            lines.append(eval(line.strip()))
+
+    random.shuffle(lines)
+    print('读取数据完毕')
+
+    d=int(0.7*len(lines))
+
+    train_x=[s[:-2] for s in lines[0:d]]
+    train_y=[s[-1] for s in lines[0:d]]
+    test_x=[s[:-2] for s in lines[d:]]
+    test_y=[s[-1] for s in lines[d:]]
+
+    print('转换数据完毕')
+
+    ros = RandomOverSampler(random_state=0)
+    X_resampled, y_resampled = ros.fit_sample(np.array(train_x), np.array(train_y))
+
+    print('数据重采样完毕')
+
+    return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
+
+
+def mul_train(name="10_18d"):
+    for x in range(0, 8):
+        score = train(data_dir + 'kmeans\\' + name + "_trai_" + str(x) + ".log", x) # stock160_18d_trai_0
+
+        with open(data_dir + name + '_lstm.log', 'a') as f:
+            f.write(str(x) + ':' + str(score[1]) + '\n')
+
+
+def train(file_path, idx):
+    train_x,train_y,test_x,test_y=read_data(file_path)
+
+    train_x_a = train_x[:,:18*24]
+    train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24)
+    # train_x_b = train_x[:, 18*24:18*16+10*18]
+    # train_x_b = train_x_b.reshape(train_x.shape[0], 18, 10, 1)
+    train_x_c = train_x[:,18*24:]
+
+    # create the MLP and CNN models
+    mlp = create_mlp(train_x_c.shape[1], regress=False)
+    cnn_0 = create_lstm(train_x_a.shape[1], 18, 24)
+    # cnn_1 = create_cnn(18, 10, 1, kernel_size=(3, 5), filters=32, regress=False, output=120)
+
+    # create the input to our final set of layers as the *output* of both
+    # the MLP and CNN
+    combinedInput = concatenate([mlp.output, cnn_0.output,])
+
+    # our final FC layer head will have two dense layers, the final one
+    # being our regression head
+    x = Dense(256, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
+    x = Dropout(0.2)(x)
+    x = Dense(256, activation="relu")(x)
+    x = Dense(512, activation="relu")(x)
+    # 在建设一层
+    x = Dense(5, activation="softmax")(x)
+
+    # our final model will accept categorical/numerical data on the MLP
+    # input and images on the CNN input, outputting a single value (the
+    # predicted price of the house)
+    model = Model(inputs=[mlp.input, cnn_0.input,], outputs=x)
+
+
+    print("Starting training ")
+    # h = model.fit(train_x, train_y, batch_size=4096*2, epochs=500, shuffle=True)
+
+    # compile the model using mean absolute percentage error as our loss,
+    # implying that we seek to minimize the absolute percentage difference
+    # between our price *predictions* and the *actual prices*
+    opt = Adam(lr=1e-3, decay=1e-3 / 200)
+    model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'],
+                  )
+
+    # train the model
+    print("[INFO] training model...")
+    model.fit(
+        [train_x_c, train_x_a, ], train_y,
+        # validation_data=([testAttrX, testImagesX], testY),
+        # epochs=int(3*train_x_a.shape[0]/1300),
+        epochs=epochs,
+        batch_size=4096, shuffle=True,
+        callbacks=[early_stopping]
+    )
+
+    test_x_a = test_x[:,:18*24]
+    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 24)
+    # test_x_b = test_x[:, 18*16:18*16+10*18]
+    # test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+    test_x_c = test_x[:,18*24:]
+
+    # make predictions on the testing data
+    print("[INFO] predicting house prices...")
+    score  = model.evaluate([test_x_c, test_x_a], test_y)
+
+    print(score)
+    print('Test score:', score[0])
+    print('Test accuracy:', score[1])
+
+    model.save(model_path.split('.')[0] + '_' + str(idx) + '.h5')
+
+    return score
+
+
+def create_mlp(dim, regress=False):
+    # define our MLP network
+    model = Sequential()
+    model.add(Dense(64, input_dim=dim, activation="relu"))
+    model.add(Dense(64, activation="relu"))
+
+    # check to see if the regression node should be added
+    if regress:
+        model.add(Dense(1, activation="linear"))
+
+    # return our model
+    return model
+
+
+def create_cnn(width, height, depth, filters=32, kernel_size=(5, 6), regress=False, output=24):
+    # initialize the input shape and channel dimension, assuming
+    # TensorFlow/channels-last ordering
+    inputShape = (width, height, 1)
+    chanDim = -1
+
+    # define the model input
+    inputs = Input(shape=inputShape)
+
+    x = inputs
+
+    # CONV => RELU => BN => POOL
+    x = Conv2D(filters, kernel_size, strides=(2,2), padding="same",
+               # data_format='channels_first'
+               )(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    # x = MaxPooling2D(pool_size=(2, 2))(x)
+    # if width > 2:
+    #     x = Conv2D(32, (10, 6), padding="same")(x)
+    #     x = Activation("relu")(x)
+    #     x = BatchNormalization(axis=chanDim)(x)
+
+    # flatten the volume, then FC => RELU => BN => DROPOUT
+    x = Flatten()(x)
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    x = Dropout(0.2)(x)
+
+    # apply another FC layer, this one to match the number of nodes
+    # coming out of the MLP
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+
+    # check to see if the regression node should be added
+    if regress:
+        x = Dense(1, activation="linear")(x)
+
+    # construct the CNN
+    model = Model(inputs, x)
+
+    # return the CNN
+    return model
+
+
+def create_lstm(sample, timesteps, input_dim):
+    inputShape = (timesteps, input_dim)
+
+    # define the model input
+    inputs = Input(shape=inputShape)
+
+    x = inputs
+
+    x = LSTM(units = 64, input_shape=(timesteps, input_dim), dropout=0.2
+               )(x)
+    # x = LSTM(16*16, return_sequences=False)
+    # x = Activation("relu")(x)
+    x = Dense(64)(x)
+    x = Dropout(0.2)(x)
+    x = Activation("relu")(x)
+
+    # construct the CNN
+    model = Model(inputs, x)
+
+    # return the CNN
+    return model
+
+
+if __name__ == '__main__':
+    mul_train('stock160_18d')

mix/lstm_predict.py

@@ -102,7 +102,7 @@ def predict(file_path='', model_path='15min_dnn_seq.h5', idx=-1):
 
 
 if __name__ == '__main__':
-    predict(file_path='D:\\data\\quantization\\stock17_18d_test.log', model_path='17_18d_lstm_seq.h5')
+    predict(file_path='D:\\data\\quantization\\stock160_18d_train1.log', model_path='160_18d_lstm_5D_ma5_s_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
     # multi_predict(model='15_18d')
     # predict_today(20200229, model='11_18d')

mix/lstm_predict_by_day.py

@@ -0,0 +1,81 @@
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import load_model
+import joblib
+
+
+def read_data(path):
+    day_lines = {}
+    with open(path) as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            date = str(line[-1][-1])
+            if date in day_lines:
+                day_lines[date].append(line)
+            else:
+                day_lines[date] = [line]
+    # print(len(day_lines['20191230']))
+    return day_lines
+
+
+def predict(file_path='', model_path='15min_dnn_seq'):
+    day_lines = read_data(file_path)
+    print('数据读取完毕')
+
+    model=load_model(model_path)
+    print('模型加载完毕')
+
+    items = sorted(day_lines.keys())
+    for key in items:
+        # print(day)
+        lines = day_lines[key]
+
+        up_num = 0
+        down_num = 0
+        size = len(lines[0])
+        x0 = 0
+        x1 = 0
+        x2 = 0
+        x3 = 0
+        x4 = 0
+
+        for line in lines:
+            train_x = np.array([line[:size - 1]])
+            train_x_a = train_x[:,:18*24]
+            train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24)
+            # train_x_b = train_x[:, 18*18:18*18+2*18]
+            # train_x_b = train_x_b.reshape(train_x.shape[0], 18, 2, 1)
+            train_x_c = train_x[:,18*24:]
+
+            result = model.predict([train_x_c, train_x_a])
+
+            if result[0][3] + result[0][4] > 0.5:
+                down_num = down_num + 1
+            elif result[0][1] + result[0][0] > 0.5:
+                up_num = up_num + 0.6
+            # else:
+            #     up_num = up_num + 0.4 # 乐观调大 悲观调小
+            #     down_num = down_num + 0.6
+
+            # if result[0][0] > 0.5:
+            #     x0 = x0 + 1
+            # if result[0][1] > 0.5:
+            #     x1 = x1 + 1
+            # if result[0][2] > 0.5:
+            #     x2 = x3 + 1
+            # if result[0][3] > 0.5:
+            #     x3 = x3 + 1
+            # if result[0][4] > 0.5:
+            #     x4 = x4 + 1
+
+        print(key, int(up_num), int(down_num), (down_num*1.2 + 2)/(up_num*1.2 + 2), )
+        # print(key, x0, x1, x2,x3,x4)
+
+
+if __name__ == '__main__':
+    # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock9_18_20200220.log', model_path='18d_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock9_18_2.log', model_path='18d_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock16_18d_20200310.log', model_path='16_18d_mix_seq')
+    predict(file_path='D:\\data\\quantization\\stock16_18d_20191225_20200310.log', model_path='16_18d_lstm_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock9_18_4.log', model_path='18d_dnn_seq.h5')

mix/lstm_predict_everyday.py

@@ -0,0 +1,207 @@
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import load_model
+import joblib
+
+
+holder_stock_list = [
+                        '000063.SZ', '002093.SZ'
+                        '300253.SZ', '300807.SZ',
+
+                         # b账户
+
+    ]
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            if line[-2][0].startswith('0') or line[-2][0].startswith('3'):
+                lines.append(line)
+
+    size = len(lines[0])
+    train_x=[s[:size - 2] for s in lines]
+    train_y=[s[size-1] for s in lines]
+    return np.array(train_x),np.array(train_y),lines
+
+
+import pymongo
+from util.mongodb import get_mongo_table_instance
+code_table = get_mongo_table_instance('tushare_code')
+k_table = get_mongo_table_instance('stock_day_k')
+stock_concept_table = get_mongo_table_instance('tushare_concept_detail')
+all_concept_code_list = list(get_mongo_table_instance('tushare_concept').find({}))
+
+
+industry = ['家用电器', '元器件', 'IT设备', '汽车服务',
+            '汽车配件', '软件服务',
+            '互联网', '纺织',
+            '塑料', '半导体',]
+
+A_concept_code_list = [   'TS2', # 5G
+                        'TS24', # OLED
+                        'TS26', #健康中国
+                        'TS43',  #新能源整车
+                        'TS59', # 特斯拉
+                        'TS65', #汽车整车
+                        'TS142', # 物联网
+                        'TS153', # 无人驾驶
+                        'TS163', # 雄安板块-智慧城市
+                        'TS175', # 工业自动化
+                        'TS232', # 新能源汽车
+                        'TS254', # 人工智能
+                        'TS258', # 互联网医疗
+                        'TS264', # 工业互联网
+                        'TS266', # 半导体
+                        'TS269', # 智慧城市
+                        'TS271', # 3D玻璃
+                        'TS295', # 国产芯片
+                        'TS303', # 医疗信息化
+                        'TS323', # 充电桩
+                        'TS328', # 虹膜识别
+                        'TS361', # 病毒
+    ]
+
+
+gainian_map = {}
+hangye_map = {}
+
+def predict_today(file, day, model='10_18d', log=True):
+    lines = []
+    with open(file) as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            # if line[-1][0].startswith('0') or line[-1][0].startswith('3'):
+            lines.append(line)
+
+    size = len(lines[0])
+
+    model=load_model(model)
+
+    for line in lines:
+        train_x = np.array([line[:size - 1]])
+        train_x_a = train_x[:,:18*24]
+        train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24)
+        # train_x_b = train_x[:, 18*18:18*18+2*18]
+        # train_x_b = train_x_b.reshape(train_x.shape[0], 18, 2, 1)
+        train_x_c = train_x[:,18*24:]
+
+        result = model.predict([train_x_c, train_x_a])
+        # print(result, line[-1])
+        stock = code_table.find_one({'ts_code':line[-1][0]})
+
+        if result[0][0] + result[0][1] > 0.5:
+            if line[-1][0].startswith('688'):
+                continue
+            # 去掉ST
+            if stock['name'].startswith('ST') or stock['name'].startswith('N') or stock['name'].startswith('*'):
+                continue
+
+            if stock['ts_code'] in holder_stock_list:
+                print(stock['ts_code'], stock['name'], '维持买入评级')
+
+            # 跌的
+            k_table_list = list(k_table.find({'code':line[-1][0], 'tradeDate':{'$lte':day}}).sort("tradeDate", pymongo.DESCENDING).limit(5))
+            # if k_table_list[0]['close'] > k_table_list[-1]['close']*1.20:
+            #     continue
+            # if k_table_list[0]['close'] < k_table_list[-1]['close']*0.90:
+            #     continue
+            # if k_table_list[-1]['close'] > 80:
+            #     continue
+
+            # 指定某几个行业
+            # if stock['industry'] in industry:
+            concept_code_list = list(stock_concept_table.find({'ts_code':stock['ts_code']}))
+            concept_detail_list = []
+
+            # 处理行业
+            if stock['sw_industry'] in hangye_map:
+                i_c = hangye_map[stock['sw_industry']]
+                hangye_map[stock['sw_industry']] = i_c + 1
+            else:
+                hangye_map[stock['sw_industry']] = 1
+
+            if len(concept_code_list) > 0:
+                for concept in concept_code_list:
+                    for c in all_concept_code_list:
+                        if c['code'] == concept['concept_code']:
+                            concept_detail_list.append(c['name'])
+
+                            if c['name'] in gainian_map:
+                                g_c = gainian_map[c['name']]
+                                gainian_map[c['name']] = g_c + 1
+                            else:
+                                gainian_map[c['name']] = 1
+
+            print(line[-1], stock['name'], stock['sw_industry'], str(concept_detail_list), 'buy', k_table_list[0]['pct_chg'])
+
+            if log is True:
+                with open('D:\\data\\quantization\\predict\\' + str(day) + '_lstm.txt', mode='a', encoding="utf-8") as f:
+                    f.write(str(line[-1]) + ' ' + stock['name'] + ' ' + stock['sw_industry'] + ' ' + str(concept_detail_list) + ' buy' + '\n')
+
+        elif result[0][2] > 0.5:
+            if stock['ts_code'] in holder_stock_list:
+                print(stock['ts_code'], stock['name'], '震荡评级')
+
+        elif result[0][3] + result[0][4] > 0.5:
+            if stock['ts_code'] in holder_stock_list:
+                print(stock['ts_code'], stock['name'], '赶紧卖出')
+        else:
+            if stock['ts_code'] in holder_stock_list:
+                print(stock['ts_code'], stock['name'], result[0],)
+
+    # print(gainian_map)
+    # print(hangye_map)
+
+    gainian_list = [(key, gainian_map[key])for key in gainian_map]
+    gainian_list = sorted(gainian_list, key=lambda x:x[1], reverse=True)
+
+    hangye_list = [(key, hangye_map[key])for key in hangye_map]
+    hangye_list = sorted(hangye_list, key=lambda x:x[1], reverse=True)
+
+    print(gainian_list)
+    print(hangye_list)
+
+def _read_pfile_map(path):
+    s_list = []
+    with open(path, encoding='utf-8') as f:
+        for line in f.readlines()[:]:
+            s_list.append(line)
+    return s_list
+
+
+def join_two_day(a, b):
+    a_list = _read_pfile_map('D:\\data\\quantization\\predict\\' + str(a) + '.txt')
+    b_list = _read_pfile_map('D:\\data\\quantization\\predict\\dmi_' + str(b) + '.txt')
+    for a in a_list:
+        for b in b_list:
+            if a[2:11] == b[2:11]:
+                print(a)
+
+
+def check_everyday(day, today):
+    a_list = _read_pfile_map('D:\\data\\quantization\\predict\\' + str(day) + '.txt')
+    x = 0
+    for a in a_list:
+        print(a[:-1])
+        k_day_list = list(k_table.find({'code':a[2:11], 'tradeDate':{'$lte':int(today)}}).sort('tradeDate', pymongo.DESCENDING).limit(5))
+        if k_day_list is not None and len(k_day_list) > 0:
+            k_day = k_day_list[0]
+            k_day_0 = k_day_list[-1]
+            k_day_last = k_day_list[1]
+            if ((k_day_last['close'] - k_day_0['pre_close'])/k_day_0['pre_close']) < 0.2:
+                print(k_day['open'], k_day['close'], 100*(k_day['close'] - k_day_last['close'])/k_day_last['close'])
+                x = x + 100*(k_day['close'] - k_day_last['close'])/k_day_last['close']
+
+    print(x/len(a_list))
+
+
+if __name__ == '__main__':
+    # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
+    # multi_predict()
+    predict_today("D:\\data\\quantization\\stock160_18d_20200312.log", 20200313, model='160_18d_lstm_5D_ma5_s_seq.h5', log=True)
+    # join_two_day(20200305, 20200305)
+    # check_everyday(20200311, 20200312)

mix/lstm_train.py

@@ -20,8 +20,10 @@ from keras.callbacks import EarlyStopping
 
 early_stopping = EarlyStopping(monitor='accuracy', patience=5, verbose=2)
 
-epochs= 200
-size = 380000
+epochs= 440
+size = 580000  # 61W
+file_path = 'D:\\data\\quantization\\stock160_18d_train.log'
+model_path = '160_18d_lstm_5D_ma5_s_seq.h5'
 
 def read_data(path):
     lines = []
@@ -49,7 +51,7 @@ def read_data(path):
     return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
 
 
-train_x,train_y,test_x,test_y=read_data("D:\\data\\quantization\\stock17_18d_train.log")
+train_x,train_y,test_x,test_y=read_data(file_path)
 
 train_x_a = train_x[:,:18*24]
 train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24)
@@ -61,8 +63,8 @@ train_x_c = train_x[:,18*24:]
 def create_mlp(dim, regress=False):
     # define our MLP network
     model = Sequential()
-    model.add(Dense(16, input_dim=dim, activation="relu"))
-    model.add(Dense(16, activation="relu"))
+    model.add(Dense(64, input_dim=dim, activation="relu"))
+    model.add(Dense(64, activation="relu"))
 
     # check to see if the regression node should be added
     if regress:
@@ -151,10 +153,10 @@ combinedInput = concatenate([mlp.output, cnn_0.output,])
 
 # our final FC layer head will have two dense layers, the final one
 # being our regression head
-x = Dense(666, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
+x = Dense(256, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
 x = Dropout(0.2)(x)
-x = Dense(666, activation="relu")(x)
-x = Dense(666, activation="relu")(x)
+x = Dense(256, activation="relu")(x)
+x = Dense(512, activation="relu")(x)
 # 在建设一层
 x = Dense(5, activation="softmax")(x)
 
@@ -181,7 +183,7 @@ model.fit(
     # validation_data=([testAttrX, testImagesX], testY),
     # epochs=int(3*train_x_a.shape[0]/1300),
     epochs=epochs,
-    batch_size=2048, shuffle=True,
+    batch_size=4096, shuffle=True,
     callbacks=[early_stopping]
 )
 
@@ -199,6 +201,6 @@ print(score)
 print('Test score:', score[0])
 print('Test accuracy:', score[1])
 
-path="17_18d_lstm_seq.h5"
-model.save(path)
+
+model.save(model_path)
 model=None

mix/mix_kmeans_predict_1.py

@@ -0,0 +1,125 @@
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import load_model
+import joblib
+
+model_path = '160_18d_mix_5D_ma5_s_seq.h5'
+data_dir = 'D:\\data\\quantization\\'
+kmeans = 'roc'
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            if line[-2][0].startswith('0') or line[-2][0].startswith('3'):
+                lines.append(line)
+
+    size = len(lines[0])
+    train_x=[s[:size - 2] for s in lines]
+    train_y=[s[size-1] for s in lines]
+    return np.array(train_x),np.array(train_y),lines
+
+
+def _score(fact, line):
+    with open('mix_predict_dmi_18d.txt', 'a') as f:
+        f.write(str([line[-2], line[-1]]) + "\n")
+
+    up_right = 0
+    up_error = 0
+
+    if fact[0] == 1:
+        up_right = up_right + 1.12
+    elif fact[1] == 1:
+        up_right = up_right + 1.06
+    elif fact[2] == 1:
+        up_right = up_right + 1
+        up_error = up_error + 0.5
+    elif fact[3] == 1:
+        up_right = up_right + 0.94
+        up_error = up_error + 1
+    else:
+        up_error = up_error + 1
+        up_right = up_right + 0.88
+    return up_right,up_error
+
+
+def mul_predict(name="10_18d"):
+    r = 0
+    p = 0
+
+    for x in range(0, 8):
+        win_dnn, up_ratio,down_ratio  = predict(data_dir + kmeans + '\\stock160_18d_train1_B_' + str(x) + ".log", x) # stock160_18d_trai_0
+
+        r = r + up_ratio
+        p = p + down_ratio
+
+    print(r, p)
+
+
+def predict(file_path='', idx=-1):
+    test_x,test_y,lines=read_data(file_path)
+    print(idx, 'Load data success')
+
+    test_x_a = test_x[:,:18*24]
+    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 24, 1)
+    # test_x_b = test_x[:, 18*16:18*16+10*18]
+    # test_x_b = test_x_b.reshape(test_x.shape[0], 18, 10, 1)
+    test_x_c = test_x[:,18*24:]
+
+    model=load_model(model_path.split('.')[0] + '_' + str(idx) + '.h5')
+    score = model.evaluate([test_x_c, test_x_a, ], test_y)
+    print('MIX', score)
+
+    up_num = 0
+    up_error = 0
+    up_right = 0
+    down_num = 0
+    down_error = 0
+    down_right = 0
+    i = 0
+    result=model.predict([test_x_c, test_x_a, ])
+    win_dnn = []
+    for r in result:
+        fact = test_y[i]
+
+        if idx in [-2]:
+            if r[0] > 0.5 or r[1] > 0.5:
+                pass
+        else:
+            if r[0] > 0.6 or r[1] > 0.6:
+                tmp_right,tmp_error = _score(fact, lines[i])
+                up_right = tmp_right + up_right
+                up_error = tmp_error + up_error
+                up_num = up_num + 1
+            elif r[3] > 0.7 or r[4] > 0.7:
+                if fact[0] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.12
+                elif fact[1] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.06
+                elif fact[2] == 1:
+                    down_error = down_error + 0.5
+                    down_right = down_right + 1
+                elif fact[3] == 1:
+                    down_right = down_right + 0.94
+                else:
+                    down_right = down_right + 0.88
+                down_num = down_num + 1
+
+        i = i + 1
+    if up_num == 0:
+        up_num = 1
+    if down_num == 0:
+        down_num = 1
+    print('MIX', up_right, up_num, up_right/up_num, up_error/up_num, down_right/down_num, down_error/down_num)
+    return win_dnn,up_right/up_num,down_right/down_num
+
+
+if __name__ == '__main__':
+    # predict(file_path='D:\\data\\quantization\\stock160_18d_10D_test.log', model_path='160_18d_lstm_5D_ma5_s_seq.h5')
+    # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
+    mul_predict(name='stock160_18d')
+    # predict_today(20200229, model='11_18d')

mix/mix_kmeans_train_1.py

@@ -0,0 +1,187 @@
+import keras
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import Sequential
+# 优化方法选用Adam(其实可选项有很多，如SGD)
+from keras.optimizers import Adam
+import random
+from keras.models import load_model
+from imblearn.over_sampling import RandomOverSampler
+from keras.utils import np_utils
+# 用于模型初始化，Conv2D模型初始化、Activation激活函数，MaxPooling2D是池化层
+# Flatten作用是将多位输入进行一维化
+# Dense是全连接层
+from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense,Dropout,Input,MaxPooling2D,BatchNormalization,concatenate
+from keras import regularizers
+from keras.models import Model
+
+epochs= 50
+size = 580000
+file_path = 'D:\\data\\quantization\\stock160_18d_10D_train.log'
+model_path = '160_18d_mix_5D_ma5_s_seq.h5'
+data_dir = 'D:\\data\\quantization\\'
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        i = 0
+        for line in f.readlines()[:]:
+            lines.append(eval(line.strip()))
+
+    random.shuffle(lines)
+    print('读取数据完毕')
+
+    d=int(0.7*len(lines))
+
+    train_x=[s[:-2] for s in lines[0:d]]
+    train_y=[s[-1] for s in lines[0:d]]
+    test_x=[s[:-2] for s in lines[d:]]
+    test_y=[s[-1] for s in lines[d:]]
+
+    print('转换数据完毕')
+
+    ros = RandomOverSampler(random_state=0)
+    X_resampled, y_resampled = ros.fit_sample(np.array(train_x), np.array(train_y))
+
+    print('数据重采样完毕')
+
+    return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
+
+
+def mul_train(name="10_18d"):
+    for x in range(0, 8):
+        score = train(data_dir + 'kmeans\\' + name + "_trai_" + str(x) + ".log", x) # stock160_18d_trai_0
+
+        with open(data_dir + name + '_mix.log', 'a') as f:
+            f.write(str(x) + ':' + str(score[1]) + '\n')
+
+
+def train(file_path_name, idx):
+    train_x,train_y,test_x,test_y=read_data(file_path_name)
+
+    train_x_a = train_x[:,:18*24]
+    train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24, 1)
+    # train_x_b = train_x[:, 18*18:18*18+2*18]
+    # train_x_b = train_x_b.reshape(train_x.shape[0], 18, 2, 1)
+    train_x_c = train_x[:,18*24:]
+
+    # create the MLP and CNN models
+    mlp = create_mlp(train_x_c.shape[1], regress=False)
+    cnn_0 = create_cnn(18, 24, 1, kernel_size=(6, 6), regress=False, output=256)
+    # cnn_1 = create_cnn(18, 2, 1, kernel_size=(6,2), regress=False, output=36)
+
+    # create the input to our final set of layers as the *output* of both
+    # the MLP and CNN
+    combinedInput = concatenate([mlp.output, cnn_0.output])
+
+    # our final FC layer head will have two dense layers, the final one
+    # being our regression head
+    x = Dense(512, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
+    x = Dropout(0.2)(x)
+    x = Dense(512, activation="relu")(x)
+    x = Dense(512, activation="relu")(x)
+    # 在建设一层
+    x = Dense(5, activation="softmax")(x)
+
+    # our final model will accept categorical/numerical data on the MLP
+    # input and images on the CNN input, outputting a single value (the
+    # predicted price of the house)
+    model = Model(inputs=[mlp.input, cnn_0.input], outputs=x)
+
+
+    print("Starting training ")
+    # h = model.fit(train_x, train_y, batch_size=4096*2, epochs=500, shuffle=True)
+
+    # compile the model using mean absolute percentage error as our loss,
+    # implying that we seek to minimize the absolute percentage difference
+    # between our price *predictions* and the *actual prices*
+    opt = Adam(lr=1e-3, decay=1e-3 / 200)
+    model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'])
+
+    # train the model
+    print("[INFO] training model...")
+    model.fit(
+        [train_x_c, train_x_a], train_y,
+        # validation_data=([testAttrX, testImagesX], testY),
+        # epochs=int(3*train_x_a.shape[0]/1300),
+        epochs=epochs,
+        batch_size=2048, shuffle=True)
+
+    test_x_a = test_x[:,:18*24]
+    test_x_a = test_x_a.reshape(test_x.shape[0], 18, 24, 1)
+    # test_x_b = test_x[:, 18*18:18*18+2*18]
+    # test_x_b = test_x_b.reshape(test_x.shape[0], 18, 2, 1)
+    test_x_c = test_x[:,18*24:]
+
+    # make predictions on the testing data
+    print("[INFO] predicting house prices...")
+    score  = model.evaluate([test_x_c, test_x_a], test_y)
+
+    print(score)
+    print('Test score:', score[0])
+    print('Test accuracy:', score[1])
+
+    model.save(model_path.split('.')[0] + '_' + str(idx) + '.h5')
+    return score
+
+def create_mlp(dim, regress=False):
+    # define our MLP network
+    model = Sequential()
+    model.add(Dense(64, input_dim=dim, activation="relu"))
+    model.add(Dense(64, activation="relu"))
+
+    # check to see if the regression node should be added
+    if regress:
+        model.add(Dense(1, activation="linear"))
+
+    # return our model
+    return model
+
+
+def create_cnn(width, height, depth, filters=(4, 6), kernel_size=(5, 6), regress=False, output=24):
+    # initialize the input shape and channel dimension, assuming
+    # TensorFlow/channels-last ordering
+    inputShape = (width, height, 1)
+    chanDim = -1
+
+    # define the model input
+    inputs = Input(shape=inputShape)
+
+    x = inputs
+
+    # CONV => RELU => BN => POOL
+    x = Conv2D(32, kernel_size, strides=2, padding="same")(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    # x = MaxPooling2D(pool_size=(2, 2))(x)
+    # if width > 2:
+    #     x = Conv2D(32, (10, 6), padding="same")(x)
+    #     x = Activation("relu")(x)
+    #     x = BatchNormalization(axis=chanDim)(x)
+
+    # flatten the volume, then FC => RELU => BN => DROPOUT
+    x = Flatten()(x)
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    x = Dropout(0.2)(x)
+
+    # apply another FC layer, this one to match the number of nodes
+    # coming out of the MLP
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+
+    # check to see if the regression node should be added
+    if regress:
+        x = Dense(1, activation="linear")(x)
+
+    # construct the CNN
+    model = Model(inputs, x)
+
+    # return the CNN
+    return model
+
+
+if __name__ == '__main__':
+    mul_train('stock160_18d')

mix/mix_predict_1.py

@@ -100,7 +100,7 @@ def predict(file_path='', model_path='15min_dnn_seq.h5', idx=-1):
 
 
 if __name__ == '__main__':
-    predict(file_path='D:\\data\\quantization\\stock16_18d_test_close.log', model_path='16_18d_mix_seq.h5')
+    predict(file_path='D:\\data\\quantization\\stock160_18d_train1.log', model_path='160_18d_mix_5D_ma5_s_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
     # multi_predict(model='15_18d')
     # predict_today(20200229, model='11_18d')

mix/mix_predict_by_day.py

@@ -33,6 +33,11 @@ def predict(file_path='', model_path='15min_dnn_seq'):
         up_num = 0
         down_num = 0
         size = len(lines[0])
+        x0 = 0
+        x1 = 0
+        x2 = 0
+        x3 = 0
+        x4 = 0
 
         for line in lines:
             train_x = np.array([line[:size - 1]])
@@ -44,15 +49,28 @@ def predict(file_path='', model_path='15min_dnn_seq'):
 
             result = model.predict([train_x_c, train_x_a])
 
-            if result[0][3] > 0.5 or result[0][4] > 0.5:
+            if result[0][3] + result[0][4] > 0.5:
                 down_num = down_num + 1
-            elif result[0][1] > 0.5 or result[0][0] > 0.5:
-                up_num = up_num + 1
-            else:
-                up_num = up_num + 0.4 # 乐观调大 悲观调小
-                down_num = down_num + 0.6
+            elif result[0][1] + result[0][0] > 0.5:
+                up_num = up_num + 0.6
+            # else:
+            #     up_num = up_num + 0.4 # 乐观调大 悲观调小
+            #     down_num = down_num + 0.6
+
+            maxx = max(result[0])
+            if maxx - result[0][0] == 0:
+                x0 = x0 + 1
+            if maxx - result[0][1] == 0:
+                x1 = x1 + 1
+            if maxx - result[0][2] == 0:
+                x2 = x2 + 1
+            if maxx - result[0][3] == 0:
+                x3 = x3 + 1
+            if maxx - result[0][4] == 0:
+                x4 = x4 + 1
 
-        print(key, int(up_num), int(down_num), (down_num*1.2 + 2)/(up_num*1.2 + 2))
+        # print(key, int(up_num), int(down_num), (down_num*1.2 + 2)/(up_num*1.2 + 2), )
+        print(key, x0, x1, x2,x3,x4)
 
 
 if __name__ == '__main__':
@@ -60,5 +78,5 @@ if __name__ == '__main__':
     # predict(file_path='D:\\data\\quantization\\stock9_18_20200220.log', model_path='18d_dnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock9_18_2.log', model_path='18d_dnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock16_18d_20200310.log', model_path='16_18d_mix_seq')
-    predict(file_path='D:\\data\\quantization\\stock16_18d_20191225_20200310_1.log', model_path='16_18d_mix_seq')
+    predict(file_path='D:\\data\\quantization\\stock160_18d_10D_20200313.log', model_path='160_18d_mix_10D_ma5_s_seq')
     # predict(file_path='D:\\data\\quantization\\stock9_18_4.log', model_path='18d_dnn_seq.h5')

mix/mix_predict_everyday.py

@@ -5,15 +5,11 @@ import joblib
 
 
 holder_stock_list = [
-                        '000063.SZ',
-                        '002373.SZ',
-                        '300253.SZ',
-                        '300059.SZ',
-                        '300807.SZ',
-                        '600345.SH',
+                        '000063.SZ', '002093.SZ'
+                        '300253.SZ', '300807.SZ',
+
                          # b账户
-                        '300422.SZ',
-                        '300468.SZ',
+
     ]
 
 
@@ -72,9 +68,9 @@ A_concept_code_list = [   'TS2', # 5G
 gainian_map = {}
 hangye_map = {}
 
-def predict_today(day, model='10_18d', log=True):
+def predict_today(file, day, model='10_18d', log=True):
     lines = []
-    with open('D:\\data\\quantization\\stock' + model[:-4] + '_' +  str(day) +'.log') as f:
+    with open(file) as f:
         for line in f.readlines()[:]:
             line = eval(line.strip())
             # if line[-1][0].startswith('0') or line[-1][0].startswith('3'):
@@ -82,7 +78,7 @@ def predict_today(day, model='10_18d', log=True):
 
     size = len(lines[0])
 
-    model=load_model(model + '_seq.h5')
+    model=load_model(model)
 
     for line in lines:
         train_x = np.array([line[:size - 1]])
@@ -96,7 +92,7 @@ def predict_today(day, model='10_18d', log=True):
         # print(result, line[-1])
         stock = code_table.find_one({'ts_code':line[-1][0]})
 
-        if result[0][0] > 0.5 or result[0][1] > 0.5:
+        if result[0][0] + result[0][1] > 0.5:
             if line[-1][0].startswith('688'):
                 continue
             # 去掉ST
@@ -127,42 +123,46 @@ def predict_today(day, model='10_18d', log=True):
             else:
                 hangye_map[stock['sw_industry']] = 1
 
-            # if len(concept_code_list) > 0:
-            #     for concept in concept_code_list:
-            #         for c in all_concept_code_list:
-            #             if c['code'] == concept['concept_code']:
-            #                 concept_detail_list.append(c['name'])
-            #
-            #                 if c['name'] in gainian_map:
-            #                     g_c = gainian_map[c['name']]
-            #                     gainian_map[c['name']] = g_c + 1
-            #                 else:
-            #                     gainian_map[c['name']] = 1
+            if len(concept_code_list) > 0:
+                for concept in concept_code_list:
+                    for c in all_concept_code_list:
+                        if c['code'] == concept['concept_code']:
+                            concept_detail_list.append(c['name'])
+
+                            if c['name'] in gainian_map:
+                                g_c = gainian_map[c['name']]
+                                gainian_map[c['name']] = g_c + 1
+                            else:
+                                gainian_map[c['name']] = 1
 
             print(line[-1], stock['name'], stock['sw_industry'], str(concept_detail_list), 'buy', k_table_list[0]['pct_chg'])
 
             if log is True:
-                with open('D:\\data\\quantization\\predict\\' + str(day) + '.txt', mode='a', encoding="utf-8") as f:
+                with open('D:\\data\\quantization\\predict\\' + str(day) + '_mix.txt', mode='a', encoding="utf-8") as f:
                     f.write(str(line[-1]) + ' ' + stock['name'] + ' ' + stock['sw_industry'] + ' ' + str(concept_detail_list) + ' buy' + '\n')
 
-
-            # concept_list = list(stock_concept_table.find({'ts_code':stock['ts_code']}))
-            # concept_list = [c['concept_code'] for c in concept_list]
-
         elif result[0][2] > 0.5:
             if stock['ts_code'] in holder_stock_list:
                 print(stock['ts_code'], stock['name'], '震荡评级')
 
-        elif result[0][3] > 0.5 or result[0][4] > 0.5:
+        elif result[0][3] + result[0][4] > 0.5:
             if stock['ts_code'] in holder_stock_list:
                 print(stock['ts_code'], stock['name'], '赶紧卖出')
         else:
             if stock['ts_code'] in holder_stock_list:
                 print(stock['ts_code'], stock['name'], result[0],)
 
-    print(gainian_map)
-    print(hangye_map)
+    # print(gainian_map)
+    # print(hangye_map)
+
+    gainian_list = [(key, gainian_map[key])for key in gainian_map]
+    gainian_list = sorted(gainian_list, key=lambda x:x[1], reverse=True)
 
+    hangye_list = [(key, hangye_map[key])for key in hangye_map]
+    hangye_list = sorted(hangye_list, key=lambda x:x[1], reverse=True)
+
+    print(gainian_list)
+    print(hangye_list)
 
 def _read_pfile_map(path):
     s_list = []
@@ -181,9 +181,27 @@ def join_two_day(a, b):
                 print(a)
 
 
+def check_everyday(day, today):
+    a_list = _read_pfile_map('D:\\data\\quantization\\predict\\' + str(day) + '.txt')
+    x = 0
+    for a in a_list:
+        print(a[:-1])
+        k_day_list = list(k_table.find({'code':a[2:11], 'tradeDate':{'$lte':int(today)}}).sort('tradeDate', pymongo.DESCENDING).limit(5))
+        if k_day_list is not None and len(k_day_list) > 0:
+            k_day = k_day_list[0]
+            k_day_0 = k_day_list[-1]
+            k_day_last = k_day_list[1]
+            if ((k_day_last['close'] - k_day_0['pre_close'])/k_day_0['pre_close']) < 0.2:
+                print(k_day['open'], k_day['close'], 100*(k_day['close'] - k_day_last['close'])/k_day_last['close'])
+                x = x + 100*(k_day['close'] - k_day_last['close'])/k_day_last['close']
+
+    print(x/len(a_list))
+
+
 if __name__ == '__main__':
     # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
     # multi_predict()
-    predict_today(20200310, model='16_18d_mix', log=True)
-    # join_two_day(20200305, 20200305)
+    predict_today("D:\\data\\quantization\\stock160_18d_10D_20200313.log", 20200313, model='160_18d_mix_5D_ma5_s_seq.h5', log=True)
+    # join_two_day(20200305, 20200305)
+    # check_everyday(20200311, 20200312)

mix/mix_train.py

@@ -21,7 +21,8 @@ early_stopping = EarlyStopping(monitor='accuracy', patience=5, verbose=2)
 
 epochs= 120
 size = 380000
-
+model_path="160_18d_mix_5D_ma5_s_seq.h5"
+# 已废弃
 
 def read_data(path):
     lines = []
@@ -176,6 +177,6 @@ print(score)
 print('Test score:', score[0])
 print('Test accuracy:', score[1])
 
-path="16_18d_mix_seq.h5"
-model.save(path)
+
+model.save(model_path)
 model=None

mix/mix_train_1.py

@@ -15,15 +15,19 @@ from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense,Dropout,I
 from keras import regularizers
 from keras.models import Model
 
-epochs= 130
-size = 380000
+epochs= 440
+size = 580000
+file_path = 'D:\\data\\quantization\\stock160_18d_train.log'
+model_path = '160_18d_mix_5D_ma5_s_seq.h5'
 
 
 def read_data(path):
     lines = []
     with open(path) as f:
-        for x in range(size): #380000
-            lines.append(eval(f.readline().strip()))
+        i = 0
+        for x in range(size): #610000
+            line = eval(f.readline().strip())
+            lines.append(line)
 
     random.shuffle(lines)
     print('读取数据完毕')
@@ -45,7 +49,7 @@ def read_data(path):
     return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
 
 
-train_x,train_y,test_x,test_y=read_data("D:\\data\\quantization\\stock16_18d_train.log")
+train_x,train_y,test_x,test_y=read_data(file_path)
 
 train_x_a = train_x[:,:18*24]
 train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24, 1)
@@ -168,6 +172,4 @@ print(score)
 print('Test score:', score[0])
 print('Test accuracy:', score[1])
 
-path="16_18d_mix_seq.h5"
-model.save(path)
-model=None
+model.save(model_path)

mix/mix_train_2.py

@@ -0,0 +1,175 @@
+import keras
+# -*- encoding:utf-8 -*-
+import numpy as np
+from keras.models import Sequential
+# 优化方法选用Adam(其实可选项有很多，如SGD)
+from keras.optimizers import Adam
+import random
+from keras.models import load_model
+from imblearn.over_sampling import RandomOverSampler
+from keras.utils import np_utils
+# 用于模型初始化，Conv2D模型初始化、Activation激活函数，MaxPooling2D是池化层
+# Flatten作用是将多位输入进行一维化
+# Dense是全连接层
+from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense,Dropout,Input,MaxPooling2D,BatchNormalization,concatenate
+from keras import regularizers
+from keras.models import Model
+
+epochs= 440
+size = 580000
+file_path = 'D:\\data\\quantization\\stock160_18d_train.log'
+model_path = '160_18d_mix_5D_ma5_s_seq.h5'
+
+
+def read_data(path):
+    lines = []
+    with open(path) as f:
+        i = 0
+        for x in range(size): #610000
+            line = eval(f.readline().strip())
+            lines.append(line)
+
+    random.shuffle(lines)
+    print('读取数据完毕')
+
+    d=int(0.7*len(lines))
+
+    train_x=[s[:-2] for s in lines[0:d]]
+    train_y=[s[-1] for s in lines[0:d]]
+    test_x=[s[:-2] for s in lines[d:]]
+    test_y=[s[-1] for s in lines[d:]]
+
+    print('转换数据完毕')
+
+    ros = RandomOverSampler(random_state=0)
+    X_resampled, y_resampled = ros.fit_sample(np.array(train_x), np.array(train_y))
+
+    print('数据重采样完毕')
+
+    return X_resampled,y_resampled,np.array(test_x),np.array(test_y)
+
+
+train_x,train_y,test_x,test_y=read_data(file_path)
+
+train_x_a = train_x[:,:18*24]
+train_x_a = train_x_a.reshape(train_x.shape[0], 18, 24, 1)
+# train_x_b = train_x[:, 18*18:18*18+2*18]
+# train_x_b = train_x_b.reshape(train_x.shape[0], 18, 2, 1)
+train_x_c = train_x[:,18*24:]
+
+
+def create_mlp(dim, regress=False):
+    # define our MLP network
+    model = Sequential()
+    model.add(Dense(64, input_dim=dim, activation="relu"))
+    model.add(Dense(64, activation="relu"))
+
+    # check to see if the regression node should be added
+    if regress:
+        model.add(Dense(1, activation="linear"))
+
+    # return our model
+    return model
+
+
+def create_cnn(width, height, depth, filters=(4, 6), kernel_size=(5, 6), regress=False, output=24):
+    # initialize the input shape and channel dimension, assuming
+    # TensorFlow/channels-last ordering
+    inputShape = (width, height, 1)
+    chanDim = -1
+
+    # define the model input
+    inputs = Input(shape=inputShape)
+
+    x = inputs
+
+    # CONV => RELU => BN => POOL
+    x = Conv2D(32, kernel_size, strides=2, padding="same")(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    # x = MaxPooling2D(pool_size=(2, 2))(x)
+    # if width > 2:
+    #     x = Conv2D(32, (10, 6), padding="same")(x)
+    #     x = Activation("relu")(x)
+    #     x = BatchNormalization(axis=chanDim)(x)
+
+    # flatten the volume, then FC => RELU => BN => DROPOUT
+    x = Flatten()(x)
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+    x = BatchNormalization(axis=chanDim)(x)
+    x = Dropout(0.2)(x)
+
+    # apply another FC layer, this one to match the number of nodes
+    # coming out of the MLP
+    x = Dense(output)(x)
+    x = Activation("relu")(x)
+
+    # check to see if the regression node should be added
+    if regress:
+        x = Dense(1, activation="linear")(x)
+
+    # construct the CNN
+    model = Model(inputs, x)
+
+    # return the CNN
+    return model
+
+
+# create the MLP and CNN models
+mlp = create_mlp(train_x_c.shape[1], regress=False)
+cnn_0 = create_cnn(18, 24, 1, kernel_size=(6, 6), regress=False, output=256)
+# cnn_1 = create_cnn(18, 2, 1, kernel_size=(6,2), regress=False, output=36)
+
+# create the input to our final set of layers as the *output* of both
+# the MLP and CNN
+combinedInput = concatenate([mlp.output, cnn_0.output])
+
+# our final FC layer head will have two dense layers, the final one
+# being our regression head
+x = Dense(512, activation="relu", kernel_regularizer=regularizers.l1(0.003))(combinedInput)
+x = Dropout(0.2)(x)
+x = Dense(512, activation="relu")(x)
+x = Dense(512, activation="relu")(x)
+# 在建设一层
+x = Dense(5, activation="softmax")(x)
+
+# our final model will accept categorical/numerical data on the MLP
+# input and images on the CNN input, outputting a single value (the
+# predicted price of the house)
+model = Model(inputs=[mlp.input, cnn_0.input], outputs=x)
+
+
+print("Starting training ")
+# h = model.fit(train_x, train_y, batch_size=4096*2, epochs=500, shuffle=True)
+
+# compile the model using mean absolute percentage error as our loss,
+# implying that we seek to minimize the absolute percentage difference
+# between our price *predictions* and the *actual prices*
+opt = Adam(lr=1e-3, decay=1e-3 / 200)
+model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'])
+
+# train the model
+print("[INFO] training model...")
+model.fit(
+    [train_x_c, train_x_a], train_y,
+    # validation_data=([testAttrX, testImagesX], testY),
+    # epochs=int(3*train_x_a.shape[0]/1300),
+    epochs=epochs,
+    batch_size=2048, shuffle=True)
+
+test_x_a = test_x[:,:18*24]
+test_x_a = test_x_a.reshape(test_x.shape[0], 18, 24, 1)
+# test_x_b = test_x[:, 18*18:18*18+2*18]
+# test_x_b = test_x_b.reshape(test_x.shape[0], 18, 2, 1)
+test_x_c = test_x[:,18*24:]
+
+# make predictions on the testing data
+print("[INFO] predicting house prices...")
+score  = model.evaluate([test_x_c, test_x_a], test_y)
+
+print(score)
+print('Test score:', score[0])
+print('Test accuracy:', score[1])
+
+model.save(model_path)