modify

stock/dnn_predict_dmi.py

@@ -1,13 +1,16 @@
 # -*- encoding:utf-8 -*-
 import numpy as np
 from keras.models import load_model
+import joblib
 
 
 def read_data(path):
     lines = []
     with open(path) as f:
         for line in f.readlines()[:]:
-            lines.append(eval(line.strip()))
+            line = eval(line.strip())
+            if line[-2][0].startswith('0') or line[-2][0].startswith('3'):
+                lines.append(line)
 
     size = len(lines[0])
     train_x=[s[:size - 2] for s in lines]
@@ -23,40 +26,184 @@ def predict(file_path='', model_path='15min_dnn_seq.h5'):
     print('DNN', score)
 
     up_num = 0
+    up_error = 0
     up_right = 0
+    down_num = 0
+    down_error = 0
+    down_right = 0
     i = 0
     result=model.predict(test_x)
     win_dnn = []
-    with open('dnn_predict_dmi_14d.txt', 'a') as f:
+    with open('dnn_predict_dmi_18d.txt', 'a') as f:
         for r in result:
             fact = test_y[i]
             if r[0] > 0.5:
                 f.write(str([lines[i][-2], lines[i][-1]]) + "\n")
                 win_dnn.append([lines[i][-2], lines[i][-1]])
                 if fact[0] == 1:
+                    up_right = up_right + 1.12
+                elif fact[1] == 1:
                     up_right = up_right + 1.06
+                elif fact[2] == 1:
+                    up_right = up_right + 1
+                elif fact[3] == 1:
+                    up_right = up_right + 0.94
+                else:
+                    up_error = up_error + 1
+                    up_right = up_right + 0.88
+                up_num = up_num + 1
+            elif r[1] > 0.5:
+                f.write(str([lines[i][-2], lines[i][-1]]) + "\n")
+                win_dnn.append([lines[i][-2], lines[i][-1]])
+                if fact[0] == 1:
+                    up_right = up_right + 1.12
                 elif fact[1] == 1:
+                    up_right = up_right + 1.06
+                elif fact[2] == 1:
                     up_right = up_right + 1
+                elif fact[3] == 1:
+                    up_right = up_right + 0.94
                 else:
-                    up_right = up_right + 0.9
+                    up_error = up_error + 1
+                    up_right = up_right + 0.88
                 up_num = up_num + 1
 
+            if r[3] > 0.6:
+                f.write(str([lines[i][-2], lines[i][-1]]) + "\n")
+                win_dnn.append([lines[i][-2], lines[i][-1]])
+                if fact[0] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.12
+                elif fact[1] == 1:
+                    down_right = down_right + 1.06
+                elif fact[2] == 1:
+                    down_right = down_right + 1
+                elif fact[3] == 1:
+                    down_right = down_right + 0.94
+                else:
+                    down_right = down_right + 0.88
+                down_num = down_num + 1
+            elif r[4] > 0.6:
+                f.write(str([lines[i][-2], lines[i][-1]]) + "\n")
+                win_dnn.append([lines[i][-2], lines[i][-1]])
+                if fact[0] == 1:
+                    down_error = down_error + 1
+                    down_right = down_right + 1.12
+                elif fact[1] == 1:
+                    down_right = down_right + 1.06
+                elif fact[2] == 1:
+                    down_right = down_right + 1
+                elif fact[3] == 1:
+                    down_right = down_right + 0.94
+                else:
+                    down_right = down_right + 0.88
+                down_num = down_num + 1
+
             i = i + 1
     if up_num == 0:
         up_num = 1
-    print('DNN', up_right, up_num, up_right/up_num)
-    return win_dnn,up_right/up_num
+    print('DNN', up_right, up_num, up_right/up_num, up_error/up_num, down_right/down_num, down_error/down_num)
+    return win_dnn,up_right/up_num,down_right/down_num
 
 
 def multi_predict():
-    r = 0
-    for x in [0, 2, 4, 5, 7]:
-        win_dnn, ratio = predict(file_path='D:\\data\\quantization\\kmeans\\stock7_14_' + str(x) + '_test.log', model_path='14d_dnn_seq_' + str(x) + '.h5')
-        r = r + ratio
-    print(r)
+    r = 0;
+    p = 0
+    # for x in range(0, 12): # 0,2,3,4,6,8,9,10,11
+    # for x in [5,6,11]:
+    for x in [2,4,7,10]: # 2表现最好 优秀的
+        print(x)
+    # for x in [0,2,5,6,7]: # 5表现最好
+        win_dnn, up_ratio,down_ratio = predict(file_path='D:\\data\\quantization\\kmeans\\stock9_18_test_' + str(x) + '.log', model_path='18d_dnn_seq_' + str(x) + '.h5')
+        r = r + up_ratio
+        p = p + down_ratio
+    print(r, p)
+
+import pymongo
+from util.mongodb import get_mongo_table_instance
+code_table = get_mongo_table_instance('tushare_code')
+k_table = get_mongo_table_instance('stock_day_k')
+
+industry = ['全国地产', '区域地产', '酒店餐饮',
+            '家用电器', '文教休闲', '元器件', 'IT设备', '汽车服务',
+            '汽车配件', '港口', '机场', '商贸代理', '软件服务', '证券',
+            '供气供热', '多元金融', '百货','食品', '水务',
+            '互联网', '纺织', '保险', '航空',  '超市连锁', '软饮料',
+            '塑料', '电器连锁', '半导体', '乳制品',]
+
+
+def predict_today(day):
+    lines = []
+    with open('D:\\data\\quantization\\stock9_18_' +  str(day) +'.log') as f:
+        for line in f.readlines()[:]:
+            line = eval(line.strip())
+            if line[-1][0].startswith('0') or line[-1][0].startswith('3'):
+                lines.append(line)
+
+    size = len(lines[0])
+    train_x=[s[:size - 1] for s in lines]
+    np.array(train_x)
+
+    estimator = joblib.load('km_dmi_18.pkl')
+
+    models = []
+    for x in range(0, 12):
+        models.append(load_model('18d_dnn_seq_' + str(x) + '.h5'))
+
+    x = 21 # 每条数据项数
+    k = 18 # 周期
+    for line in lines:
+        v = line[1:x*k + 1]
+        v = np.array(v)
+        v = v.reshape(k, x)
+        v = v[:,4:8]
+        v = v.reshape(1, 4*k)
+        # print(v)
+        r = estimator.predict(v)
+
+        if r[0] in [5,6,11]:
+            train_x = np.array([line[:size - 1]])
+
+            result = models[r[0]].predict(train_x)
+            if result[0][3] > 0.5 or result[0][4] > 0.5:
+                stock = code_table.find_one({'ts_code':line[-1][0]})
+                if stock['name'].startswith('ST') or stock['name'].startswith('N') or stock['name'].startswith('*'):
+                    continue
+                if line[0] > 80:
+                    continue
+                if stock['industry'] in industry:
+                    pass
+                    # print(line[-1], stock['name'], stock['industry'], 'sell')
+
+        if r[0] in [2,4,7,10]:
+            train_x = np.array([line[:size - 1]])
+
+            result = models[r[0]].predict(train_x)
+            # print(result, line[-1])
+            if result[0][0] > 0.5 or result[0][1] > 0.5:
+                if line[-1][0].startswith('688'):
+                    continue
+                # 去掉ST
+                stock = code_table.find_one({'ts_code':line[-1][0]})
+                if stock['name'].startswith('ST') or stock['name'].startswith('N') or stock['name'].startswith('*'):
+                    continue
+
+                # 跌的
+                k_table_list = list(k_table.find({'code':line[-1][0], 'tradeDate':{'$lte':20200214}}).sort("tradeDate", pymongo.DESCENDING).limit(5))
+                if k_table_list[0]['close'] > k_table_list[-1]['close']*1.20:
+                    continue
+                if k_table_list[0]['close'] < k_table_list[-1]['close']*0.90:
+                    continue
+                if k_table_list[-1]['close'] > 80:
+                    continue
+
+                # 指定某几个行业
+                # if stock['industry'] in industry:
+                print(line[-1], stock['name'], stock['industry'], 'buy')
 
 
 if __name__ == '__main__':
     # predict(file_path='D:\\data\\quantization\\stock6_5_test.log', model_path='5d_dnn_seq.h5')
     # predict(file_path='D:\\data\\quantization\\stock6_test.log', model_path='15m_dnn_seq.h5')
-    multi_predict()
+    multi_predict()
+    # predict_today(20200219)

stock/dnn_train.py

@@ -43,7 +43,7 @@ def read_data(path):
 def resample(path):
     lines = []
     with open(path) as f:
-        for x in range(160000):
+        for x in range(330000):
             lines.append(eval(f.readline().strip()))
     estimator = joblib.load('km.pkl')
 
@@ -56,7 +56,7 @@ def resample(path):
         v = v.reshape(1, 40)
         # print(v)
         r = estimator.predict(v)
-        with open('D:\\data\\quantization\\kmeans\\stock2_10_' + str(r[0]) + '.log', 'a') as f:
+        with open('D:\\data\\quantization\\kmeans\\stock8_14_train_' + str(r[0]) + '.log', 'a') as f:
             f.write(str(line) + '\n')
 
 
@@ -106,5 +106,5 @@ def train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stoc
 if __name__ == '__main__':
     # train(input_dim=176, result_class=5, file_path="D:\\data\\quantization\\stock6_5.log", model_name='5d_dnn_seq.h5')
     # train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stock6.log", model_name='15m_dnn_seq.h5')
-    # resample('D:\\data\\quantization\\stock6_5.log')
-    mul_train()
+    resample('D:\\data\\quantization\\stock8_14.log')
+    # mul_train()

stock/dnn_train_dmi.py

@@ -1,14 +1,11 @@
-import keras
 # -*- encoding:utf-8 -*-
 import numpy as np
 from keras.models import Sequential
 from keras.layers import Dense,Dropout
 import random
 from keras import regularizers
-from keras.models import load_model
 from imblearn.over_sampling import RandomOverSampler
 import joblib
-import tensorflow
 
 
 def read_data(path):
@@ -43,12 +40,19 @@ def read_data(path):
 def resample(path):
     lines = []
     with open(path) as f:
-        for x in range(76000):
+        i = 0
+        for x in range(110000):
+            # print(i)
             lines.append(eval(f.readline().strip()))
-    estimator = joblib.load('km_dmi.pkl')
+            i = i + 1
+    estimator = joblib.load('km_dmi_18.pkl')
+
+    file_list = []
+    for x in range(0, 12):
+        file_list.append(open('D:\\data\\quantization\\kmeans\\stock9_18_train_' + str(x) + '.log', 'a'))
 
     x = 21 # 每条数据项数
-    k = 14 # 周期
+    k = 18 # 周期
     for line in lines:
         v = line[1:x*k + 1]
         v = np.array(v)
@@ -57,18 +61,18 @@ def resample(path):
         v = v.reshape(1, 4*k)
         # print(v)
         r = estimator.predict(v)
-        with open('D:\\data\\quantization\\kmeans\\stock7_14_' + str(r[0]) + '_test.log', 'a') as f:
-            f.write(str(line) + '\n')
+        file_list[r[0]].write(str(line) + '\n')
 
 
 def mul_train():
-    # for x in range(0, 8):
-    for x in [0, 2, 4, 5, 7]:
-        score = train(input_dim=300, result_class=3, file_path="D:\\data\\quantization\\kmeans\\stock7_14_" + str(x) + ".log",
-              model_name='14d_dnn_seq_' + str(x) + '.h5')
+    # for x in range(0, 12):
+    for x in [11,0,1,3,8,9]:
+    # for x in [2,4,7,10]:
+        score = train(input_dim=384, result_class=5, file_path="D:\\data\\quantization\\kmeans\\stock9_18_train_" + str(x) + ".log",
+              model_name='18d_dnn_seq_' + str(x) + '.h5')
 
-        with open('D:\\data\\quantization\\kmeans\\stock7_14_' + str(x) + '_dmi.log', 'a') as f:
-            f.write(str(score[1]) + '\n')
+        with open('D:\\data\\quantization\\kmeans\\stock9_18_dmi.log', 'a') as f:
+            f.write(str(x) + ':' + str(score[1]) + '\n')
 
 
 def train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stock6.log", model_name=''):
@@ -76,29 +80,30 @@ def train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stoc
 
     model = Sequential()
     model.add(Dense(units=120+input_dim, input_dim=input_dim,  activation='relu'))
-    model.add(Dense(units=120+input_dim, activation='relu',kernel_regularizer=regularizers.l1(0.001)))
+    model.add(Dense(units=120+input_dim, activation='relu',kernel_regularizer=regularizers.l1(0.002)))
     model.add(Dropout(0.2))
     model.add(Dense(units=120+input_dim, activation='relu'))
+    model.add(Dense(units=120+input_dim, activation='relu'))
+    model.add(Dense(units=120+input_dim, activation='relu',kernel_regularizer=regularizers.l1(0.002)))
     model.add(Dropout(0.2))
     model.add(Dense(units=120 + input_dim, activation='relu'))
     model.add(Dropout(0.2))
     model.add(Dense(units=120+input_dim, activation='selu'))
-    model.add(Dropout(0.1))
-    # model.add(Dense(units=60+input_dim, activation='selu'))
-    # model.add(Dropout(0.2))
+    model.add(Dropout(0.2))
+    model.add(Dense(units=120+input_dim, activation='selu'))
     model.add(Dense(units=512, activation='relu'))
 
     model.add(Dense(units=result_class, activation='softmax'))
     model.compile(loss='categorical_crossentropy', optimizer="adam",metrics=['accuracy'])
 
     print("Starting training ")
-    # model.fit(train_x, train_y, batch_size=1024, epochs=400 + 4*int(len(train_x)/1000), shuffle=True)
-    model.fit(train_x, train_y, batch_size=2048, epochs=500 + 6*int(len(train_x)/700), shuffle=True)
+    model.fit(train_x, train_y, batch_size=4096, epochs=900 + 6*int(len(train_x)/600), shuffle=True)
     score = model.evaluate(test_x, test_y)
     print(score)
     print('Test score:', score[0])
     print('Test accuracy:', score[1])
 
+
     model.save(model_name)
 
     return score
@@ -110,7 +115,5 @@ def train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stoc
 
 
 if __name__ == '__main__':
-    # train(input_dim=176, result_class=5, file_path="D:\\data\\quantization\\stock6_5.log", model_name='5d_dnn_seq.h5')
-    # train(input_dim=400, result_class=3, file_path="D:\\data\\quantization\\stock6.log", model_name='15m_dnn_seq.h5')
-    # resample('D:\\data\\quantization\\stock7_14_test.log')
+    # resample('D:\\data\\quantization\\stock9_18_1.log')
     mul_train()

stock/kmeans.py

@@ -7,7 +7,7 @@ import joblib
 def read_data(path):
     lines = []
     with open(path) as f:
-        for x in range(100000):
+        for x in range(20000):
             line = eval(f.readline().strip())
             # if line[-1][0] == 1 or line[-1][1] == 1:
             lines.append(line)
@@ -15,27 +15,36 @@ def read_data(path):
     return lines
 
 
-length = 14  # 周期是多少
-j = 21
+length = 18  # 周期是多少
+j = 20
 def class_fic(file_path=''):
     lines = read_data(file_path)
     print('读取数据完毕')
     size = len(lines[0])
-    train_x = np.array([s[5:9] + s[j+5:j+9] + s[j*2+5:j*2+9] + s[j*3+5:j*3+9] + s[j*4+5:j*4+9] + s[j*5+5:j*5+9] + s[j*6+5:j*6+9]
-                        + s[j*7+5:j*7+9] + s[j*8+5:j*8+9] + s[j*9+5:j*9+9] + s[j*10+5:j*10+9] + s[j*11+5:j*11+9] + s[j*12+5:j*12+9] + s[j*13+5:j*13+9]for s in lines])
+    x_list = []
+    for s in lines:
+        tmp_list = []
+        for x in range(0, length):
+            tmp_list = tmp_list + s[x*j+5:x*j+9]
+        x_list.append(tmp_list)
+    train_x = np.array(x_list)
+    # train_x = np.array([s[5:9] + s[j+5:j+9] + s[j*2+5:j*2+9] + s[j*3+5:j*3+9] + s[j*4+5:j*4+9] + s[j*5+5:j*5+9] + s[j*6+5:j*6+9]
+    #                     + s[j*7+5:j*7+9] + s[j*8+5:j*8+9] + s[j*9+5:j*9+9] + s[j*10+5:j*10+9] + s[j*11+5:j*11+9] + s[j*12+5:j*12+9] + s[j*13+5:j*13+9]
+    #                     + s[j*14+5:j*14+9] + s[j*15+5:j*15+9] + s[j*9+5:j*9+9] + s[j*10+5:j*10+9] + s[j*11+5:j*11+9] + s[j*12+5:j*12+9] + s[j*13+5:j*13+9]
+    #                     for s in lines])
     # train_y = [s[size - 1] for s in lines]
     v_x = train_x.reshape(train_x.shape[0], 4*length)
     stock_list = [s[size - 2] for s in lines]
 
-    estimator = KMeans(n_clusters=8, random_state=19)
+    estimator = KMeans(n_clusters=12, random_state=129)
     estimator.fit(v_x)
     label_pred = estimator.labels_  # 获取聚类标签
     centroids = estimator.cluster_centers_
-    joblib.dump(estimator , 'km_dmi.pkl')
+    joblib.dump(estimator , 'km_dmi_18.pkl')
 
     print(estimator.predict(v_x[:10]))
 
-    estimator = joblib.load('km_dmi.pkl')
+    estimator = joblib.load('km_dmi_18.pkl')
     print(estimator.predict(v_x[10:20]))
     # annoy_sim(v_x)
     # print('save数据完毕')
@@ -107,4 +116,4 @@ def find_annoy(lines, stock_list):
 
 if __name__ == '__main__':
     # class_fic(file_path="D:\\data\\quantization\\stock2_10.log")
-    class_fic(file_path="D:\\data\\quantization\\stock7_5.log")
+    class_fic(file_path="D:\\data\\quantization\\stock9_18.log")

util/mongodb.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python
+# -*- coding:utf-8 -*-
+
+import pymongo
+import datetime
+import time
+import random
+import logging
+from util.config import config
+
+
+class MongoConfig(object):
+    HOST = config.get('mongodb', 'host')
+    USER = config.get('mongodb', 'user')
+    PORT = int(config.get('mongodb', 'port'))
+    PASSWORD = config.get('mongodb', 'password')
+    DB = config.get('mongodb', 'db')
+
+
+def _get_default_mongodb_instance():
+    mongo_client = pymongo.MongoClient(MongoConfig.HOST, MongoConfig.PORT)
+    db = mongo_client[MongoConfig.DB]
+    if MongoConfig.PASSWORD is not None and MongoConfig.PASSWORD != '':
+        db.authenticate(MongoConfig.USER, MongoConfig.PASSWORD)
+    return db
+
+
+def get_mongodb_tablenames():
+    db = _get_default_mongodb_instance()
+    return db.collection_names(include_system_collections=False)
+
+
+def get_mongo_table_instance(tablename):
+    db = _get_default_mongodb_instance()
+    return db[tablename]
+
+
+def del_mongodb_table(tablename):
+    try:
+        db = _get_default_mongodb_instance()
+        db.drop_collection(tablename)
+        return True
+    except:
+        return False
+
+
+def rename_mongodb_table(from_tablename, to_tablename):
+    try:
+        get_mongo_table_instance(from_tablename).rename(to_tablename)
+        return True
+    except:
+        return False
+
+
+def get_mongodb_table_indexes(tablename):
+    try:
+        ret = []
+
+        ftable = get_mongo_table_instance(tablename)
+        findex = ftable.list_indexes()
+        listindex = list(findex)
+        if listindex is None or len(listindex) == 0:
+            return ret
+
+        for i in listindex:
+            sonobj = i['key']
+            indexes = []
+
+            for k, v in sonobj.iteritems():
+                v1 = pymongo.ASCENDING if v > 0 else pymongo.DESCENDING
+                temp = (k, v1)
+                indexes.append(temp)
+
+            ret.append(indexes)
+
+        return ret
+    except Exception as e:
+        return None
+
+
+def set_mongodb_table_indexes(tablename, indexes):
+    try:
+        ftable = get_mongo_table_instance(tablename)
+
+        if indexes is None or len(indexes) == 0:
+            return True
+        for i in indexes:
+            ftable.ensure_index(i)
+
+        return True
+    except Exception as e:
+        return False
+
+
+def copy_mongodb_table_indexes(from_tablename, to_tablename):
+    indexes = get_mongodb_table_indexes(from_tablename)
+    set_mongodb_table_indexes(to_tablename, indexes)