产生测试数据和训练数据

bbztx/get_data.py

@@ -7,10 +7,10 @@ query_articles_sql = '''
 select a.aid,a.title,a.cities,a.cid,a.other_info,"array_agg"(t.tag_value),days,a.recom,a."rank",read_count
 from articles a
 LEFT JOIN article_tags t on a.aid = t.aid
-where a.crt_time > '2018-01-01' and a.atype = '0'
+where %s and a.atype = '0'
 and a.stock_aid is NULL
 GROUP BY a.aid
-limit 100 OFFSET 0
+limit 200 OFFSET 0
 '''
 
 query_sell_main_sql = '''
@@ -42,8 +42,8 @@ and status in (1,2)
 '''
 
 
-def get_articles():
-    rows = pgsql_util.get_rows(query_articles_sql)
+def get_articles(filter):
+    rows = pgsql_util.get_rows(query_articles_sql % (filter))
 
     attr_list = []
     for row in rows:
@@ -93,7 +93,7 @@ def to_a1(attr_list):
         dtype = attr['dtype']
         recruit = attr['recruit']
         country = attr['country']
-        price = attr['price']
+        price = attr['price'].replace('起', '')
         days = attr['days']
         recom = attr['recom']
         rank = attr['rank']
@@ -146,14 +146,25 @@ def to_list_attr(item, a_list):
     return c_list
 
 
-def to_file(data_list):
-    with open("train_data", "w") as f:
+def to_file(data_list, file_name):
+    with open(file_name, "w") as f:
         for line in data_list:
             line = [line[:-2], [line[-1]]]
             f.write(str(line) + "\n")
 
 
-if __name__ == '__main__':
-    attr_list = get_articles()
+def ge_train():
+    attr_list = get_articles("a.crt_time > '2018-01-01' ")
+    new_attr_list = to_a1(attr_list)
+    to_file(new_attr_list, "train_data")
+
+
+def ge_test():
+    attr_list = get_articles("a.crt_time > '2019-09-01' ")
     new_attr_list = to_a1(attr_list)
-    to_file(new_attr_list)
+    to_file(new_attr_list, "test_data")
+
+
+if __name__ == '__main__':
+    ge_train()
+    ge_test()