py_script/task_scrapy/scrapy_all.py

#/usr/bin/env python
#coding=utf-8

import random
import sys
import time

from selenium import webdriver
from urllib import quote

from fty_util.common import Mysql

reload(sys)
sys.setdefaultencoding('utf8')

def parse_playtimes():
    conn = Mysql.createOfflineConn()

    sql = """
        select id, tv_name, url, playtimes from scrapy.wangju_all_url order by id asc
    """

    rows = Mysql.getAll(sql, conn=conn)

    for row in rows:
        _id = row['id']
        tv_name = row['tv_name']
        url = row['url']
        playtimes = row['playtimes']

        if playtimes is not None and len(playtimes.split('*')) == 2:
            first_num, second_num = playtimes.split('*')
            first_num = float(first_num)
            second_num = int(second_num)

            playtimes_new = first_num * second_num
            sql = """
                update scrapy.wangju_all_url set playtimes = '%s' where url = '%s'
            """
            sql = sql % (str(int(playtimes_new)), url)
            Mysql.execute(sql, conn=conn)
            
def update_fields():
    conn = Mysql.createOfflineConn()

    sql = """
        select id, tv_name, score, playtimes, source from scrapy.wangju_all_url order by id asc
    """

    rows = Mysql.getAll(sql, conn=conn)

    for row in rows:
        _id = row['id']
        tv_name = row['tv_name']
        score = row['score']
        playtimes = row['playtimes']

        source = row['source']
        if 'pptv' == source:
            sql = """
                update scrapy.wangju_url set pptv_score = '%s', pptv_playtimes = '%s' where id = %s
            """

        if 'youku' == source:
            sql = """
                update scrapy.wangju_url set youku_score = '%s', youku_playtimes = '%s' where id = %s
            """

        if 'sohu' == source:
            sql = """
                update scrapy.wangju_url set sohu_score = '%s', sohu_playtimes = '%s' where id = %s
            """

        if 'leshi' == source:
            sql = """
                update scrapy.wangju_url set leshi_score = '%s', leshi_playtimes = '%s' where id = %s
            """

        if 'huashutv' == source:
            sql = """
                update scrapy.wangju_url set huashutv_score = '%s', huashutv_playtimes = '%s' where id = %s
            """

        if 'iqiyi' == source:
            sql = """
                update scrapy.wangju_url set iqiyi_score = '%s', iqiyi_playtimes = '%s' where id = %s
            """

        if 'tengxun' == source:
            sql = """
                update scrapy.wangju_url set tengxun_score = '%s', tengxun_playtimes = '%s' where id = %s
            """
        sql = sql % (score, playtimes, _id)
        Mysql.execute(sql, conn=conn)
        

        
if __name__ == '__main__':
    # parse_playtimes()
    update_fields()