diff --git a/python/replace_domain.py b/python/replace_domain.py index a475687f8..c7ce8c564 100644 --- a/python/replace_domain.py +++ b/python/replace_domain.py @@ -1,13 +1,10 @@ -import pymysql -import uuid -import requests -from qiniu import Auth, put_file import uuid import os import pymysql import requests from qiniu import Auth, put_file +from concurrent.futures import ThreadPoolExecutor bucket_name = 'peko-prod' @@ -20,6 +17,7 @@ MYSQL_CONFIG = { "db": "peko" } +thread = None def connect(): return pymysql.connect( @@ -50,6 +48,14 @@ def update(sql): conn.close() +def async_parse(data, table_name): + # 启动线程执行 + try: + thread.submit(parse, data, table_name) + except Exception as e: + print("thread submit error.", e) + + def parse(data, table_name): if data is None or len(data) == 0: return @@ -103,32 +109,34 @@ def upload(filename): if __name__ == '__main__': - parse(select_list("select pic_url, vgg_url, lucky_Gift_Svga_Url, gift_explain_url, view_url from gift where (pic_url like '%zhongjialx%' or vgg_url like '%zhongjialx%' or lucky_Gift_Svga_Url like '%zhongjialx%' or gift_explain_url like '%zhongjialx%' or view_url like '%zhongjialx%');"), 'gift') - parse(select_list("select pic from info_card where (pic like '%zhongjialx%');"), 'info_card') - parse(select_list("select android_url, ios_url from chat_bubble where (android_url like '%zhongjialx%' or ios_url like '%zhongjialx%');"), 'chat_bubble') - parse(select_list("select icon_pic from nameplate where (icon_pic like '%zhongjialx%');"), 'nameplate') - parse(select_list("select nameplate_image from user_nameplate where (nameplate_image like '%zhongjialx%');"), 'user_nameplate') - parse(select_list("select pic, effect from headwear where (pic like '%zhongjialx%' or effect like '%zhongjialx%');"), 'headwear') - parse(select_list("select user_voice, avatar from users where (user_voice like '%zhongjialx%' or avatar like '%zhongjialx%');"), 'users') - parse(select_list("select game_picture, game_icon, tag_icon from game_info where (game_picture like '%zhongjialx%' or game_icon like '%zhongjialx%' or tag_icon like '%zhongjialx%');"), 'game_info') - parse(select_list("select prize_img_url from activity_award where (prize_img_url like '%zhongjialx%');"), 'activity_award') - parse(select_list("select pic, effect, redirect_link, view_url from car_goods where (pic like '%zhongjialx%' or effect like '%zhongjialx%' or redirect_link like '%zhongjialx%' or view_url like '%zhongjialx%');"), 'car_goods') - parse(select_list("select alert_win_pic, skip_url from charge_activity where (alert_win_pic like '%zhongjialx%' or skip_url like '%zhongjialx%');"), 'charge_activity') - parse(select_list("select prize_img_url from draw_lottery_record where (prize_img_url like '%zhongjialx%');"), 'draw_lottery_record') - parse(select_list("select icon from family where (icon like '%zhongjialx%');"), 'family') - parse(select_list("select icon from family_group_chat where (icon like '%zhongjialx%');"), 'family_group_chat') - parse(select_list("select magic_icon, magic_svg_url, effect_svg_url from gift_magic where (magic_icon like '%zhongjialx%' or magic_svg_url like '%zhongjialx%' or effect_svg_url like '%zhongjialx%');"), 'gift_magic') - parse(select_list("select icon from hall_group_chat where (icon like '%zhongjialx%');"), 'hall_group_chat') - parse(select_list("select url, url1 from level_charm where (url like '%zhongjialx%' or url1 like '%zhongjialx%');"), 'level_charm') - parse(select_list("select url, url1 from level_experience where (url like '%zhongjialx%' or url1 like '%zhongjialx%');"), 'level_experience') - parse(select_list("select value, preview from noble_res where (value like '%zhongjialx%' or preview like '%zhongjialx%');"), 'noble_res') - parse(select_list("select zip_url, res_conf from noble_zip where (zip_url like '%zhongjialx%' or res_conf like '%zhongjialx%');"), 'noble_zip') - parse(select_list("select act_image from operation_act where (act_image like '%zhongjialx%');"), 'operation_act') - parse(select_list("select prize_img_url from prize where (prize_img_url like '%zhongjialx%');"), 'prize') - parse(select_list("select avatar, voice from user_doll where (avatar like '%zhongjialx%' or voice like '%zhongjialx%');"), 'user_doll') - parse(select_list("select magic_icon from user_magic_wall where (magic_icon like '%zhongjialx%');"), 'user_magic_wall') - parse(select_list("select voice_url from user_voice where (voice_url like '%zhongjialx%');"), 'user_voice') - parse(select_list("select pic_url from anchor_grade where (pic_url like '%zhongjialx%');"), 'anchor_grade') - parse(select_list("select url from anchor_level_experience where (url like '%zhongjialx%');"), 'anchor_level_experience') - parse(select_list("select badge, cardbg, zonebg, room_background from noble_users where (badge like '%zhongjialx%' or cardbg like '%zhongjialx%' or zonebg like '%zhongjialx%' or room_background like '%zhongjialx%');"), 'noble_users') - parse(select_list("select tag_pict, badge, avatar, back_pic from room where (tag_pict like '%zhongjialx%' or badge like '%zhongjialx%' or avatar like '%zhongjialx%' or back_pic like '%zhongjialx%');"), 'room') + thread = ThreadPoolExecutor(max_workers=100) + async_parse(select_list("select pic_url, vgg_url, lucky_Gift_Svga_Url, gift_explain_url, view_url from gift where (pic_url like '%zhongjialx%' or vgg_url like '%zhongjialx%' or lucky_Gift_Svga_Url like '%zhongjialx%' or gift_explain_url like '%zhongjialx%' or view_url like '%zhongjialx%');"), 'gift') + async_parse(select_list("select pic from info_card where (pic like '%zhongjialx%');"), 'info_card') + async_parse(select_list("select android_url, ios_url from chat_bubble where (android_url like '%zhongjialx%' or ios_url like '%zhongjialx%');"), 'chat_bubble') + async_parse(select_list("select icon_pic from nameplate where (icon_pic like '%zhongjialx%');"), 'nameplate') + async_parse(select_list("select nameplate_image from user_nameplate where (nameplate_image like '%zhongjialx%');"), 'user_nameplate') + async_parse(select_list("select pic, effect from headwear where (pic like '%zhongjialx%' or effect like '%zhongjialx%');"), 'headwear') + async_parse(select_list("select user_voice, avatar from users where (user_voice like '%zhongjialx%' or avatar like '%zhongjialx%');"), 'users') + async_parse(select_list("select game_picture, game_icon, tag_icon from game_info where (game_picture like '%zhongjialx%' or game_icon like '%zhongjialx%' or tag_icon like '%zhongjialx%');"), 'game_info') + async_parse(select_list("select prize_img_url from activity_award where (prize_img_url like '%zhongjialx%');"), 'activity_award') + async_parse(select_list("select pic, effect, redirect_link, view_url from car_goods where (pic like '%zhongjialx%' or effect like '%zhongjialx%' or redirect_link like '%zhongjialx%' or view_url like '%zhongjialx%');"), 'car_goods') + async_parse(select_list("select alert_win_pic, skip_url from charge_activity where (alert_win_pic like '%zhongjialx%' or skip_url like '%zhongjialx%');"), 'charge_activity') + async_parse(select_list("select prize_img_url from draw_lottery_record where (prize_img_url like '%zhongjialx%');"), 'draw_lottery_record') + async_parse(select_list("select icon from family where (icon like '%zhongjialx%');"), 'family') + async_parse(select_list("select icon from family_group_chat where (icon like '%zhongjialx%');"), 'family_group_chat') + async_parse(select_list("select magic_icon, magic_svg_url, effect_svg_url from gift_magic where (magic_icon like '%zhongjialx%' or magic_svg_url like '%zhongjialx%' or effect_svg_url like '%zhongjialx%');"), 'gift_magic') + async_parse(select_list("select icon from hall_group_chat where (icon like '%zhongjialx%');"), 'hall_group_chat') + async_parse(select_list("select url, url1 from level_charm where (url like '%zhongjialx%' or url1 like '%zhongjialx%');"), 'level_charm') + async_parse(select_list("select url, url1 from level_experience where (url like '%zhongjialx%' or url1 like '%zhongjialx%');"), 'level_experience') + async_parse(select_list("select value, preview from noble_res where (value like '%zhongjialx%' or preview like '%zhongjialx%');"), 'noble_res') + async_parse(select_list("select zip_url, res_conf from noble_zip where (zip_url like '%zhongjialx%' or res_conf like '%zhongjialx%');"), 'noble_zip') + async_parse(select_list("select act_image from operation_act where (act_image like '%zhongjialx%');"), 'operation_act') + async_parse(select_list("select prize_img_url from prize where (prize_img_url like '%zhongjialx%');"), 'prize') + async_parse(select_list("select avatar, voice from user_doll where (avatar like '%zhongjialx%' or voice like '%zhongjialx%');"), 'user_doll') + async_parse(select_list("select magic_icon from user_magic_wall where (magic_icon like '%zhongjialx%');"), 'user_magic_wall') + async_parse(select_list("select voice_url from user_voice where (voice_url like '%zhongjialx%');"), 'user_voice') + async_parse(select_list("select pic_url from anchor_grade where (pic_url like '%zhongjialx%');"), 'anchor_grade') + async_parse(select_list("select url from anchor_level_experience where (url like '%zhongjialx%');"), 'anchor_level_experience') + async_parse(select_list("select badge, cardbg, zonebg, room_background from noble_users where (badge like '%zhongjialx%' or cardbg like '%zhongjialx%' or zonebg like '%zhongjialx%' or room_background like '%zhongjialx%');"), 'noble_users') + async_parse(select_list("select tag_pict, badge, avatar, back_pic from room where (tag_pict like '%zhongjialx%' or badge like '%zhongjialx%' or avatar like '%zhongjialx%' or back_pic like '%zhongjialx%');"), 'room') + thread.shutdown()