from LoggerUtils import Logger, initLogger from bs4 import BeautifulSoup as bs from urllib.request import urlopen, Request import json import Contant from sqlalchemy import create_engine from entity.ChannelEntity import Channel from service.ChannelService import ChannelService import sqlite3 import pandas as pd def moveChannel(sqliteDir): # 读取excel file_path = './channel_region.xlsx' df = pd.read_excel(file_path) # 从sqlite去读所有内容 conn = sqlite3.connect(sqliteDir) cursor = conn.cursor() cursor.execute("SELECT * from Channel_copy where is_copy = 0 limit 50") channel_list = cursor.fetchall() while len(channel_list) > 0: for channel in channel_list: Logger.info(channel) channelId = channel[1] channeTitle = channel[2] language = channel[3] repiteTime = channel[4] flag = (df['channelID'] == channelId).any() if flag: region = df.loc[df['channelID'] == channelId]['region'].iloc[0] else: region = "unKnown" ChannelService.insertOneByValues( channelId, channeTitle, language, region, repiteTime) cursor.execute(f"UPDATE Channel_copy SET is_copy = 1 WHERE channelId = '{channelId}'") conn.commit() cursor.execute("SELECT * from Channel_copy where is_copy = 0 limit 50") channel_list = cursor.fetchall() cursor.close() conn.close() # py .\init.py --db=../db/youtube_prod.db --logDir=./logs if __name__ == "__main__": # 读取配置文件 with open('move_data_config.json', 'r', encoding='utf-8') as f: # 使用json.load()方法读取文件内容 data = json.load(f) # 初始化日志 Contant.logDir = data['log']['dir'] Contant.logFileName = data['log']['fileName'] initLogger(Contant.logDir, Contant.logFileName) # 连接mysql dbHost = data['mysql']['host'] dbPort = data['mysql']['port'] dbUserName = data['mysql']['username'] dbPassword = data['mysql']['password'] dbDatabase = data['mysql']['database'] Logger.info("尝试连接mysql host:'{}' port:'{}' username:'{}' password:'{}' database:'{}'", dbHost, dbPort, dbUserName, dbPassword, dbDatabase) Contant.engin = create_engine( f'mysql+mysqlconnector://{dbUserName}:{dbPassword}@{dbHost}:{dbPort}/{dbDatabase}') Logger.info("连接mysql成功") moveTable = data['sqlite']['table'] sqliteDir = data['sqlite']['dir'] Logger.info("move data table:{}".format(moveTable)) if moveTable == 'Channel': moveChannel(sqliteDir)