9 changed files with 383 additions and 3 deletions
@ -0,0 +1,173 @@ |
|||||
|
import httplib2 |
||||
|
import googleapiclient.discovery |
||||
|
import googleapiclient.errors |
||||
|
from LoggerUtils import Logger |
||||
|
import operator |
||||
|
import time |
||||
|
from entity.ChannelEntity import Channel |
||||
|
from entity.VideoEntity import Video |
||||
|
from service.ChannelService import ChannelService |
||||
|
from service.VideoService import VideoService |
||||
|
|
||||
|
|
||||
|
class YouTubeUtil: |
||||
|
# apiKeys = ["AIzaSyDlRgmPXVQEjF2gbmomI5FUZX_uAOBmEGI", "AIzaSyBI5i5vFZpQErMnEXKMf0VUS2Bel8jGrTk", |
||||
|
# "AIzaSyAnmA0Ggy1yXsZZACfItmeZAa7wcmh6SbM", "AIzaSyC4O8tBoAfkupmBybxDah2JUxgj4ct5uk0", |
||||
|
# "AIzaSyDJ2S9Ijhw_hULx3nHvPUoGUpMENbZOIl8", "AIzaSyA87Ckpna3hOQ31nISs8V8rp--OLw0m6Aw", |
||||
|
# "AIzaSyDIWbV0EOLHkOr9tWpANose6ggd2r9vcLg", "AIzaSyBKE3lYwWFIYc9Vx4YKMbRpkOXigZlY52U"] |
||||
|
|
||||
|
# AIzaSyCTBSbq0YjyxTtjmNsnDyKAwHamlv_ST-s |
||||
|
# AIzaSyAESnwtbTIBtU707iZowtQkmAo-qKuEOcY |
||||
|
# AIzaSyCsYUC5vN0pB6y9xsCj0B1ehAoqOJ3WMf0 |
||||
|
# AIzaSyDjPkCgDQ9Tv_xcChjY2E6GpJ6IzngnD5I |
||||
|
# AIzaSyAxIycOdQYGB5kWhwe3B-kJAYRo7wOnp8o |
||||
|
apiKeys = [ |
||||
|
"AIzaSyARaW3mqO9szQiHgWZR4el0HWvdyheSHBc", |
||||
|
"AIzaSyChPXesnVx6fweon_BckhR6UiJWvi5Ma4s" |
||||
|
|
||||
|
# "AIzaSyCTBSbq0YjyxTtjmNsnDyKAwHamlv_ST-s", |
||||
|
# "AIzaSyAESnwtbTIBtU707iZowtQkmAo-qKuEOcY" |
||||
|
|
||||
|
|
||||
|
# "AIzaSyDjPkCgDQ9Tv_xcChjY2E6GpJ6IzngnD5I", |
||||
|
# "AIzaSyAxIycOdQYGB5kWhwe3B-kJAYRo7wOnp8o", |
||||
|
# "AIzaSyCsYUC5vN0pB6y9xsCj0B1ehAoqOJ3WMf0" |
||||
|
] |
||||
|
apiIndex = 0 |
||||
|
|
||||
|
# 获取youtube对象 |
||||
|
def getYoutube(): |
||||
|
# 本地测试使用代码 |
||||
|
# proxy_info = httplib2.ProxyInfo( |
||||
|
# proxy_type=httplib2.socks.PROXY_TYPE_HTTP, proxy_host="127.0.0.1", proxy_port=7890) |
||||
|
# http = httplib2.Http(timeout=10, proxy_info=proxy_info, |
||||
|
# disable_ssl_certificate_validation=False) |
||||
|
http = httplib2.Http( |
||||
|
timeout=10, disable_ssl_certificate_validation=False) |
||||
|
api_service_name = "youtube" |
||||
|
api_version = "v3" |
||||
|
# 获取apiKey |
||||
|
apiKey = YouTubeUtil.apiKeys[YouTubeUtil.apiIndex] |
||||
|
Logger.info( |
||||
|
"当前APIKey:{},当前apiIndex:{},totalIndex:{}".format( |
||||
|
apiKey, YouTubeUtil.apiIndex, len(YouTubeUtil.apiKeys) - 1 |
||||
|
) |
||||
|
) |
||||
|
# 等于7，还原成0 |
||||
|
if YouTubeUtil.apiIndex == (len(YouTubeUtil.apiKeys) - 1): |
||||
|
YouTubeUtil.apiIndex = 0 |
||||
|
else: |
||||
|
YouTubeUtil.apiIndex = YouTubeUtil.apiIndex + 1 |
||||
|
|
||||
|
# 获取对象 |
||||
|
youtube = googleapiclient.discovery.build( |
||||
|
api_service_name, api_version, developerKey=apiKey, http=http |
||||
|
) |
||||
|
return youtube |
||||
|
|
||||
|
def getVidoeLen(videoIds): |
||||
|
youtube = YouTubeUtil.getYoutube() |
||||
|
request = youtube.videos().list(part="contentDetails", id=videoIds) |
||||
|
response = request.execute() |
||||
|
response["items"][0]["contentDetails"] |
||||
|
return response |
||||
|
|
||||
|
def getVideoLenByStr(str): |
||||
|
len = 0 |
||||
|
str = str.split("PT")[1] |
||||
|
if operator.contains(str, "H"): |
||||
|
H = str.split("H")[0] |
||||
|
len = len + int(H) * 3600 |
||||
|
str = str.split("H")[1] |
||||
|
if operator.contains(str, "M"): |
||||
|
M = str.split("M")[0] |
||||
|
len = len + int(M) * 60 |
||||
|
str = str.split("M")[1] |
||||
|
if operator.contains(str, "S"): |
||||
|
S = str.split("S")[0] |
||||
|
len = len + int(S) |
||||
|
return len |
||||
|
|
||||
|
def getByChannelId(channelId, startTime, endTime): |
||||
|
channel = ChannelService.queryOneByChannelId(channelId) |
||||
|
# 检查是否存在 |
||||
|
if channel is None: |
||||
|
Logger.info("没有相应的频道 channelId:{}".format(channelId)) |
||||
|
return |
||||
|
# 回去youtube并查询 |
||||
|
videoLanguage = str(channel.channelLanguage) |
||||
|
youtube = YouTubeUtil.getYoutube() |
||||
|
request = youtube.search().list( |
||||
|
part="snippet", |
||||
|
channelId=channelId, |
||||
|
maxResults=50, |
||||
|
order="date", |
||||
|
publishedAfter=startTime, |
||||
|
publishedBefore=endTime, |
||||
|
type="video", |
||||
|
) |
||||
|
response = request.execute() |
||||
|
while True: |
||||
|
videosRequest = "" |
||||
|
videosRequestCount = 0 |
||||
|
idList = [] |
||||
|
for i in response["items"]: |
||||
|
try: |
||||
|
videoId = i["id"]["videoId"] |
||||
|
publisTime = i["snippet"]["publishedAt"] |
||||
|
videoTitle = i["snippet"]["title"] |
||||
|
videoType = "video" |
||||
|
# 查询是否存在Video，如果不存在就插入 |
||||
|
video: Video = VideoService.queryOneByVideoId(videoId) |
||||
|
if video == None: |
||||
|
VideoService.insertOne( |
||||
|
videoId=videoId, ChannelId=channelId, videoTitle=videoTitle, videoLen=0, |
||||
|
videoType=videoType, videoPublishTime=publisTime, videoLanguage=videoLanguage, isDownload=0) |
||||
|
videosRequest = videosRequest + "," + str(videoId) |
||||
|
videosRequestCount = videosRequestCount + 1 |
||||
|
Logger.info( |
||||
|
"存储VideoUrl:https://www.youtube.com/watch?v=" + videoId |
||||
|
) |
||||
|
else: |
||||
|
Logger.info("已存在VideoId:{}".format(videoId)) |
||||
|
idList.append(str(videoId)) |
||||
|
if videosRequest != "" and videosRequestCount >= 10: |
||||
|
lenRes = YouTubeUtil.getVidoeLen(videosRequest) |
||||
|
for i in lenRes["items"]: |
||||
|
tmpId = i["id"] |
||||
|
videoLenStr = i["contentDetails"]["duration"] |
||||
|
videoLen = YouTubeUtil.getVideoLenByStr( |
||||
|
videoLenStr) |
||||
|
VideoService.updateLenByVideoId( |
||||
|
videoId=tmpId, videoLen=videoLen) |
||||
|
Logger.info( |
||||
|
"更新时长,videoId:{},len:{}".format( |
||||
|
tmpId, videoLen) |
||||
|
) |
||||
|
videosRequestCount = 0 |
||||
|
videosRequest = "" |
||||
|
except: |
||||
|
pass |
||||
|
# 获取最后一个视频 |
||||
|
video: Video = VideoService.getLastVideoByChannelId(channelId) |
||||
|
ChannelService.updateTimeByChannelId( |
||||
|
channelId, video.videoPublishTime) |
||||
|
time.sleep(5) |
||||
|
# 继续获取下一页 |
||||
|
try: |
||||
|
# youtube = YouTubeUtil.getYoutube |
||||
|
request = youtube.search().list( |
||||
|
part="snippet", |
||||
|
channelId=channelId, |
||||
|
maxResults=50, |
||||
|
order="date", |
||||
|
publishedAfter=startTime, |
||||
|
publishedBefore=endTime, |
||||
|
type="video", |
||||
|
pageToken=response["nextPageToken"], |
||||
|
) |
||||
|
response = request.execute() |
||||
|
except Exception as e: |
||||
|
Logger.error(e) |
||||
|
print("no nextPageToken") |
||||
|
break |
||||
@ -0,0 +1,19 @@ |
|||||
|
from sqlalchemy import create_engine, Column, Integer, String, Boolean |
||||
|
from sqlalchemy.ext.declarative import declarative_base |
||||
|
|
||||
|
# 如果没有创建 Base，请取消注释下一行 |
||||
|
Base = declarative_base() |
||||
|
|
||||
|
|
||||
|
class Video(Base): |
||||
|
__tablename__ = 'Videos' |
||||
|
|
||||
|
id = Column(Integer, primary_key=True, autoincrement=True) |
||||
|
videoId = Column(String(255), nullable=False) |
||||
|
channelId = Column(String(255), nullable=False) |
||||
|
videoTitle = Column(String(255), nullable=False) |
||||
|
videoLen = Column(Integer, nullable=False) |
||||
|
videoType = Column(String(255), nullable=False) |
||||
|
videoPublishTime = Column(String(255), nullable=False) |
||||
|
videoLanguage = Column(String(255), nullable=False) |
||||
|
isDownload = Column(Integer, nullable=False) |
||||
@ -0,0 +1,50 @@ |
|||||
|
from LoggerUtils import Logger, initLogger |
||||
|
from bs4 import BeautifulSoup as bs |
||||
|
from urllib.request import urlopen, Request |
||||
|
import json |
||||
|
import Contant |
||||
|
from sqlalchemy import create_engine |
||||
|
from entity.ChannelEntity import Channel |
||||
|
from service.ChannelService import ChannelService |
||||
|
from common.YoutubeUtils import YouTubeUtil |
||||
|
import operator |
||||
|
import argparse |
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
# 读取参数 |
||||
|
parser = argparse.ArgumentParser(description="") |
||||
|
parser.add_argument("--start", type=str, default="") |
||||
|
parser.add_argument("--end", type=str, default="") |
||||
|
args = parser.parse_args() |
||||
|
startTime = args.start |
||||
|
endTime = args.end |
||||
|
# 读取配置文件 |
||||
|
with open('search_video_config.json', 'r', encoding='utf-8') as f: |
||||
|
# 使用json.load()方法读取文件内容 |
||||
|
data = json.load(f) |
||||
|
|
||||
|
# 初始化日志 |
||||
|
Contant.logDir = data['log']['dir'] |
||||
|
Contant.logFileName = data['log']['fileName'] |
||||
|
initLogger(Contant.logDir, Contant.logFileName) |
||||
|
|
||||
|
# 连接mysql |
||||
|
dbHost = data['mysql']['host'] |
||||
|
dbPort = data['mysql']['port'] |
||||
|
dbUserName = data['mysql']['username'] |
||||
|
dbPassword = data['mysql']['password'] |
||||
|
dbDatabase = data['mysql']['database'] |
||||
|
Logger.info("尝试连接mysql host:'{}' port:'{}' username:'{}' password:'{}' database:'{}'", |
||||
|
dbHost, dbPort, dbUserName, dbPassword, dbDatabase) |
||||
|
Contant.engin = create_engine( |
||||
|
f'mysql+mysqlconnector://{dbUserName}:{dbPassword}@{dbHost}:{dbPort}/{dbDatabase}') |
||||
|
Logger.info("连接mysql成功") |
||||
|
|
||||
|
YouTubeUtil.getByChannelId("channel.channelId", startTime,endTime) |
||||
|
# 查询出所有Channel |
||||
|
# channels = ChannelService.queryAllChannel() |
||||
|
# Logger.info("Channels length:{}".format(len(channels))) |
||||
|
# for channel in channels: |
||||
|
# channel : Channel = channel |
||||
|
# # 通过channelId查询videos |
||||
|
# YouTubeUtil.getByChannelId(channel.channelId) |
||||
@ -0,0 +1,13 @@ |
|||||
|
{ |
||||
|
"mysql": { |
||||
|
"host": "47.108.20.249", |
||||
|
"port": "3306", |
||||
|
"username": "root", |
||||
|
"password": "casino888!", |
||||
|
"database": "youtube" |
||||
|
}, |
||||
|
"log": { |
||||
|
"dir": "./logs", |
||||
|
"fileName": "search_video" |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
from entity.VideoEntity import Video |
||||
|
from common.Utils import getSession |
||||
|
from sqlalchemy import update |
||||
|
|
||||
|
|
||||
|
class VideoService: |
||||
|
|
||||
|
def queryOneByVideoId(videoId): |
||||
|
session = getSession() |
||||
|
video = session.query(Video).filter( |
||||
|
Video.videoId == videoId).one_or_none() |
||||
|
session.close() |
||||
|
return video |
||||
|
|
||||
|
def insertOne(videoId, ChannelId, videoTitle, videoLen, videoType, videoPublishTime, videoLanguage, isDownload): |
||||
|
session = getSession() |
||||
|
video: Video = Video(videoId=videoId, ChannelId=ChannelId, videoTitle=videoTitle, |
||||
|
videoLen=videoLen, videoType=videoType, videoPublishTime=videoPublishTime, |
||||
|
videoLanguage=videoLanguage, isDownload=isDownload) |
||||
|
session.add(video) |
||||
|
session.commit() |
||||
|
session.close() |
||||
|
|
||||
|
def updateLenByVideoId(videoId, videoLen): |
||||
|
session = getSession() |
||||
|
updateSql = update(Video).where( |
||||
|
Video.videoId == videoId).values(videoLen=videoLen) |
||||
|
resutl = session.execute(updateSql) |
||||
|
session.commit() |
||||
|
session.close() |
||||
|
|
||||
|
def getLastVideoByChannelId(channelId): |
||||
|
session = getSession() |
||||
|
video:Video = session.query(Video).where(Video.channelId==channelId).order_by(Video.videoPublishTime.desc()).first() |
||||
|
session.close() |
||||
|
return video |
||||
@ -0,0 +1,46 @@ |
|||||
|
from LoggerUtils import Logger, initLogger |
||||
|
from bs4 import BeautifulSoup as bs |
||||
|
from urllib.request import urlopen, Request |
||||
|
import json |
||||
|
import Contant |
||||
|
from sqlalchemy import create_engine |
||||
|
from entity.ChannelEntity import Channel |
||||
|
from entity.VideoEntity import Video |
||||
|
from service.ChannelService import ChannelService |
||||
|
from service.VideoService import VideoService |
||||
|
from common.YoutubeUtils import YouTubeUtil |
||||
|
import operator |
||||
|
import argparse |
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
# 读取配置文件 |
||||
|
with open('test_config.json', 'r', encoding='utf-8') as f: |
||||
|
# 使用json.load()方法读取文件内容 |
||||
|
data = json.load(f) |
||||
|
|
||||
|
# 初始化日志 |
||||
|
Contant.logDir = data['log']['dir'] |
||||
|
Contant.logFileName = data['log']['fileName'] |
||||
|
initLogger(Contant.logDir, Contant.logFileName) |
||||
|
|
||||
|
# 连接mysql |
||||
|
dbHost = data['mysql']['host'] |
||||
|
dbPort = data['mysql']['port'] |
||||
|
dbUserName = data['mysql']['username'] |
||||
|
dbPassword = data['mysql']['password'] |
||||
|
dbDatabase = data['mysql']['database'] |
||||
|
Logger.info("尝试连接mysql host:'{}' port:'{}' username:'{}' password:'{}' database:'{}'", |
||||
|
dbHost, dbPort, dbUserName, dbPassword, dbDatabase) |
||||
|
Contant.engin = create_engine( |
||||
|
f'mysql+mysqlconnector://{dbUserName}:{dbPassword}@{dbHost}:{dbPort}/{dbDatabase}') |
||||
|
Logger.info("连接mysql成功") |
||||
|
|
||||
|
videoId = "oZhBWA3HNhA" |
||||
|
video = VideoService.queryOneByVideoId(videoId) |
||||
|
Logger.info(video) |
||||
|
# VideoService.updateLenByVideoId(videoId, 5344) |
||||
|
video = VideoService.getLastVideoByChannelId("UC67Wr_9pA4I0glIxDt_Cpyw") |
||||
|
if video == None: |
||||
|
Logger.info("meiyou") |
||||
|
else: |
||||
|
Logger.info(video.videoPublishTime) |
||||
@ -0,0 +1,13 @@ |
|||||
|
{ |
||||
|
"mysql": { |
||||
|
"host": "47.108.20.249", |
||||
|
"port": "3306", |
||||
|
"username": "root", |
||||
|
"password": "casino888!", |
||||
|
"database": "youtube" |
||||
|
}, |
||||
|
"log": { |
||||
|
"dir": "./logs", |
||||
|
"fileName": "test" |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue