Browse Source

修复search和download相关bug

master
appolli 5 months ago
parent
commit
09a2292968
  1. 18
      common/DownloadUtils.py
  2. 24
      common/YoutubeUtils.py
  3. 1
      search_video.py
  4. 6
      service/SrtFileService.py
  5. 4
      service/VideoService.py
  6. 27
      test.py

18
common/DownloadUtils.py

@ -30,17 +30,24 @@ class DownloadUtil:
Logger.info("VideoId: {} 已收录", videoId) Logger.info("VideoId: {} 已收录", videoId)
return return
subs = pysrt.open(srtFilePath) subs = pysrt.open(srtFilePath)
srtFiles = []
ordinal = 1 ordinal = 1
for sub in subs: for sub in subs:
srtStartTime = str(sub.start.to_time()).rstrip("0") srtStartTime = str(sub.start.to_time()).rstrip("0")
if ordinal == 1:
srtStartTime = sub.start.to_time()
srtEndTime = str(sub.end.to_time()).rstrip("0") srtEndTime = str(sub.end.to_time()).rstrip("0")
SrtFileService.insertOne(videoId=videoId, channelId=channelId, ordinal=ordinal, srtFile: Srtfile = Srtfile(videoId=videoId, channelId=channelId, ordinal=ordinal,
srtStartTime=srtStartTime, srtEndTime=srtEndTime, srtText=sub.text, isScan=0) srtStartTime=srtStartTime, srtEndTime=srtEndTime, srtText=sub.text, isScan=0)
ordinal = ordinal + 1 ordinal = ordinal + 1
srtFiles.append(srtFile)
# 批量插入字幕数据
SrtFileService.insertList(srtFiles=srtFiles)
Logger.info(
f"读取srt文件成功 videoId:{videoId} channelId:{channelId} srtFilePath:{srtFilePath}")
def downLoadMP3(videoId, storePath): def downLoadMP3(videoId, storePath):
video:Video = VideoService.getOneByVideoId(videoId) video: Video = VideoService.queryOneByVideoId(videoId)
channel:Channel = ChannelService.queryOneByChannelId(video.channelId)
videoUrl = "https://www.youtube.com/watch?v={}".format(videoId) videoUrl = "https://www.youtube.com/watch?v={}".format(videoId)
yt = YouTube(videoUrl, on_progress_callback=on_progress) yt = YouTube(videoUrl, on_progress_callback=on_progress)
ys = yt.streams.get_audio_only() ys = yt.streams.get_audio_only()
@ -48,7 +55,7 @@ class DownloadUtil:
if not os.path.exists(mp3OutPutPath): if not os.path.exists(mp3OutPutPath):
Logger.info("开始创建文件夹:" + mp3OutPutPath) Logger.info("开始创建文件夹:" + mp3OutPutPath)
os.makedirs(mp3OutPutPath) os.makedirs(mp3OutPutPath)
fileName = "{}.mp3".format(videoId) fileName = "{}".format(videoId)
ys.download(output_path=mp3OutPutPath, filename=fileName, mp3=True) ys.download(output_path=mp3OutPutPath, filename=fileName, mp3=True)
@func_set_timeout(60) @func_set_timeout(60)
@ -98,8 +105,7 @@ class DownloadUtil:
videoId=videoId) videoId=videoId)
if downloadInfo is not None: if downloadInfo is not None:
DownloadInfoService.updateIsFinishByVideoId(videoId, 1, 1) DownloadInfoService.updateIsFinishByVideoId(videoId, 1, 1)
DownloadUtil.iterateSrt(storePath, videoId, video.channelId) DownloadUtil.iterateSrt(storePathSrt, videoId, video.channelId)
pass
except Exception as e: except Exception as e:
Logger.error(e) Logger.error(e)
logStr = "Exception...{}".format(e) logStr = "Exception...{}".format(e)

24
common/YoutubeUtils.py

@ -22,10 +22,10 @@ class YouTubeUtil:
# AIzaSyDjPkCgDQ9Tv_xcChjY2E6GpJ6IzngnD5I # AIzaSyDjPkCgDQ9Tv_xcChjY2E6GpJ6IzngnD5I
# AIzaSyAxIycOdQYGB5kWhwe3B-kJAYRo7wOnp8o # AIzaSyAxIycOdQYGB5kWhwe3B-kJAYRo7wOnp8o
apiKeys = [ apiKeys = [
"AIzaSyARaW3mqO9szQiHgWZR4el0HWvdyheSHBc", # "AIzaSyARaW3mqO9szQiHgWZR4el0HWvdyheSHBc",
"AIzaSyChPXesnVx6fweon_BckhR6UiJWvi5Ma4s" # "AIzaSyChPXesnVx6fweon_BckhR6UiJWvi5Ma4s"
# "AIzaSyCTBSbq0YjyxTtjmNsnDyKAwHamlv_ST-s", "AIzaSyCTBSbq0YjyxTtjmNsnDyKAwHamlv_ST-s"
# "AIzaSyAESnwtbTIBtU707iZowtQkmAo-qKuEOcY" # "AIzaSyAESnwtbTIBtU707iZowtQkmAo-qKuEOcY"
@ -38,12 +38,12 @@ class YouTubeUtil:
# 获取youtube对象 # 获取youtube对象
def getYoutube(): def getYoutube():
# 本地测试使用代码 # 本地测试使用代码
# proxy_info = httplib2.ProxyInfo( proxy_info = httplib2.ProxyInfo(
# proxy_type=httplib2.socks.PROXY_TYPE_HTTP, proxy_host="127.0.0.1", proxy_port=7890) proxy_type=httplib2.socks.PROXY_TYPE_HTTP, proxy_host="127.0.0.1", proxy_port=7890)
# http = httplib2.Http(timeout=10, proxy_info=proxy_info, http = httplib2.Http(timeout=10, proxy_info=proxy_info,
# disable_ssl_certificate_validation=False) disable_ssl_certificate_validation=False)
http = httplib2.Http( # http = httplib2.Http(
timeout=10, disable_ssl_certificate_validation=False) # timeout=10, disable_ssl_certificate_validation=False)
api_service_name = "youtube" api_service_name = "youtube"
api_version = "v3" api_version = "v3"
# 获取apiKey # 获取apiKey
@ -121,7 +121,7 @@ class YouTubeUtil:
video: Video = VideoService.queryOneByVideoId(videoId) video: Video = VideoService.queryOneByVideoId(videoId)
if video == None: if video == None:
VideoService.insertOne( VideoService.insertOne(
videoId=videoId, ChannelId=channelId, videoTitle=videoTitle, videoLen=0, videoId=videoId, channelId=channelId, videoTitle=videoTitle, videoLen=0,
videoType=videoType, videoPublishTime=publisTime, videoLanguage=videoLanguage, isDownload=0) videoType=videoType, videoPublishTime=publisTime, videoLanguage=videoLanguage, isDownload=0)
videosRequest = videosRequest + "," + str(videoId) videosRequest = videosRequest + "," + str(videoId)
videosRequestCount = videosRequestCount + 1 videosRequestCount = videosRequestCount + 1
@ -146,8 +146,8 @@ class YouTubeUtil:
) )
videosRequestCount = 0 videosRequestCount = 0
videosRequest = "" videosRequest = ""
except: except Exception as e:
pass Logger.error(e)
# 获取最后一个视频 # 获取最后一个视频
video: Video = VideoService.getLastVideoByChannelId(channelId) video: Video = VideoService.getLastVideoByChannelId(channelId)
ChannelService.updateTimeByChannelId( ChannelService.updateTimeByChannelId(

1
search_video.py

@ -10,6 +10,7 @@ from common.YoutubeUtils import YouTubeUtil
import operator import operator
import argparse import argparse
# --start="2023-09-10T00:00:01Z" --end="2023-09-11T00:00:01Z"
if __name__ == "__main__": if __name__ == "__main__":
# 读取参数 # 读取参数
parser = argparse.ArgumentParser(description="") parser = argparse.ArgumentParser(description="")

6
service/SrtFileService.py

@ -21,3 +21,9 @@ class SrtFileService:
session.add(srtFile) session.add(srtFile)
session.commit() session.commit()
session.close() session.close()
def insertList(srtFiles):
session = getSession()
session.bulk_save_objects(srtFiles)
session.commit()
session.close()

4
service/VideoService.py

@ -19,9 +19,9 @@ class VideoService:
session.close() session.close()
return videos return videos
def insertOne(videoId, ChannelId, videoTitle, videoLen, videoType, videoPublishTime, videoLanguage, isDownload): def insertOne(videoId, channelId, videoTitle, videoLen, videoType, videoPublishTime, videoLanguage, isDownload):
session = getSession() session = getSession()
video: Video = Video(videoId=videoId, ChannelId=ChannelId, videoTitle=videoTitle, video: Video = Video(videoId=videoId, channelId=channelId, videoTitle=videoTitle,
videoLen=videoLen, videoType=videoType, videoPublishTime=videoPublishTime, videoLen=videoLen, videoType=videoType, videoPublishTime=videoPublishTime,
videoLanguage=videoLanguage, isDownload=isDownload) videoLanguage=videoLanguage, isDownload=isDownload)
session.add(video) session.add(video)

27
test.py

@ -9,6 +9,7 @@ from entity.VideoEntity import Video
from service.ChannelService import ChannelService from service.ChannelService import ChannelService
from service.VideoService import VideoService from service.VideoService import VideoService
from common.YoutubeUtils import YouTubeUtil from common.YoutubeUtils import YouTubeUtil
from common.DownloadUtils import DownloadUtil
import operator import operator
import argparse import argparse
@ -35,12 +36,20 @@ if __name__ == "__main__":
f'mysql+mysqlconnector://{dbUserName}:{dbPassword}@{dbHost}:{dbPort}/{dbDatabase}') f'mysql+mysqlconnector://{dbUserName}:{dbPassword}@{dbHost}:{dbPort}/{dbDatabase}')
Logger.info("连接mysql成功") Logger.info("连接mysql成功")
videoId = "oZhBWA3HNhA" # YouTubeUtil测试
video = VideoService.queryOneByVideoId(videoId) # channelId = "UCBM86JVoHLqg9irpR2XKvGw"
Logger.info(video) # startTime = "2024-08-22T00:00:01Z"
# VideoService.updateLenByVideoId(videoId, 5344) # endTime = "2024-08-24T00:00:01Z"
video = VideoService.getLastVideoByChannelId("UC67Wr_9pA4I0glIxDt_Cpyw") # YouTubeUtil.getByChannelId(channelId, startTime, endTime)
if video == None:
Logger.info("meiyou") # download测试
else: # videoId = "pBSWhJV0VVU"
Logger.info(video.videoPublishTime) # channelId = "UCBM86JVoHLqg9irpR2XKvGw"
# rootPath = "D:/Work/Code/youtube_dev/mysql"
# storePath = "D:/Work/Code/youtube_dev/mysql/main/Korea/UCBM86JVoHLqg9irpR2XKvGw-달란트투자"
# srtFilePath = "D:/Work/Code/youtube_dev/mysql/main/Korea/UCBM86JVoHLqg9irpR2XKvGw-달란트투자/pBSWhJV0VVU.srt"
# DownloadUtil.downloadOne(videoId=videoId, rootPath=rootPath)
# DownloadUtil.iterateSrt(srtFilePath=srtFilePath,
# videoId=videoId, channelId=channelId)
# DownloadUtil.downLoadMP3(videoId=videoId, storePath=storePath)

Loading…
Cancel
Save