You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
108 lines
3.2 KiB
108 lines
3.2 KiB
package main
|
|
|
|
import (
|
|
"container/list"
|
|
"database/sql"
|
|
"keyword_analyse/config"
|
|
"keyword_analyse/entity"
|
|
"strings"
|
|
"sync"
|
|
|
|
_ "github.com/mattn/go-sqlite3"
|
|
)
|
|
|
|
var (
|
|
dbStr string = "D:/Work/Code/youtube_dev/youtube_prod/db/youtube_prod.db"
|
|
dbWriteLock sync.Mutex
|
|
)
|
|
|
|
func main() {
|
|
config.InitConfig()
|
|
config.Logger.Info("aaaaa")
|
|
DB, err := sql.Open("sqlite3", dbStr)
|
|
if err != nil {
|
|
config.Logger.Fatal(err)
|
|
}
|
|
config.Logger.Info("连接成功")
|
|
// 获取关键字
|
|
keyWordList := list.New()
|
|
keyWordRows, err := DB.Query("SELECT * FROM keyWord")
|
|
if err != nil {
|
|
config.Logger.Fatal(err)
|
|
}
|
|
for keyWordRows.Next() {
|
|
// entity := &entity.KeyWord{1, "aa", "aaa"}
|
|
entity := new(entity.KeyWord)
|
|
if err := keyWordRows.Scan(&entity.Id, &entity.Region, &entity.Word); err != nil {
|
|
config.Logger.Fatal(err)
|
|
}
|
|
keyWordList.PushBack(entity)
|
|
}
|
|
keyWordRows.Close()
|
|
continueFlag := true
|
|
for continueFlag {
|
|
// 获取未匹配的字幕,每次1000行
|
|
rows, err := DB.Query("SELECT * FROM srtFile WHERE isScan = 0 limit 500")
|
|
if err != nil {
|
|
config.Logger.Fatal(err)
|
|
}
|
|
SrtFileList := list.New()
|
|
for rows.Next() {
|
|
entity := new(entity.SrtFile)
|
|
if err := rows.Scan(&entity.Id, &entity.VideoId, &entity.ChannelId, &entity.Ordinal, &entity.SrtStartTime, &entity.SrtEndTime, &entity.SrtText, &entity.IsScan); err != nil {
|
|
config.Logger.Fatal(err)
|
|
}
|
|
SrtFileList.PushBack(entity)
|
|
}
|
|
rows.Close()
|
|
var wg sync.WaitGroup
|
|
for e := SrtFileList.Front(); e != nil; e = e.Next() {
|
|
wg.Add(1)
|
|
srtFile := e.Value.(*entity.SrtFile)
|
|
go analyse(*srtFile, *keyWordList, &wg)
|
|
}
|
|
wg.Wait()
|
|
config.Logger.Info("所有携程执行完毕")
|
|
tmpRows, _ := DB.Query("SELECT * FROM srtFile WHERE isScan = 0 limit 1")
|
|
continueFlag = tmpRows.Next()
|
|
tmpRows.Close()
|
|
}
|
|
|
|
}
|
|
|
|
func analyse(srtFile entity.SrtFile, keyWordList list.List, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
filterList := list.New()
|
|
for e := keyWordList.Front(); e != nil; e = e.Next() {
|
|
keyWord := e.Value.(*entity.KeyWord)
|
|
if strings.Contains(srtFile.SrtText, keyWord.Word) {
|
|
config.Logger.Info(keyWord.Word)
|
|
filterList.PushBack(keyWord)
|
|
}
|
|
}
|
|
dbWriteLock.Lock()
|
|
DB, _ := sql.Open("sqlite3", dbStr)
|
|
// 存入WordResutlSet
|
|
for e := filterList.Front(); e != nil; e = e.Next() {
|
|
keyWord := e.Value.(*entity.KeyWord)
|
|
// INSERT INTO "main"."srtfile" ("id", "videoId", "channelId", "ordinal", "srtStartTime", "srtEndTime", "srtText", "isScan") VALUES (1, 'oZhBWA3HNhA', 'UC67Wr_9pA4I0glIxDt_Cpyw', 1, '00:00:30.74', '00:00:35.579', '皆さんおはようございますさあ今日も', 1);
|
|
insterSql, _ := DB.Prepare("INSERT INTO 'main'.'World_Result_Set' ('keyWordId','wordText','srtId','srtOrdinal','srtText','videoId') VALUES(?,?,?,?,?,?)")
|
|
_, err := insterSql.Exec(keyWord.Id, keyWord.Word, srtFile.Id, srtFile.Ordinal, srtFile.SrtText, srtFile.VideoId)
|
|
if err != nil {
|
|
config.Logger.Error(err)
|
|
}
|
|
insterSql.Close()
|
|
}
|
|
// 设置isScan为1
|
|
sqlStr, err := DB.Prepare("UPDATE srtFile SET isScan = 1 WHERE id = ?")
|
|
if err != nil {
|
|
config.Logger.Error(err)
|
|
}
|
|
_, err = sqlStr.Exec(srtFile.Id)
|
|
if err != nil {
|
|
config.Logger.Error(err)
|
|
}
|
|
sqlStr.Close()
|
|
DB.Close()
|
|
dbWriteLock.Unlock()
|
|
}
|
|
|