youtube-golang/keyword_analyse/main.go

package main

import (
	"container/list"
	"database/sql"
	"keyword_analyse/config"
	"keyword_analyse/entity"
	"strings"
	"sync"

	_ "github.com/mattn/go-sqlite3"
)

var (
	dbStr       string = "D:/Work/Code/youtube_dev/youtube_prod/db/youtube_prod.db"
	dbWriteLock sync.Mutex
)

func main() {
	config.InitConfig()
	config.Logger.Info("aaaaa")
	DB, err := sql.Open("sqlite3", dbStr)
	if err != nil {
		config.Logger.Fatal(err)
	}
	config.Logger.Info("连接成功")
	// 获取关键字
	keyWordList := list.New()
	keyWordRows, err := DB.Query("SELECT * FROM keyWord")
	if err != nil {
		config.Logger.Fatal(err)
	}
	for keyWordRows.Next() {
		// entity := &entity.KeyWord{1, "aa", "aaa"}
		entity := new(entity.KeyWord)
		if err := keyWordRows.Scan(&entity.Id, &entity.Region, &entity.Word); err != nil {
			config.Logger.Fatal(err)
		}
		keyWordList.PushBack(entity)
	}
	keyWordRows.Close()
	continueFlag := true
	for continueFlag {
		// 获取未匹配的字幕,每次1000行
		rows, err := DB.Query("SELECT * FROM srtFile WHERE isScan = 0 limit 500")
		if err != nil {
			config.Logger.Fatal(err)
		}
		SrtFileList := list.New()
		for rows.Next() {
			entity := new(entity.SrtFile)
			if err := rows.Scan(&entity.Id, &entity.VideoId, &entity.ChannelId, &entity.Ordinal, &entity.SrtStartTime, &entity.SrtEndTime, &entity.SrtText, &entity.IsScan); err != nil {
				config.Logger.Fatal(err)
			}
			SrtFileList.PushBack(entity)
		}
		rows.Close()
		var wg sync.WaitGroup
		for e := SrtFileList.Front(); e != nil; e = e.Next() {
			wg.Add(1)
			srtFile := e.Value.(*entity.SrtFile)
			go analyse(*srtFile, *keyWordList, &wg)
		}
		wg.Wait()
		config.Logger.Info("所有携程执行完毕")
		tmpRows, _ := DB.Query("SELECT * FROM srtFile WHERE isScan = 0 limit 1")
		continueFlag = tmpRows.Next()
		tmpRows.Close()
	}

}

func analyse(srtFile entity.SrtFile, keyWordList list.List, wg *sync.WaitGroup) {
	defer wg.Done()
	filterList := list.New()
	for e := keyWordList.Front(); e != nil; e = e.Next() {
		keyWord := e.Value.(*entity.KeyWord)
		if strings.Contains(srtFile.SrtText, keyWord.Word) {
			config.Logger.Info(keyWord.Word)
			filterList.PushBack(keyWord)
		}
	}
	dbWriteLock.Lock()
	DB, _ := sql.Open("sqlite3", dbStr)
	// 存入WordResutlSet
	for e := filterList.Front(); e != nil; e = e.Next() {
		keyWord := e.Value.(*entity.KeyWord)
		// INSERT INTO "main"."srtfile" ("id", "videoId", "channelId", "ordinal", "srtStartTime", "srtEndTime", "srtText", "isScan") VALUES (1, 'oZhBWA3HNhA', 'UC67Wr_9pA4I0glIxDt_Cpyw', 1, '00:00:30.74', '00:00:35.579', '皆さんおはようございますさあ今日も', 1);
		insterSql, _ := DB.Prepare("INSERT INTO 'main'.'World_Result_Set' ('keyWordId','wordText','srtId','srtOrdinal','srtText','videoId') VALUES(?,?,?,?,?,?)")
		_, err := insterSql.Exec(keyWord.Id, keyWord.Word, srtFile.Id, srtFile.Ordinal, srtFile.SrtText, srtFile.VideoId)
		if err != nil {
			config.Logger.Error(err)
		}
		insterSql.Close()
	}
	// 设置isScan为1
	sqlStr, err := DB.Prepare("UPDATE srtFile SET isScan = 1 WHERE id = ?")
	if err != nil {
		config.Logger.Error(err)
	}
	_, err = sqlStr.Exec(srtFile.Id)
	if err != nil {
		config.Logger.Error(err)
	}
	sqlStr.Close()
	DB.Close()
	dbWriteLock.Unlock()
}