commit

keyword_analyse/main.go

@@ -6,27 +6,30 @@ import (
 	"keyword_analyse/config"
 	"keyword_analyse/entity"
 	"strings"
+	"sync"
 
 	_ "github.com/mattn/go-sqlite3"
 )
 
+var (
+	dbStr       string = "D:/Work/Code/youtube_dev/youtube_prod/db/youtube_prod.db"
+	dbWriteLock sync.Mutex
+)
+
 func main() {
 	config.InitConfig()
 	config.Logger.Info("aaaaa")
-	db, err := sql.Open("sqlite3", "E:/code/youtube_prod/db/youtube_prod.db")
+	DB, err := sql.Open("sqlite3", dbStr)
 	if err != nil {
 		config.Logger.Fatal(err)
 	}
-	defer db.Close()
 	config.Logger.Info("连接成功")
 	// 获取关键字
 	keyWordList := list.New()
-	SrtFileList := list.New()
-	keyWordRows, err := db.Query("SELECT * FROM keyWord")
+	keyWordRows, err := DB.Query("SELECT * FROM keyWord")
 	if err != nil {
 		config.Logger.Fatal(err)
 	}
-	defer keyWordRows.Close()
 	for keyWordRows.Next() {
 		// entity := &entity.KeyWord{1, "aa", "aaa"}
 		entity := new(entity.KeyWord)
@@ -35,34 +38,71 @@ func main() {
 		}
 		keyWordList.PushBack(entity)
 	}
-	// 获取未匹配的字幕
-	rows, err := db.Query("SELECT * FROM srtFile WHERE isScan = 0")
-	if err != nil {
-		config.Logger.Fatal(err)
-	}
-	defer rows.Close()
-	for rows.Next() {
-		entity := new(entity.SrtFile)
-		if err := rows.Scan(&entity.Id, &entity.VideoId, &entity.ChannelId, &entity.Ordinal, &entity.SrtStartTime, &entity.SrtEndTime, &entity.SrtText, &entity.IsScan); err != nil {
+	keyWordRows.Close()
+	continueFlag := true
+	for continueFlag {
+		// 获取未匹配的字幕,每次1000行
+		rows, err := DB.Query("SELECT * FROM srtFile WHERE isScan = 0 limit 500")
+		if err != nil {
 			config.Logger.Fatal(err)
 		}
-		SrtFileList.PushBack(entity)
-	}
-	config.Logger.Infof("keyWordLen:%d,srtFileLen:%d", keyWordList.Len(), SrtFileList.Len())
-	for e := SrtFileList.Front(); e != nil; e = e.Next() {
-		srtFile := e.Value.(*entity.SrtFile)
-		go analyse(*srtFile, *keyWordList)
+		SrtFileList := list.New()
+		for rows.Next() {
+			entity := new(entity.SrtFile)
+			if err := rows.Scan(&entity.Id, &entity.VideoId, &entity.ChannelId, &entity.Ordinal, &entity.SrtStartTime, &entity.SrtEndTime, &entity.SrtText, &entity.IsScan); err != nil {
+				config.Logger.Fatal(err)
+			}
+			SrtFileList.PushBack(entity)
+		}
+		rows.Close()
+		var wg sync.WaitGroup
+		for e := SrtFileList.Front(); e != nil; e = e.Next() {
+			wg.Add(1)
+			srtFile := e.Value.(*entity.SrtFile)
+			go analyse(*srtFile, *keyWordList, &wg)
+		}
+		wg.Wait()
+		config.Logger.Info("所有携程执行完毕")
+		tmpRows, _ := DB.Query("SELECT * FROM srtFile WHERE isScan = 0 limit 1")
+		continueFlag = tmpRows.Next()
+		tmpRows.Close()
 	}
 
 }
 
-func analyse(srtFile entity.SrtFile, keyWordList list.List) {
+func analyse(srtFile entity.SrtFile, keyWordList list.List, wg *sync.WaitGroup) {
+	defer wg.Done()
+	filterList := list.New()
 	for e := keyWordList.Front(); e != nil; e = e.Next() {
 		keyWord := e.Value.(*entity.KeyWord)
 		if strings.Contains(srtFile.SrtText, keyWord.Word) {
 			config.Logger.Info(keyWord.Word)
-		} else {
-			config.Logger.Info("no contains")
+			filterList.PushBack(keyWord)
 		}
 	}
+	dbWriteLock.Lock()
+	DB, _ := sql.Open("sqlite3", dbStr)
+	// 存入WordResutlSet
+	for e := filterList.Front(); e != nil; e = e.Next() {
+		keyWord := e.Value.(*entity.KeyWord)
+		// INSERT INTO "main"."srtfile" ("id", "videoId", "channelId", "ordinal", "srtStartTime", "srtEndTime", "srtText", "isScan") VALUES (1, 'oZhBWA3HNhA', 'UC67Wr_9pA4I0glIxDt_Cpyw', 1, '00:00:30.74', '00:00:35.579', '皆さんおはようございますさあ今日も', 1);
+		insterSql, _ := DB.Prepare("INSERT INTO 'main'.'World_Result_Set' ('keyWordId','wordText','srtId','srtOrdinal','srtText','videoId') VALUES(?,?,?,?,?,?)")
+		_, err := insterSql.Exec(keyWord.Id, keyWord.Word, srtFile.Id, srtFile.Ordinal, srtFile.SrtText, srtFile.VideoId)
+		if err != nil {
+			config.Logger.Error(err)
+		}
+		insterSql.Close()
+	}
+	// 设置isScan为1
+	sqlStr, err := DB.Prepare("UPDATE srtFile SET isScan = 1 WHERE id = ?")
+	if err != nil {
+		config.Logger.Error(err)
+	}
+	_, err = sqlStr.Exec(srtFile.Id)
+	if err != nil {
+		config.Logger.Error(err)
+	}
+	sqlStr.Close()
+	DB.Close()
+	dbWriteLock.Unlock()
 }