first commit

.gitignore

@@ -14,4 +14,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
-
+.vscode
+logs

keyword_analyse/config.yml

@@ -0,0 +1,4 @@
+log:
+  logenv: dev
+  logpath: keyword_analyse
+

keyword_analyse/config/EnvConfig.go

@@ -0,0 +1,10 @@
+package config
+
+type EnvConfig struct {
+	Log LogEntity
+}
+
+type LogEntity struct {
+	LogEnv  string
+	LogPath string
+}

keyword_analyse/config/config.go

@@ -0,0 +1,153 @@
+package config
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"time"
+
+	rotatelogs "github.com/lestrrat-go/file-rotatelogs"
+	"github.com/logrusorgru/aurora"
+	"github.com/rifflock/lfshook"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	maxAgeHour   = 168
+	rotationHour = 24
+)
+
+var isInit bool
+var configFile string
+var MyEnv EnvConfig
+var Logger *logrus.Logger
+
+func init() {
+	flag.StringVar(&configFile, "cf", "config.yml", "配置文件名")
+	isInit = false
+}
+
+func checkInitLog() bool {
+	return isInit
+}
+
+func InitConfig() {
+	flag.Parse()
+	//读取配置文件
+	data, _ := ioutil.ReadFile(configFile)
+	err := yaml.Unmarshal(data, &MyEnv)
+	if err != nil {
+		fmt.Println("读取配置文件错误...")
+		return
+	}
+	if checkInitLog() {
+		return
+	}
+	Logger = initLogger(MyEnv.Log.LogEnv, MyEnv.Log.LogPath)
+}
+
+func initLogger(env string, logPath string) *logrus.Logger {
+	logFilePath := ""
+	errFilePath := ""
+	if dir, err := os.Getwd(); err == nil {
+		logFilePath = dir + "/logs/" + logPath + "/all.log"
+		errFilePath = dir + "/logs/" + logPath + "/error.log"
+	}
+
+	accessWriter, err := rotatelogs.New(
+		logFilePath+".%Y-%m-%d",
+		rotatelogs.WithLinkName(logFilePath),
+		rotatelogs.WithRotationTime(time.Hour*rotationHour),
+		rotatelogs.WithMaxAge(time.Hour*maxAgeHour),
+	)
+	if err != nil {
+		panic(err)
+	}
+	errorWriter, err := rotatelogs.New(
+		errFilePath+".%Y-%m-%d",
+		rotatelogs.WithLinkName(errFilePath),
+		rotatelogs.WithRotationTime(time.Hour*rotationHour),
+		rotatelogs.WithMaxAge(time.Hour*maxAgeHour),
+	)
+	if err != nil {
+		panic(err)
+	}
+	allWriter := io.MultiWriter(accessWriter, errorWriter)
+	//实例化
+	logger := logrus.New()
+	logger.Out = os.Stdout
+	//设置日志格式
+	logger.SetFormatter(&ConsoleFormatter{})
+	logger.SetReportCaller(true)
+	//设置输出
+	if env != "dev" {
+		logger.AddHook(lfshook.NewHook(
+			lfshook.WriterMap{
+				logrus.DebugLevel: accessWriter,
+				logrus.InfoLevel:  accessWriter,
+				logrus.ErrorLevel: allWriter,
+				logrus.PanicLevel: allWriter,
+			},
+			&SaveFormatter{},
+		))
+	}
+	//设置日志级别
+	logger.SetLevel(logrus.DebugLevel)
+	Logger = logger
+	isInit = true
+	return logger
+}
+
+type ConsoleFormatter struct {
+}
+
+func (m *ConsoleFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	var b *bytes.Buffer
+	if entry.Buffer != nil {
+		b = entry.Buffer
+	} else {
+		b = &bytes.Buffer{}
+	}
+	timestamp := entry.Time.Format("2006-01-02 15:04:05")
+	var newLog string
+	var levelString aurora.Value
+	switch entry.Level.String() {
+	case "info":
+		levelString = aurora.Green(entry.Level)
+	case "warning":
+		levelString = aurora.Yellow(entry.Level)
+	case "debug":
+		levelString = aurora.Gray(16-1, entry.Level)
+	case "error":
+		levelString = aurora.Red(entry.Level)
+	case "fatal":
+		levelString = aurora.Red(entry.Level)
+	case "panic":
+		levelString = aurora.Red(entry.Level)
+	}
+	newLog = fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, levelString, entry.Message)
+	b.WriteString(newLog)
+	return b.Bytes(), nil
+}
+
+type SaveFormatter struct {
+}
+
+func (m *SaveFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	var b *bytes.Buffer
+	if entry.Buffer != nil {
+		b = entry.Buffer
+	} else {
+		b = &bytes.Buffer{}
+	}
+	timestamp := entry.Time.Format("2006-01-02 15:04:05")
+	var newLog string
+	newLog = fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, entry.Level, entry.Message)
+	b.WriteString(newLog)
+	return b.Bytes(), nil
+}

keyword_analyse/entity/KeyWord.go

@@ -0,0 +1,18 @@
+package entity
+
+type KeyWord struct {
+	Id     int
+	Region string
+	Word   string
+}
+
+type SrtFile struct {
+	Id           int
+	VideoId      string
+	ChannelId    string
+	Ordinal      int
+	SrtStartTime string
+	SrtEndTime   string
+	SrtText      string
+	IsScan       int
+}

keyword_analyse/go.mod

@@ -0,0 +1,21 @@
+module keyword_analyse
+
+go 1.22.2
+
+require (
+	github.com/sirupsen/logrus v1.9.3
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/lestrrat-go/strftime v1.0.6 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+)
+
+require (
+	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+)

keyword_analyse/go.sum

@@ -0,0 +1,27 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
+github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
+github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

keyword_analyse/main.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"container/list"
+	"database/sql"
+	"keyword_analyse/config"
+	"keyword_analyse/entity"
+	"strings"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+func main() {
+	config.InitConfig()
+	config.Logger.Info("aaaaa")
+	db, err := sql.Open("sqlite3", "E:/code/youtube_prod/db/youtube_prod.db")
+	if err != nil {
+		config.Logger.Fatal(err)
+	}
+	defer db.Close()
+	config.Logger.Info("连接成功")
+	// 获取关键字
+	keyWordList := list.New()
+	SrtFileList := list.New()
+	keyWordRows, err := db.Query("SELECT * FROM keyWord")
+	if err != nil {
+		config.Logger.Fatal(err)
+	}
+	defer keyWordRows.Close()
+	for keyWordRows.Next() {
+		// entity := &entity.KeyWord{1, "aa", "aaa"}
+		entity := new(entity.KeyWord)
+		if err := keyWordRows.Scan(&entity.Id, &entity.Region, &entity.Word); err != nil {
+			config.Logger.Fatal(err)
+		}
+		keyWordList.PushBack(entity)
+	}
+	// 获取未匹配的字幕
+	rows, err := db.Query("SELECT * FROM srtFile WHERE isScan = 0")
+	if err != nil {
+		config.Logger.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		entity := new(entity.SrtFile)
+		if err := rows.Scan(&entity.Id, &entity.VideoId, &entity.ChannelId, &entity.Ordinal, &entity.SrtStartTime, &entity.SrtEndTime, &entity.SrtText, &entity.IsScan); err != nil {
+			config.Logger.Fatal(err)
+		}
+		SrtFileList.PushBack(entity)
+	}
+	config.Logger.Infof("keyWordLen:%d,srtFileLen:%d", keyWordList.Len(), SrtFileList.Len())
+	for e := SrtFileList.Front(); e != nil; e = e.Next() {
+		srtFile := e.Value.(*entity.SrtFile)
+		go analyse(*srtFile, *keyWordList)
+	}
+
+}
+
+func analyse(srtFile entity.SrtFile, keyWordList list.List) {
+	for e := keyWordList.Front(); e != nil; e = e.Next() {
+		keyWord := e.Value.(*entity.KeyWord)
+		if strings.Contains(srtFile.SrtText, keyWord.Word) {
+			config.Logger.Info(keyWord.Word)
+		} else {
+			config.Logger.Info("no contains")
+		}
+	}
+}