diff --git a/.gitignore b/.gitignore index f4d432a..248ab06 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ # Dependency directories (remove the comment below to include it) # vendor/ - +.vscode +logs \ No newline at end of file diff --git a/keyword_analyse/config.yml b/keyword_analyse/config.yml new file mode 100644 index 0000000..3d72e5b --- /dev/null +++ b/keyword_analyse/config.yml @@ -0,0 +1,4 @@ +log: + logenv: dev + logpath: keyword_analyse + diff --git a/keyword_analyse/config/EnvConfig.go b/keyword_analyse/config/EnvConfig.go new file mode 100644 index 0000000..3f405f0 --- /dev/null +++ b/keyword_analyse/config/EnvConfig.go @@ -0,0 +1,10 @@ +package config + +type EnvConfig struct { + Log LogEntity +} + +type LogEntity struct { + LogEnv string + LogPath string +} diff --git a/keyword_analyse/config/config.go b/keyword_analyse/config/config.go new file mode 100644 index 0000000..86a2544 --- /dev/null +++ b/keyword_analyse/config/config.go @@ -0,0 +1,153 @@ +package config + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "time" + + rotatelogs "github.com/lestrrat-go/file-rotatelogs" + "github.com/logrusorgru/aurora" + "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +const ( + maxAgeHour = 168 + rotationHour = 24 +) + +var isInit bool +var configFile string +var MyEnv EnvConfig +var Logger *logrus.Logger + +func init() { + flag.StringVar(&configFile, "cf", "config.yml", "配置文件名") + isInit = false +} + +func checkInitLog() bool { + return isInit +} + +func InitConfig() { + flag.Parse() + //读取配置文件 + data, _ := ioutil.ReadFile(configFile) + err := yaml.Unmarshal(data, &MyEnv) + if err != nil { + fmt.Println("读取配置文件错误...") + return + } + if checkInitLog() { + return + } + Logger = initLogger(MyEnv.Log.LogEnv, MyEnv.Log.LogPath) +} + +func initLogger(env string, logPath string) *logrus.Logger { + logFilePath := "" + errFilePath := "" + if dir, err := os.Getwd(); err == nil { + logFilePath = dir + "/logs/" + logPath + "/all.log" + errFilePath = dir + "/logs/" + logPath + "/error.log" + } + + accessWriter, err := rotatelogs.New( + logFilePath+".%Y-%m-%d", + rotatelogs.WithLinkName(logFilePath), + rotatelogs.WithRotationTime(time.Hour*rotationHour), + rotatelogs.WithMaxAge(time.Hour*maxAgeHour), + ) + if err != nil { + panic(err) + } + errorWriter, err := rotatelogs.New( + errFilePath+".%Y-%m-%d", + rotatelogs.WithLinkName(errFilePath), + rotatelogs.WithRotationTime(time.Hour*rotationHour), + rotatelogs.WithMaxAge(time.Hour*maxAgeHour), + ) + if err != nil { + panic(err) + } + allWriter := io.MultiWriter(accessWriter, errorWriter) + //实例化 + logger := logrus.New() + logger.Out = os.Stdout + //设置日志格式 + logger.SetFormatter(&ConsoleFormatter{}) + logger.SetReportCaller(true) + //设置输出 + if env != "dev" { + logger.AddHook(lfshook.NewHook( + lfshook.WriterMap{ + logrus.DebugLevel: accessWriter, + logrus.InfoLevel: accessWriter, + logrus.ErrorLevel: allWriter, + logrus.PanicLevel: allWriter, + }, + &SaveFormatter{}, + )) + } + //设置日志级别 + logger.SetLevel(logrus.DebugLevel) + Logger = logger + isInit = true + return logger +} + +type ConsoleFormatter struct { +} + +func (m *ConsoleFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var b *bytes.Buffer + if entry.Buffer != nil { + b = entry.Buffer + } else { + b = &bytes.Buffer{} + } + timestamp := entry.Time.Format("2006-01-02 15:04:05") + var newLog string + var levelString aurora.Value + switch entry.Level.String() { + case "info": + levelString = aurora.Green(entry.Level) + case "warning": + levelString = aurora.Yellow(entry.Level) + case "debug": + levelString = aurora.Gray(16-1, entry.Level) + case "error": + levelString = aurora.Red(entry.Level) + case "fatal": + levelString = aurora.Red(entry.Level) + case "panic": + levelString = aurora.Red(entry.Level) + } + newLog = fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, levelString, entry.Message) + b.WriteString(newLog) + return b.Bytes(), nil +} + +type SaveFormatter struct { +} + +func (m *SaveFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var b *bytes.Buffer + if entry.Buffer != nil { + b = entry.Buffer + } else { + b = &bytes.Buffer{} + } + timestamp := entry.Time.Format("2006-01-02 15:04:05") + var newLog string + newLog = fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, entry.Level, entry.Message) + b.WriteString(newLog) + return b.Bytes(), nil +} diff --git a/keyword_analyse/entity/KeyWord.go b/keyword_analyse/entity/KeyWord.go new file mode 100644 index 0000000..a09cdd1 --- /dev/null +++ b/keyword_analyse/entity/KeyWord.go @@ -0,0 +1,18 @@ +package entity + +type KeyWord struct { + Id int + Region string + Word string +} + +type SrtFile struct { + Id int + VideoId string + ChannelId string + Ordinal int + SrtStartTime string + SrtEndTime string + SrtText string + IsScan int +} diff --git a/keyword_analyse/go.mod b/keyword_analyse/go.mod new file mode 100644 index 0000000..c1ef071 --- /dev/null +++ b/keyword_analyse/go.mod @@ -0,0 +1,21 @@ +module keyword_analyse + +go 1.22.2 + +require ( + github.com/sirupsen/logrus v1.9.3 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/lestrrat-go/strftime v1.0.6 // indirect + github.com/pkg/errors v0.9.1 // indirect +) + +require ( + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible + github.com/logrusorgru/aurora v2.0.3+incompatible + github.com/mattn/go-sqlite3 v1.14.22 + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect +) diff --git a/keyword_analyse/go.sum b/keyword_analyse/go.sum new file mode 100644 index 0000000..055a6aa --- /dev/null +++ b/keyword_analyse/go.sum @@ -0,0 +1,27 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= +github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= +github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/keyword_analyse/main.go b/keyword_analyse/main.go new file mode 100644 index 0000000..b395d5c --- /dev/null +++ b/keyword_analyse/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "container/list" + "database/sql" + "keyword_analyse/config" + "keyword_analyse/entity" + "strings" + + _ "github.com/mattn/go-sqlite3" +) + +func main() { + config.InitConfig() + config.Logger.Info("aaaaa") + db, err := sql.Open("sqlite3", "E:/code/youtube_prod/db/youtube_prod.db") + if err != nil { + config.Logger.Fatal(err) + } + defer db.Close() + config.Logger.Info("连接成功") + // 获取关键字 + keyWordList := list.New() + SrtFileList := list.New() + keyWordRows, err := db.Query("SELECT * FROM keyWord") + if err != nil { + config.Logger.Fatal(err) + } + defer keyWordRows.Close() + for keyWordRows.Next() { + // entity := &entity.KeyWord{1, "aa", "aaa"} + entity := new(entity.KeyWord) + if err := keyWordRows.Scan(&entity.Id, &entity.Region, &entity.Word); err != nil { + config.Logger.Fatal(err) + } + keyWordList.PushBack(entity) + } + // 获取未匹配的字幕 + rows, err := db.Query("SELECT * FROM srtFile WHERE isScan = 0") + if err != nil { + config.Logger.Fatal(err) + } + defer rows.Close() + for rows.Next() { + entity := new(entity.SrtFile) + if err := rows.Scan(&entity.Id, &entity.VideoId, &entity.ChannelId, &entity.Ordinal, &entity.SrtStartTime, &entity.SrtEndTime, &entity.SrtText, &entity.IsScan); err != nil { + config.Logger.Fatal(err) + } + SrtFileList.PushBack(entity) + } + config.Logger.Infof("keyWordLen:%d,srtFileLen:%d", keyWordList.Len(), SrtFileList.Len()) + for e := SrtFileList.Front(); e != nil; e = e.Next() { + srtFile := e.Value.(*entity.SrtFile) + go analyse(*srtFile, *keyWordList) + } + +} + +func analyse(srtFile entity.SrtFile, keyWordList list.List) { + for e := keyWordList.Front(); e != nil; e = e.Next() { + keyWord := e.Value.(*entity.KeyWord) + if strings.Contains(srtFile.SrtText, keyWord.Word) { + config.Logger.Info(keyWord.Word) + } else { + config.Logger.Info("no contains") + } + } +}