diff --git a/main_program/common/Constant.go b/main_program/common/Constant.go new file mode 100644 index 0000000..e5e901a --- /dev/null +++ b/main_program/common/Constant.go @@ -0,0 +1,8 @@ +package common + +import ( + config "main_program/config" +) + +var ConfigFile string +var MyEnv config.EnvConfig diff --git a/main_program/config.yml b/main_program/config.yml new file mode 100644 index 0000000..c333a89 --- /dev/null +++ b/main_program/config.yml @@ -0,0 +1,12 @@ +log: + logenv: dev + logpath: main_program +command: move_data +movedata: + table: Channel + dbpath: D:/Work/Code/youtube_dev/youtube_prod.db +mysql: + host: 47.108.20.249:3306 + user: root + password: casino888! + database: youtube \ No newline at end of file diff --git a/main_program/config/EnvConfig.go b/main_program/config/EnvConfig.go new file mode 100644 index 0000000..258caca --- /dev/null +++ b/main_program/config/EnvConfig.go @@ -0,0 +1,25 @@ +package config + +type EnvConfig struct { + Log LogEntity + Command string + MoveData MoveDataEntity + Mysql MysqlEntity +} + +type LogEntity struct { + LogEnv string + LogPath string +} + +type MoveDataEntity struct { + Table string + DBPath string +} + +type MysqlEntity struct { + Host string + User string + Password string + Database string +} diff --git a/main_program/config/config.go b/main_program/config/config.go new file mode 100644 index 0000000..5cccd8d --- /dev/null +++ b/main_program/config/config.go @@ -0,0 +1,139 @@ +package config + +import ( + "bytes" + "fmt" + "io" + "os" + "path" + "time" + + rotatelogs "github.com/lestrrat-go/file-rotatelogs" + "github.com/logrusorgru/aurora" + "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" +) + +const ( + maxAgeHour = 168 + rotationHour = 24 +) + +var isInit bool +var Logger *logrus.Logger + +func init() { + isInit = false +} + +func checkInitLog() bool { + return isInit +} + +func InitConfig(logEnv string, logPath string) { + + if checkInitLog() { + return + } + Logger = initLogger(logEnv, logPath) +} + +func initLogger(env string, logPath string) *logrus.Logger { + logFilePath := "" + errFilePath := "" + if dir, err := os.Getwd(); err == nil { + logFilePath = dir + "/logs/" + logPath + "/all.log" + errFilePath = dir + "/logs/" + logPath + "/error.log" + } + + accessWriter, err := rotatelogs.New( + logFilePath+".%Y-%m-%d", + rotatelogs.WithLinkName(logFilePath), + rotatelogs.WithRotationTime(time.Hour*rotationHour), + rotatelogs.WithMaxAge(time.Hour*maxAgeHour), + ) + if err != nil { + panic(err) + } + errorWriter, err := rotatelogs.New( + errFilePath+".%Y-%m-%d", + rotatelogs.WithLinkName(errFilePath), + rotatelogs.WithRotationTime(time.Hour*rotationHour), + rotatelogs.WithMaxAge(time.Hour*maxAgeHour), + ) + if err != nil { + panic(err) + } + allWriter := io.MultiWriter(accessWriter, errorWriter) + //实例化 + logger := logrus.New() + logger.Out = os.Stdout + //设置日志格式 + logger.SetFormatter(&ConsoleFormatter{}) + logger.SetReportCaller(true) + //设置输出 + if env != "dev" { + logger.AddHook(lfshook.NewHook( + lfshook.WriterMap{ + logrus.DebugLevel: accessWriter, + logrus.InfoLevel: accessWriter, + logrus.ErrorLevel: allWriter, + logrus.PanicLevel: allWriter, + }, + &SaveFormatter{}, + )) + } + //设置日志级别 + logger.SetLevel(logrus.DebugLevel) + Logger = logger + isInit = true + return logger +} + +type ConsoleFormatter struct { +} + +func (m *ConsoleFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var b *bytes.Buffer + if entry.Buffer != nil { + b = entry.Buffer + } else { + b = &bytes.Buffer{} + } + timestamp := entry.Time.Format("2006-01-02 15:04:05") + var newLog string + var levelString aurora.Value + switch entry.Level.String() { + case "info": + levelString = aurora.Green(entry.Level) + case "warning": + levelString = aurora.Yellow(entry.Level) + case "debug": + levelString = aurora.Gray(16-1, entry.Level) + case "error": + levelString = aurora.Red(entry.Level) + case "fatal": + levelString = aurora.Red(entry.Level) + case "panic": + levelString = aurora.Red(entry.Level) + } + newLog = fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, levelString, entry.Message) + b.WriteString(newLog) + return b.Bytes(), nil +} + +type SaveFormatter struct { +} + +func (m *SaveFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var b *bytes.Buffer + if entry.Buffer != nil { + b = entry.Buffer + } else { + b = &bytes.Buffer{} + } + timestamp := entry.Time.Format("2006-01-02 15:04:05") + newLog := fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, entry.Level, entry.Message) + b.WriteString(newLog) + return b.Bytes(), nil +} diff --git a/main_program/entity/Channel.go b/main_program/entity/Channel.go new file mode 100644 index 0000000..9f71700 --- /dev/null +++ b/main_program/entity/Channel.go @@ -0,0 +1,23 @@ +package entity + +type ChannelCopy struct { + Id int + ChannelId string + ChannelTitle string + ChannelLanguage string + ChannelReptileTime *string + Is_Copy int +} + +type Channel struct { + Id uint `gorm:"column:id;primaryKey;autoIncrement"` + ChannelId string `gorm:"column:channelId;type:varchar(255);not null"` + ChannelTitle string `gorm:"column:channelTitle;type:varchar(255);not null"` + ChannelLanguage string `gorm:"column:channelLanguage;type:varchar(255);not null"` + ChannelReptileTime *string `gorm:"column:channelReptileTime;type:varchar(255);default:null"` + Region string `gorm:"column:region;type:varchar(255);default:null"` +} + +func (Channel) TableName() string { + return "Channel" +} diff --git a/main_program/go.mod b/main_program/go.mod new file mode 100644 index 0000000..1d8905f --- /dev/null +++ b/main_program/go.mod @@ -0,0 +1,27 @@ +module main_program + +go 1.22.2 + +require ( + github.com/sirupsen/logrus v1.9.3 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect + github.com/pkg/errors v0.9.1 // indirect + gorm.io/gorm v1.25.7 // indirect +) + +require ( + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible + github.com/logrusorgru/aurora v2.0.3+incompatible + github.com/mattn/go-sqlite3 v1.14.22 + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 + github.com/tealeg/xlsx v1.0.5 + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + gorm.io/driver/mysql v1.5.7 +) diff --git a/main_program/go.sum b/main_program/go.sum new file mode 100644 index 0000000..6f1e56d --- /dev/null +++ b/main_program/go.sum @@ -0,0 +1,43 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= +github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= +github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= +github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= diff --git a/main_program/main.go b/main_program/main.go new file mode 100644 index 0000000..41f38d7 --- /dev/null +++ b/main_program/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + + common "main_program/common" + "main_program/config" + move_data "main_program/moveData" + + "gopkg.in/yaml.v2" +) + +func init() { + flag.StringVar(&common.ConfigFile, "cf", "config.yml", "配置文件名") +} + +func main() { + flag.Parse() + //读取配置文件 + data, _ := ioutil.ReadFile(common.ConfigFile) + err := yaml.Unmarshal(data, &common.MyEnv) + if err != nil { + fmt.Println("读取配置文件错误...") + return + } + config.InitConfig(common.MyEnv.Log.LogEnv, common.MyEnv.Log.LogPath) + config.Logger.Info("初始化Logger成功...") + // 判断command + command := common.MyEnv.Command + config.Logger.Infof("Command:%s", common.MyEnv.Command) + if command == "move_data" { + // 判断参数 + table := common.MyEnv.MoveData.Table + dbPath := common.MyEnv.MoveData.DBPath + if table == "" || dbPath == "" { + config.Logger.Error("move_data配置文件错误...") + return + } + config.Logger.Infof("开始调用move_data方法") + move_data.MoveDataService.Start(table, dbPath, common.MyEnv.Mysql) + } + +} diff --git a/main_program/moveData/channel_region.xlsx b/main_program/moveData/channel_region.xlsx new file mode 100644 index 0000000..fae3d69 Binary files /dev/null and b/main_program/moveData/channel_region.xlsx differ diff --git a/main_program/moveData/moveDataService.go b/main_program/moveData/moveDataService.go new file mode 100644 index 0000000..40171dc --- /dev/null +++ b/main_program/moveData/moveDataService.go @@ -0,0 +1,105 @@ +package move_data + +import ( + "container/list" + "database/sql" + "fmt" + "main_program/config" + "main_program/entity" + + _ "github.com/mattn/go-sqlite3" + "github.com/tealeg/xlsx" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +type moveDataService struct{} + +var MoveDataService moveDataService + +func (m *moveDataService) Start(table string, sqlitePath string, mysqlConfig config.MysqlEntity) { + config.Logger.Info("开始迁移sqlite3到mysql") + sqliteDB, err := sql.Open("sqlite3", sqlitePath) + if err != nil { + config.Logger.Fatal(err) + } + config.Logger.Info("连接成功sqlite3...") + dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", mysqlConfig.User, mysqlConfig.Password, mysqlConfig.Host, mysqlConfig.Database) + mysqlDB, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) + if err != nil { + config.Logger.Fatal("数据库连接失败") + } + config.Logger.Info("连接成功mysql...") + if table == "Channel" { + m.moveChannel(sqliteDB, mysqlDB) + } +} + +func (m *moveDataService) moveChannel(sqliteDB *sql.DB, mysqlDB *gorm.DB) { + config.Logger.Info("读取xlsx获取region") + file, err := xlsx.OpenFile("D:/Work/Code/youtube_dev/youtube-golang/main_program/moveData/channel_region.xlsx") + if err != nil { + config.Logger.Fatalf("Error opening file: %s", err) + } + config.Logger.Info("开始迁移Channel表") + // 从sqlite3获取前50个Channel + continueFlag := true + for continueFlag { + rows, err := sqliteDB.Query("SELECT * FROM Channel_copy WHERE is_copy = 0 limit 50") + if err != nil { + config.Logger.Fatal(err) + } + ChannelList := list.New() + for rows.Next() { + channelCopy := new(entity.ChannelCopy) + if err := rows.Scan(&channelCopy.Id, &channelCopy.ChannelId, &channelCopy.ChannelTitle, &channelCopy.ChannelLanguage, &channelCopy.ChannelReptileTime, &channelCopy.Is_Copy); err != nil { + config.Logger.Fatal(err) + } + ChannelList.PushBack(channelCopy) + } + rows.Close() + if ChannelList.Len() <= 0 { + break + } + // 放入mysql里 + for e := ChannelList.Front(); e != nil; e = e.Next() { + channelCopy := e.Value.(*entity.ChannelCopy) + config.Logger.Info(channelCopy) + channel := new(entity.Channel) + channel.ChannelId = channelCopy.ChannelId + channel.ChannelTitle = channelCopy.ChannelTitle + channel.ChannelLanguage = channelCopy.ChannelLanguage + channel.ChannelReptileTime = channelCopy.ChannelReptileTime + channel.Region = m.getRegionByChannelId(channel.ChannelId, file) + result := mysqlDB.Create(&channel) + if result.Error != nil { + config.Logger.Fatal(result.Error) + } + // 修改sqlite里状态 + sqlStr, err := sqliteDB.Prepare("UPDATE Channel_copy SET is_copy = 1 WHERE id = ?") + if err != nil { + config.Logger.Error(err) + } + _, err = sqlStr.Exec(channelCopy.Id) + if err != nil { + config.Logger.Error(err) + } + sqlStr.Close() + } + tmpRows, _ := sqliteDB.Query("SELECT * FROM Channel_copy WHERE is_copy = 0 limit 50") + continueFlag = tmpRows.Next() + tmpRows.Close() + } + +} + +func (m *moveDataService) getRegionByChannelId(channnelId string, file *xlsx.File) string { + for _, sheet := range file.Sheets { + for _, row := range sheet.Rows { + if row.Cells[3].Value == channnelId { + return row.Cells[2].Value + } + } + } + return "" +}