golang迁移Channel表

main_program/common/Constant.go

@@ -0,0 +1,8 @@
+package common
+
+import (
+	config "main_program/config"
+)
+
+var ConfigFile string
+var MyEnv config.EnvConfig

main_program/config.yml

@@ -0,0 +1,12 @@
+log:
+  logenv: dev
+  logpath: main_program
+command: move_data
+movedata:
+  table: Channel
+  dbpath: D:/Work/Code/youtube_dev/youtube_prod.db
+mysql:
+  host: 47.108.20.249:3306
+  user: root
+  password: casino888!
+  database: youtube

main_program/config/EnvConfig.go

@@ -0,0 +1,25 @@
+package config
+
+type EnvConfig struct {
+	Log      LogEntity
+	Command  string
+	MoveData MoveDataEntity
+	Mysql    MysqlEntity
+}
+
+type LogEntity struct {
+	LogEnv  string
+	LogPath string
+}
+
+type MoveDataEntity struct {
+	Table  string
+	DBPath string
+}
+
+type MysqlEntity struct {
+	Host     string
+	User     string
+	Password string
+	Database string
+}

main_program/config/config.go

@@ -0,0 +1,139 @@
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"time"
+
+	rotatelogs "github.com/lestrrat-go/file-rotatelogs"
+	"github.com/logrusorgru/aurora"
+	"github.com/rifflock/lfshook"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	maxAgeHour   = 168
+	rotationHour = 24
+)
+
+var isInit bool
+var Logger *logrus.Logger
+
+func init() {
+	isInit = false
+}
+
+func checkInitLog() bool {
+	return isInit
+}
+
+func InitConfig(logEnv string, logPath string) {
+
+	if checkInitLog() {
+		return
+	}
+	Logger = initLogger(logEnv, logPath)
+}
+
+func initLogger(env string, logPath string) *logrus.Logger {
+	logFilePath := ""
+	errFilePath := ""
+	if dir, err := os.Getwd(); err == nil {
+		logFilePath = dir + "/logs/" + logPath + "/all.log"
+		errFilePath = dir + "/logs/" + logPath + "/error.log"
+	}
+
+	accessWriter, err := rotatelogs.New(
+		logFilePath+".%Y-%m-%d",
+		rotatelogs.WithLinkName(logFilePath),
+		rotatelogs.WithRotationTime(time.Hour*rotationHour),
+		rotatelogs.WithMaxAge(time.Hour*maxAgeHour),
+	)
+	if err != nil {
+		panic(err)
+	}
+	errorWriter, err := rotatelogs.New(
+		errFilePath+".%Y-%m-%d",
+		rotatelogs.WithLinkName(errFilePath),
+		rotatelogs.WithRotationTime(time.Hour*rotationHour),
+		rotatelogs.WithMaxAge(time.Hour*maxAgeHour),
+	)
+	if err != nil {
+		panic(err)
+	}
+	allWriter := io.MultiWriter(accessWriter, errorWriter)
+	//实例化
+	logger := logrus.New()
+	logger.Out = os.Stdout
+	//设置日志格式
+	logger.SetFormatter(&ConsoleFormatter{})
+	logger.SetReportCaller(true)
+	//设置输出
+	if env != "dev" {
+		logger.AddHook(lfshook.NewHook(
+			lfshook.WriterMap{
+				logrus.DebugLevel: accessWriter,
+				logrus.InfoLevel:  accessWriter,
+				logrus.ErrorLevel: allWriter,
+				logrus.PanicLevel: allWriter,
+			},
+			&SaveFormatter{},
+		))
+	}
+	//设置日志级别
+	logger.SetLevel(logrus.DebugLevel)
+	Logger = logger
+	isInit = true
+	return logger
+}
+
+type ConsoleFormatter struct {
+}
+
+func (m *ConsoleFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	var b *bytes.Buffer
+	if entry.Buffer != nil {
+		b = entry.Buffer
+	} else {
+		b = &bytes.Buffer{}
+	}
+	timestamp := entry.Time.Format("2006-01-02 15:04:05")
+	var newLog string
+	var levelString aurora.Value
+	switch entry.Level.String() {
+	case "info":
+		levelString = aurora.Green(entry.Level)
+	case "warning":
+		levelString = aurora.Yellow(entry.Level)
+	case "debug":
+		levelString = aurora.Gray(16-1, entry.Level)
+	case "error":
+		levelString = aurora.Red(entry.Level)
+	case "fatal":
+		levelString = aurora.Red(entry.Level)
+	case "panic":
+		levelString = aurora.Red(entry.Level)
+	}
+	newLog = fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, levelString, entry.Message)
+	b.WriteString(newLog)
+	return b.Bytes(), nil
+}
+
+type SaveFormatter struct {
+}
+
+func (m *SaveFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	var b *bytes.Buffer
+	if entry.Buffer != nil {
+		b = entry.Buffer
+	} else {
+		b = &bytes.Buffer{}
+	}
+	timestamp := entry.Time.Format("2006-01-02 15:04:05")
+	newLog := fmt.Sprintf("[%s] [%s:%d] [%s] %s \n", timestamp, path.Base(entry.Caller.File), entry.Caller.Line, entry.Level, entry.Message)
+	b.WriteString(newLog)
+	return b.Bytes(), nil
+}

main_program/entity/Channel.go

@@ -0,0 +1,23 @@
+package entity
+
+type ChannelCopy struct {
+	Id                 int
+	ChannelId          string
+	ChannelTitle       string
+	ChannelLanguage    string
+	ChannelReptileTime *string
+	Is_Copy            int
+}
+
+type Channel struct {
+	Id                 uint    `gorm:"column:id;primaryKey;autoIncrement"`
+	ChannelId          string  `gorm:"column:channelId;type:varchar(255);not null"`
+	ChannelTitle       string  `gorm:"column:channelTitle;type:varchar(255);not null"`
+	ChannelLanguage    string  `gorm:"column:channelLanguage;type:varchar(255);not null"`
+	ChannelReptileTime *string `gorm:"column:channelReptileTime;type:varchar(255);default:null"`
+	Region             string  `gorm:"column:region;type:varchar(255);default:null"`
+}
+
+func (Channel) TableName() string {
+	return "Channel"
+}

main_program/go.mod

@@ -0,0 +1,27 @@
+module main_program
+
+go 1.22.2
+
+require (
+	github.com/sirupsen/logrus v1.9.3
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/lestrrat-go/strftime v1.0.6 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	gorm.io/gorm v1.25.7 // indirect
+)
+
+require (
+	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
+	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/tealeg/xlsx v1.0.5
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	gorm.io/driver/mysql v1.5.7
+)

main_program/go.sum

@@ -0,0 +1,43 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
+github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
+github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
+github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
+github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE=
+github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

main_program/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+
+	common "main_program/common"
+	"main_program/config"
+	move_data "main_program/moveData"
+
+	"gopkg.in/yaml.v2"
+)
+
+func init() {
+	flag.StringVar(&common.ConfigFile, "cf", "config.yml", "配置文件名")
+}
+
+func main() {
+	flag.Parse()
+	//读取配置文件
+	data, _ := ioutil.ReadFile(common.ConfigFile)
+	err := yaml.Unmarshal(data, &common.MyEnv)
+	if err != nil {
+		fmt.Println("读取配置文件错误...")
+		return
+	}
+	config.InitConfig(common.MyEnv.Log.LogEnv, common.MyEnv.Log.LogPath)
+	config.Logger.Info("初始化Logger成功...")
+	// 判断command
+	command := common.MyEnv.Command
+	config.Logger.Infof("Command:%s", common.MyEnv.Command)
+	if command == "move_data" {
+		// 判断参数
+		table := common.MyEnv.MoveData.Table
+		dbPath := common.MyEnv.MoveData.DBPath
+		if table == "" || dbPath == "" {
+			config.Logger.Error("move_data配置文件错误...")
+			return
+		}
+		config.Logger.Infof("开始调用move_data方法")
+		move_data.MoveDataService.Start(table, dbPath, common.MyEnv.Mysql)
+	}
+
+}

main_program/moveData/moveDataService.go

@@ -0,0 +1,105 @@
+package move_data
+
+import (
+	"container/list"
+	"database/sql"
+	"fmt"
+	"main_program/config"
+	"main_program/entity"
+
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/tealeg/xlsx"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+)
+
+type moveDataService struct{}
+
+var MoveDataService moveDataService
+
+func (m *moveDataService) Start(table string, sqlitePath string, mysqlConfig config.MysqlEntity) {
+	config.Logger.Info("开始迁移sqlite3到mysql")
+	sqliteDB, err := sql.Open("sqlite3", sqlitePath)
+	if err != nil {
+		config.Logger.Fatal(err)
+	}
+	config.Logger.Info("连接成功sqlite3...")
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", mysqlConfig.User, mysqlConfig.Password, mysqlConfig.Host, mysqlConfig.Database)
+	mysqlDB, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		config.Logger.Fatal("数据库连接失败")
+	}
+	config.Logger.Info("连接成功mysql...")
+	if table == "Channel" {
+		m.moveChannel(sqliteDB, mysqlDB)
+	}
+}
+
+func (m *moveDataService) moveChannel(sqliteDB *sql.DB, mysqlDB *gorm.DB) {
+	config.Logger.Info("读取xlsx获取region")
+	file, err := xlsx.OpenFile("D:/Work/Code/youtube_dev/youtube-golang/main_program/moveData/channel_region.xlsx")
+	if err != nil {
+		config.Logger.Fatalf("Error opening file: %s", err)
+	}
+	config.Logger.Info("开始迁移Channel表")
+	// 从sqlite3获取前50个Channel
+	continueFlag := true
+	for continueFlag {
+		rows, err := sqliteDB.Query("SELECT * FROM Channel_copy WHERE is_copy = 0 limit 50")
+		if err != nil {
+			config.Logger.Fatal(err)
+		}
+		ChannelList := list.New()
+		for rows.Next() {
+			channelCopy := new(entity.ChannelCopy)
+			if err := rows.Scan(&channelCopy.Id, &channelCopy.ChannelId, &channelCopy.ChannelTitle, &channelCopy.ChannelLanguage, &channelCopy.ChannelReptileTime, &channelCopy.Is_Copy); err != nil {
+				config.Logger.Fatal(err)
+			}
+			ChannelList.PushBack(channelCopy)
+		}
+		rows.Close()
+		if ChannelList.Len() <= 0 {
+			break
+		}
+		// 放入mysql里
+		for e := ChannelList.Front(); e != nil; e = e.Next() {
+			channelCopy := e.Value.(*entity.ChannelCopy)
+			config.Logger.Info(channelCopy)
+			channel := new(entity.Channel)
+			channel.ChannelId = channelCopy.ChannelId
+			channel.ChannelTitle = channelCopy.ChannelTitle
+			channel.ChannelLanguage = channelCopy.ChannelLanguage
+			channel.ChannelReptileTime = channelCopy.ChannelReptileTime
+			channel.Region = m.getRegionByChannelId(channel.ChannelId, file)
+			result := mysqlDB.Create(&channel)
+			if result.Error != nil {
+				config.Logger.Fatal(result.Error)
+			}
+			// 修改sqlite里状态
+			sqlStr, err := sqliteDB.Prepare("UPDATE Channel_copy SET is_copy = 1 WHERE id = ?")
+			if err != nil {
+				config.Logger.Error(err)
+			}
+			_, err = sqlStr.Exec(channelCopy.Id)
+			if err != nil {
+				config.Logger.Error(err)
+			}
+			sqlStr.Close()
+		}
+		tmpRows, _ := sqliteDB.Query("SELECT * FROM Channel_copy WHERE is_copy = 0 limit 50")
+		continueFlag = tmpRows.Next()
+		tmpRows.Close()
+	}
+
+}
+
+func (m *moveDataService) getRegionByChannelId(channnelId string, file *xlsx.File) string {
+	for _, sheet := range file.Sheets {
+		for _, row := range sheet.Rows {
+			if row.Cells[3].Value == channnelId {
+				return row.Cells[2].Value
+			}
+		}
+	}
+	return ""
+}