迁移Videos表

main_program/config.yml

@@ -3,7 +3,7 @@ log:
   logpath: main_program
 command: move_data
 movedata:
-  table: Channel
+  table: Videos
   dbpath: D:/Work/Code/youtube_dev/youtube_prod.db
 mysql:
   host: 47.108.20.249:3306

main_program/entity/Video.go

@@ -0,0 +1,30 @@
+package entity
+
+type VideoCopy struct {
+	Id               int
+	VideoId          string
+	ChannelId        string
+	VideoTitle       string
+	VideoLen         int
+	VideoType        string
+	VideoPublishTime string
+	VideoLanguage    string
+	IsDownload       int
+	IsCopy           int
+}
+
+type Video struct {
+	Id               uint   `gorm:"column:id;primaryKey;autoIncrement"`
+	VideoId          string `gorm:"column:videoId;type:varchar(255);not null"`
+	ChannelId        string `gorm:"column:channelId;type:varchar(255);not null"`
+	VideoTitle       string `gorm:"column:videoTitle;type:varchar(255);not null"`
+	VideoLen         int    `gorm:"column:videoLen;type:int;not null"`
+	VideoType        string `gorm:"column:videoType;type:varchar(255);not null"`
+	VideoPublishTime string `gorm:"column:videoPublishTime;type:varchar(255);not null"`
+	VideoLanguage    string `gorm:"column:videoLanguage;type:varchar(255);not null"`
+	IsDownload       int    `gorm:"column:isDownload;type:int;not null"`
+}
+
+func (Video) TableName() string {
+	return "Videos"
+}

main_program/moveData/moveDataService.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"main_program/config"
 	"main_program/entity"
+	"regexp"
 
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/tealeg/xlsx"
@@ -18,7 +19,7 @@ type moveDataService struct{}
 var MoveDataService moveDataService
 
 func (m *moveDataService) Start(table string, sqlitePath string, mysqlConfig config.MysqlEntity) {
-	config.Logger.Info("开始迁移sqlite3到mysql")
+	config.Logger.Info("开始迁移sqlite3到mysql...")
 	sqliteDB, err := sql.Open("sqlite3", sqlitePath)
 	if err != nil {
 		config.Logger.Fatal(err)
@@ -27,21 +28,23 @@ func (m *moveDataService) Start(table string, sqlitePath string, mysqlConfig con
 	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", mysqlConfig.User, mysqlConfig.Password, mysqlConfig.Host, mysqlConfig.Database)
 	mysqlDB, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
 	if err != nil {
-		config.Logger.Fatal("数据库连接失败")
+		config.Logger.Fatal("数据库连接失败...")
 	}
 	config.Logger.Info("连接成功mysql...")
 	if table == "Channel" {
 		m.moveChannel(sqliteDB, mysqlDB)
+	} else if table == "Videos" {
+		m.moveVideos(sqliteDB, mysqlDB)
 	}
 }
 
 func (m *moveDataService) moveChannel(sqliteDB *sql.DB, mysqlDB *gorm.DB) {
-	config.Logger.Info("读取xlsx获取region")
+	config.Logger.Info("读取xlsx获取region...")
 	file, err := xlsx.OpenFile("D:/Work/Code/youtube_dev/youtube-golang/main_program/moveData/channel_region.xlsx")
 	if err != nil {
 		config.Logger.Fatalf("Error opening file: %s", err)
 	}
-	config.Logger.Info("开始迁移Channel表")
+	config.Logger.Info("开始迁移Channel表...")
 	// 从sqlite3获取前50个Channel
 	continueFlag := true
 	for continueFlag {
@@ -90,7 +93,7 @@ func (m *moveDataService) moveChannel(sqliteDB *sql.DB, mysqlDB *gorm.DB) {
 		continueFlag = tmpRows.Next()
 		tmpRows.Close()
 	}
-
+	config.Logger.Info("完成迁移Channel表...")
 }
 
 func (m *moveDataService) getRegionByChannelId(channnelId string, file *xlsx.File) string {
@@ -103,3 +106,65 @@ func (m *moveDataService) getRegionByChannelId(channnelId string, file *xlsx.Fil
 	}
 	return ""
 }
+
+func (m *moveDataService) moveVideos(sqliteDB *sql.DB, mysqlDB *gorm.DB) {
+	config.Logger.Info("开始迁移Videos表...")
+	continueFlag := true
+	count := 0
+	for continueFlag {
+		rows, err := sqliteDB.Query("SELECT * FROM Vidoes_copy WHERE isCopy = 0 and channelId != '' limit 1000")
+		if err != nil {
+			config.Logger.Fatal(err)
+		}
+		videoList := list.New()
+		for rows.Next() {
+			videoCopy := new(entity.VideoCopy)
+			if err := rows.Scan(&videoCopy.Id, &videoCopy.VideoId, &videoCopy.ChannelId, &videoCopy.VideoTitle, &videoCopy.VideoLen,
+				&videoCopy.VideoType, &videoCopy.VideoPublishTime,
+				&videoCopy.VideoLanguage, &videoCopy.IsDownload, &videoCopy.IsCopy); err != nil {
+				config.Logger.Fatal(err)
+			}
+			videoList.PushBack(videoCopy)
+		}
+		rows.Close()
+		var videos []entity.Video
+		for e := videoList.Front(); e != nil; e = e.Next() {
+			videoCpoy := e.Value.(*entity.VideoCopy)
+			video := new(entity.Video)
+			video.VideoId = videoCpoy.VideoId
+			video.ChannelId = videoCpoy.ChannelId
+			re := regexp.MustCompile("[\U00010000-\U0010ffff\u2600-\u27BF\u1f300-\u1f64F\u1f680-\u1f6FF\u2700-\u27BF]+")
+			video.VideoTitle = re.ReplaceAllString(videoCpoy.VideoTitle, "")
+			video.VideoLen = videoCpoy.VideoLen
+			video.VideoType = videoCpoy.VideoType
+			video.VideoPublishTime = videoCpoy.VideoPublishTime
+			video.VideoLanguage = videoCpoy.VideoLanguage
+			video.IsDownload = videoCpoy.IsDownload
+			videos = append(videos, *video)
+		}
+		result := mysqlDB.CreateInBatches(videos, 100)
+		if result.Error != nil {
+			config.Logger.Error(result.Error)
+			break
+		}
+		for e := videoList.Front(); e != nil; e = e.Next() {
+			videoCpoy := e.Value.(*entity.VideoCopy)
+			// 修改sqlite里状态
+			sqlStr, err := sqliteDB.Prepare("UPDATE Vidoes_copy SET isCopy = 1 WHERE id = ?")
+			if err != nil {
+				config.Logger.Error(err)
+			}
+			_, err = sqlStr.Exec(videoCpoy.Id)
+			if err != nil {
+				config.Logger.Error(err)
+			}
+			sqlStr.Close()
+		}
+		count += 1
+		config.Logger.Infof("count:%d", count)
+		tmpRows, _ := sqliteDB.Query("SELECT * FROM Vidoes_copy WHERE isCopy = 0 and channelId != '' limit 1")
+		continueFlag = tmpRows.Next()
+		tmpRows.Close()
+	}
+	config.Logger.Info("完成迁移Videos表...")
+}