diff --git a/main_program/config.yml b/main_program/config.yml index c333a89..8cfa0cd 100644 --- a/main_program/config.yml +++ b/main_program/config.yml @@ -3,7 +3,7 @@ log: logpath: main_program command: move_data movedata: - table: Channel + table: Videos dbpath: D:/Work/Code/youtube_dev/youtube_prod.db mysql: host: 47.108.20.249:3306 diff --git a/main_program/entity/Video.go b/main_program/entity/Video.go new file mode 100644 index 0000000..7dbe857 --- /dev/null +++ b/main_program/entity/Video.go @@ -0,0 +1,30 @@ +package entity + +type VideoCopy struct { + Id int + VideoId string + ChannelId string + VideoTitle string + VideoLen int + VideoType string + VideoPublishTime string + VideoLanguage string + IsDownload int + IsCopy int +} + +type Video struct { + Id uint `gorm:"column:id;primaryKey;autoIncrement"` + VideoId string `gorm:"column:videoId;type:varchar(255);not null"` + ChannelId string `gorm:"column:channelId;type:varchar(255);not null"` + VideoTitle string `gorm:"column:videoTitle;type:varchar(255);not null"` + VideoLen int `gorm:"column:videoLen;type:int;not null"` + VideoType string `gorm:"column:videoType;type:varchar(255);not null"` + VideoPublishTime string `gorm:"column:videoPublishTime;type:varchar(255);not null"` + VideoLanguage string `gorm:"column:videoLanguage;type:varchar(255);not null"` + IsDownload int `gorm:"column:isDownload;type:int;not null"` +} + +func (Video) TableName() string { + return "Videos" +} diff --git a/main_program/moveData/moveDataService.go b/main_program/moveData/moveDataService.go index 40171dc..db5567f 100644 --- a/main_program/moveData/moveDataService.go +++ b/main_program/moveData/moveDataService.go @@ -6,6 +6,7 @@ import ( "fmt" "main_program/config" "main_program/entity" + "regexp" _ "github.com/mattn/go-sqlite3" "github.com/tealeg/xlsx" @@ -18,7 +19,7 @@ type moveDataService struct{} var MoveDataService moveDataService func (m *moveDataService) Start(table string, sqlitePath string, mysqlConfig config.MysqlEntity) { - config.Logger.Info("开始迁移sqlite3到mysql") + config.Logger.Info("开始迁移sqlite3到mysql...") sqliteDB, err := sql.Open("sqlite3", sqlitePath) if err != nil { config.Logger.Fatal(err) @@ -27,21 +28,23 @@ func (m *moveDataService) Start(table string, sqlitePath string, mysqlConfig con dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", mysqlConfig.User, mysqlConfig.Password, mysqlConfig.Host, mysqlConfig.Database) mysqlDB, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) if err != nil { - config.Logger.Fatal("数据库连接失败") + config.Logger.Fatal("数据库连接失败...") } config.Logger.Info("连接成功mysql...") if table == "Channel" { m.moveChannel(sqliteDB, mysqlDB) + } else if table == "Videos" { + m.moveVideos(sqliteDB, mysqlDB) } } func (m *moveDataService) moveChannel(sqliteDB *sql.DB, mysqlDB *gorm.DB) { - config.Logger.Info("读取xlsx获取region") + config.Logger.Info("读取xlsx获取region...") file, err := xlsx.OpenFile("D:/Work/Code/youtube_dev/youtube-golang/main_program/moveData/channel_region.xlsx") if err != nil { config.Logger.Fatalf("Error opening file: %s", err) } - config.Logger.Info("开始迁移Channel表") + config.Logger.Info("开始迁移Channel表...") // 从sqlite3获取前50个Channel continueFlag := true for continueFlag { @@ -90,7 +93,7 @@ func (m *moveDataService) moveChannel(sqliteDB *sql.DB, mysqlDB *gorm.DB) { continueFlag = tmpRows.Next() tmpRows.Close() } - + config.Logger.Info("完成迁移Channel表...") } func (m *moveDataService) getRegionByChannelId(channnelId string, file *xlsx.File) string { @@ -103,3 +106,65 @@ func (m *moveDataService) getRegionByChannelId(channnelId string, file *xlsx.Fil } return "" } + +func (m *moveDataService) moveVideos(sqliteDB *sql.DB, mysqlDB *gorm.DB) { + config.Logger.Info("开始迁移Videos表...") + continueFlag := true + count := 0 + for continueFlag { + rows, err := sqliteDB.Query("SELECT * FROM Vidoes_copy WHERE isCopy = 0 and channelId != '' limit 1000") + if err != nil { + config.Logger.Fatal(err) + } + videoList := list.New() + for rows.Next() { + videoCopy := new(entity.VideoCopy) + if err := rows.Scan(&videoCopy.Id, &videoCopy.VideoId, &videoCopy.ChannelId, &videoCopy.VideoTitle, &videoCopy.VideoLen, + &videoCopy.VideoType, &videoCopy.VideoPublishTime, + &videoCopy.VideoLanguage, &videoCopy.IsDownload, &videoCopy.IsCopy); err != nil { + config.Logger.Fatal(err) + } + videoList.PushBack(videoCopy) + } + rows.Close() + var videos []entity.Video + for e := videoList.Front(); e != nil; e = e.Next() { + videoCpoy := e.Value.(*entity.VideoCopy) + video := new(entity.Video) + video.VideoId = videoCpoy.VideoId + video.ChannelId = videoCpoy.ChannelId + re := regexp.MustCompile("[\U00010000-\U0010ffff\u2600-\u27BF\u1f300-\u1f64F\u1f680-\u1f6FF\u2700-\u27BF]+") + video.VideoTitle = re.ReplaceAllString(videoCpoy.VideoTitle, "") + video.VideoLen = videoCpoy.VideoLen + video.VideoType = videoCpoy.VideoType + video.VideoPublishTime = videoCpoy.VideoPublishTime + video.VideoLanguage = videoCpoy.VideoLanguage + video.IsDownload = videoCpoy.IsDownload + videos = append(videos, *video) + } + result := mysqlDB.CreateInBatches(videos, 100) + if result.Error != nil { + config.Logger.Error(result.Error) + break + } + for e := videoList.Front(); e != nil; e = e.Next() { + videoCpoy := e.Value.(*entity.VideoCopy) + // 修改sqlite里状态 + sqlStr, err := sqliteDB.Prepare("UPDATE Vidoes_copy SET isCopy = 1 WHERE id = ?") + if err != nil { + config.Logger.Error(err) + } + _, err = sqlStr.Exec(videoCpoy.Id) + if err != nil { + config.Logger.Error(err) + } + sqlStr.Close() + } + count += 1 + config.Logger.Infof("count:%d", count) + tmpRows, _ := sqliteDB.Query("SELECT * FROM Vidoes_copy WHERE isCopy = 0 and channelId != '' limit 1") + continueFlag = tmpRows.Next() + tmpRows.Close() + } + config.Logger.Info("完成迁移Videos表...") +}