WIP: Reimplement dump and implment restore command

cmd/dump.go

@@ -44,10 +44,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
 			Value: os.TempDir(),
 			Usage: "Temporary dir path",
 		},
-		cli.StringFlag{
-			Name:  "database, d",
-			Usage: "Specify the database SQL syntax",
-		},
 		cli.BoolFlag{
 			Name:  "skip-repository, R",
 			Usage: "Skip the repository dumping",
@@ -83,10 +79,9 @@ func runDump(ctx *cli.Context) error {
 		os.Setenv("TMPDIR", tmpWorkDir)
 	}
 
-	dbDump := path.Join(tmpWorkDir, "gitea-db.sql")
+	log.Printf("Packing dump files...")
 
 	fileName := fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix())
-	log.Printf("Packing dump files...")
 	z, err := zip.Create(fileName)
 	if err != nil {
 		log.Fatalf("Failed to create %s: %v", fileName, err)
@@ -106,20 +101,23 @@ func runDump(ctx *cli.Context) error {
 		}
 	}
 
-	targetDBType := ctx.String("database")
-	if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type {
-		log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType)
-	} else {
-		log.Printf("Dumping database...")
+	log.Printf("Dumping database...")
+
+	dbDump := path.Join(tmpWorkDir, "database")
+	if err := os.MkdirAll(dbDump, os.ModePerm); err != nil {
+		log.Fatalf("Failed to create database dir: %v", err)
 	}
 
-	if err := models.DumpDatabase(dbDump, targetDBType); err != nil {
+	if err := models.DumpDatabaseFixtures(dbDump); err != nil {
 		log.Fatalf("Failed to dump database: %v", err)
 	}
 
-	if err := z.AddFile("gitea-db.sql", dbDump); err != nil {
-		log.Fatalf("Failed to include gitea-db.sql: %v", err)
+	if err := z.AddDir("database", dbDump); err != nil {
+		log.Fatalf("Failed to include database: %v", err)
 	}
+
+	log.Printf("Dumping custom directory ... %s", setting.CustomPath)
+
 	customDir, err := os.Stat(setting.CustomPath)
 	if err == nil && customDir.IsDir() {
 		if err := z.AddDir("custom", setting.CustomPath); err != nil {
@@ -130,7 +128,7 @@ func runDump(ctx *cli.Context) error {
 	}
 
 	if com.IsExist(setting.AppDataPath) {
-		log.Printf("Packing data directory...%s", setting.AppDataPath)
+		log.Printf("Dumping data directory ... %s", setting.AppDataPath)
 
 		var sessionAbsPath string
 		if setting.SessionConfig.Provider == "file" {
@@ -141,8 +139,19 @@ func runDump(ctx *cli.Context) error {
 		}
 	}
 
-	if err := z.AddDir("log", setting.LogRootPath); err != nil {
-		log.Fatalf("Failed to include log: %v", err)
+	verPath := filepath.Join(tmpWorkDir, "VERSION")
+	verf, err := os.Create(verPath)
+	if err != nil {
+		log.Fatalf("Failed to create version file: %v", err)
+	}
+	_, err = verf.WriteString(setting.AppVer)
+	verf.Close()
+	if err != nil {
+		log.Fatalf("Failed to write version to file: %v", err)
+	}
+
+	if err = z.AddFile("VERSION", verPath); err != nil {
+		log.Fatalf("Failed to add version file: %v", err)
 	}
 
 	if err = z.Close(); err != nil {

cmd/restore.go

@@ -0,0 +1,153 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/Unknwon/cae/zip"
+	"github.com/Unknwon/com"
+	"github.com/urfave/cli"
+)
+
+// CmdRestore represents the available restore sub-command.
+var CmdRestore = cli.Command{
+	Name:  "restore",
+	Usage: "Restore Gitea files and database",
+	Description: `Restore will restore all data from zip file which dumped from gitea. It will use 
+the custom config in this dump zip file, this operation will remove all the dest database and repositories.`,
+	Action: runRestore,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "config, c",
+			Value: "custom/conf/app.ini",
+			Usage: "Custom configuration file path, if empty will use dumped config file",
+		},
+		cli.BoolFlag{
+			Name:  "verbose, v",
+			Usage: "Show process details",
+		},
+		cli.StringFlag{
+			Name:  "tempdir, t",
+			Value: os.TempDir(),
+			Usage: "Temporary dir path",
+		},
+	},
+}
+
+func runRestore(ctx *cli.Context) error {
+	if len(os.Args) < 3 {
+		return errors.New("need zip file path")
+	}
+
+	tmpDir := ctx.String("tempdir")
+	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
+		log.Fatalf("Path does not exist: %s", tmpDir)
+	}
+	tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-restore-")
+	if err != nil {
+		log.Fatalf("Failed to create tmp work directory: %v", err)
+	}
+	log.Printf("Creating tmp work dir: %s", tmpWorkDir)
+
+	// work-around #1103
+	if os.Getenv("TMPDIR") == "" {
+		os.Setenv("TMPDIR", tmpWorkDir)
+	}
+
+	srcPath := os.Args[2]
+
+	zip.Verbose = ctx.Bool("verbose")
+	log.Printf("Extracting %s to %s", srcPath, tmpWorkDir)
+	err = zip.ExtractTo(srcPath, tmpWorkDir)
+	if err != nil {
+		log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err)
+	}
+
+	verData, err := ioutil.ReadFile(filepath.Join(tmpWorkDir, "VERSION"))
+	if err != nil {
+		log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err)
+	}
+
+	if setting.AppVer != string(verData) {
+		log.Fatalf("Expected gitea version to restore is %s, but get %s", string(verData), setting.AppVer)
+	}
+
+	if ctx.IsSet("config") {
+		setting.CustomConf = ctx.String("config")
+	} else {
+		setting.CustomConf = filepath.Join(tmpWorkDir, "custom", "conf", "app.ini")
+	}
+	if !com.IsExist(setting.CustomConf) {
+		log.Fatalf("Failed to load ini config file from %s", setting.CustomConf)
+	}
+
+	setting.NewContext()
+	//setting.CustomPath = filepath.Join(tmpWorkDir, "custom")
+	setting.NewXORMLogService(false)
+	models.LoadConfigs()
+
+	err = models.SetEngine()
+	if err != nil {
+		log.Fatalf("Failed to SetEngine: %v", err)
+	}
+
+	err = models.SyncDBStructs()
+	if err != nil {
+		log.Fatalf("Failed to SyncDBStructs: %v", err)
+	}
+
+	log.Printf("Restoring repo dir to %s ...", setting.RepoRootPath)
+	repoPath := filepath.Join(tmpWorkDir, "repositories")
+	err = os.RemoveAll(setting.RepoRootPath)
+	if err != nil {
+		log.Fatalf("Failed to Remove repo root path %s: %v", setting.RepoRootPath, err)
+	}
+
+	err = os.Rename(repoPath, setting.RepoRootPath)
+	if err != nil {
+		log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err)
+	}
+
+	log.Printf("Restoring custom dir to %s ...", setting.CustomPath)
+	customPath := filepath.Join(tmpWorkDir, "custom")
+	err = os.RemoveAll(setting.CustomPath)
+	if err != nil {
+		log.Fatalf("Failed to Remove repo root path %s: %v", setting.CustomPath, err)
+	}
+
+	err = os.Rename(customPath, setting.CustomPath)
+	if err != nil {
+		log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err)
+	}
+
+	log.Printf("Restoring data dir to %s ...", setting.AppDataPath)
+	dataPath := filepath.Join(tmpWorkDir, "data")
+	err = os.RemoveAll(setting.AppDataPath)
+	if err != nil {
+		log.Fatalf("Failed to Remove data root path %s: %v", setting.AppDataPath, err)
+	}
+
+	err = os.Rename(dataPath, setting.AppDataPath)
+	if err != nil {
+		log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err)
+	}
+
+	dbPath := filepath.Join(tmpWorkDir, "database")
+	log.Printf("Restoring database from %s ...", dbPath)
+	err = models.RestoreDatabaseFixtures(dbPath)
+	if err != nil {
+		log.Fatalf("Failed to restore database dir %s: %v", dbPath, err)
+	}
+
+	return nil
+}

main.go

@@ -46,6 +46,7 @@ arguments - which can alternatively be run by running the subcommand web.`
 		cmd.CmdServ,
 		cmd.CmdHook,
 		cmd.CmdDump,
+		cmd.CmdRestore,
 		cmd.CmdCert,
 		cmd.CmdAdmin,
 		cmd.CmdGenerate,

models/models.go

@@ -9,10 +9,12 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
@@ -22,6 +24,7 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/go-xorm/core"
 	"github.com/go-xorm/xorm"
+	"gopkg.in/yaml.v2"
 
 	// Needed for the Postgresql driver
 	_ "github.com/lib/pq"
@@ -301,6 +304,15 @@ func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) {
 	return nil
 }
 
+// SyncDBStructs will sync database structs
+func SyncDBStructs() error {
+	if err := x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
+		return fmt.Errorf("sync database struct error: %v", err)
+	}
+
+	return nil
+}
+
 // Statistic contains the database statistics
 type Statistic struct {
 	Counter struct {
@@ -360,3 +372,113 @@ func DumpDatabase(filePath string, dbType string) error {
 	}
 	return x.DumpTablesToFile(tbs, filePath)
 }
+
+// DumpDatabaseFixtures dumps all data from database to fixtures files on dirPath
+func DumpDatabaseFixtures(dirPath string) error {
+	for _, t := range tables {
+		if err := dumpTableFixtures(t, dirPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func dumpTableFixtures(bean interface{}, dirPath string) error {
+	table := x.TableInfo(bean)
+	f, err := os.Create(filepath.Join(dirPath, table.Name+".yml"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	const bufferSize = 100
+	var start = 0
+	for {
+		objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface()
+		if err != nil {
+			return err
+		}
+		if len(objs) == 0 {
+			break
+		}
+
+		data, err := yaml.Marshal(objs)
+		if err != nil {
+			return err
+		}
+		_, err = f.Write(data)
+		if err != nil {
+			return err
+		}
+		if len(objs) < bufferSize {
+			break
+		}
+		start += len(objs)
+	}
+
+	return nil
+}
+
+// RestoreDatabaseFixtures restores all data from dir to database
+func RestoreDatabaseFixtures(dirPath string) error {
+	for _, t := range tables {
+		if err := restoreTableFixtures(t, dirPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func restoreTableFixtures(bean interface{}, dirPath string) error {
+	table := x.TableInfo(bean)
+	data, err := ioutil.ReadFile(filepath.Join(dirPath, table.Name+".yml"))
+	if err != nil {
+		return err
+	}
+
+	const bufferSize = 100
+	var records = make([]map[string]interface{}, 0, bufferSize*10)
+	err = yaml.Unmarshal(data, &records)
+	if err != nil {
+		return err
+	}
+
+	if len(records) == 0 {
+		return nil
+	}
+
+	var columns = make([]string, 0, len(records[0]))
+	for k := range records[0] {
+		columns = append(columns, k)
+	}
+	sort.Strings(columns)
+
+	qm := strings.Repeat("?,", len(columns))
+	qm = "(" + qm[:len(qm)-1] + ")"
+
+	_, err = x.Exec("DELETE FROM `" + table.Name + "`")
+	if err != nil {
+		return err
+	}
+
+	var sql = "INSERT INTO `" + table.Name + "` (`" + strings.Join(columns, "`,`") + "`) VALUES "
+	var args = make([]interface{}, 0, bufferSize)
+	var insertSQLs = make([]string, 0, bufferSize)
+	for i, vals := range records {
+		insertSQLs = append(insertSQLs, qm)
+		for _, colName := range columns {
+			args = append(args, vals[colName])
+		}
+
+		if i+1%100 == 0 || i == len(records)-1 {
+			_, err = x.Exec(sql+strings.Join(insertSQLs, ","), args...)
+			if err != nil {
+				return err
+			}
+			insertSQLs = make([]string, 0, bufferSize)
+			args = make([]interface{}, 0, bufferSize)
+		}
+	}
+
+	return err
+}