From f86afb04a2af72c53443e38899ea4c2a87bd581d Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 12:11:38 +0100 Subject: [PATCH 1/4] Adding more error handling in dump cmd The dump cmd did not check the return value of the z.AddFile or z.AddDir when building the final archive. It caused the dump command to succeed even if an error occurred. The resulted dump archive could be corrupted/empty. (errors could be various: removal by a concurrent process, disk full, bugs in the dump cmd itself) --- cmd/dump.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 0bf385d06..279099dac 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -60,10 +60,18 @@ func runDump(ctx *cli.Context) { } workDir, _ := setting.WorkDir() - z.AddFile("gogs-repo.zip", path.Join(workDir, "gogs-repo.zip")) - z.AddFile("gogs-db.sql", path.Join(workDir, "gogs-db.sql")) - z.AddDir("custom", path.Join(workDir, "custom")) - z.AddDir("log", path.Join(workDir, "log")) + if err := z.AddFile("gogs-repo.zip", path.Join(workDir, "gogs-repo.zip")); err !=nil { + log.Fatalf("Fail to include gogs-repo.zip: %v", err) + } + if err := z.AddFile("gogs-db.sql", path.Join(workDir, "gogs-db.sql")); err !=nil { + log.Fatalf("Fail to include gogs-db.sql: %v", err) + } + if err := z.AddDir("custom", path.Join(workDir, "custom")); err !=nil { + log.Fatalf("Fail to include custom: %v", err) + } + if err := z.AddDir("log", path.Join(workDir, "log")); err !=nil { + log.Fatalf("Fail to include log: %v", err) + } // FIXME: SSH key file. if err = z.Close(); err != nil { os.Remove(fileName) From c5a9be91152de507ade9a98674d9eadfc6d1198b Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 14:07:51 +0100 Subject: [PATCH 2/4] Using a tmp dir to generate db and repo dumps Using a tmp dir makes gogs dump more robust to concurrent runs. It also permits an easier cleaning of the tmp files (gogs-db.sql and gog-repo.zip) by just removing the tmp dir. As a side effect, it partially fix bugs on workdir. Previously, 'gogs dump' created the archives in the current directory, and tried to include these archives from the directory where the gogs binary lies. ex: if gogs binary is in /usr/bin/gogs, and gogs dump is run from /tmp/, /tmp/gog-repo.zip is created, but gogs dump tried to include /usr/bin/gogs-repo.zip. --- cmd/dump.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 279099dac..0c07f36cf 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -11,6 +11,8 @@ import ( "path" "time" + "io/ioutil" + "github.com/Unknwon/cae/zip" "github.com/codegangsta/cli" @@ -38,16 +40,23 @@ func runDump(ctx *cli.Context) { models.LoadConfigs() models.SetEngine() + TmpWorkDir, err := ioutil.TempDir(os.TempDir(), "gogs-dump-") + if err != nil { + log.Fatalf("Fail to create tmp work directory: %v", err) + } + log.Printf("Creating tmp work dir: %s", TmpWorkDir) + + reposDump := path.Join(TmpWorkDir, "gogs-repo.zip") + dbDump := path.Join(TmpWorkDir, "gogs-db.sql") + log.Printf("Dumping local repositories...%s", setting.RepoRootPath) zip.Verbose = ctx.Bool("verbose") - defer os.Remove("gogs-repo.zip") - if err := zip.PackTo(setting.RepoRootPath, "gogs-repo.zip", true); err != nil { + if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil { log.Fatalf("Fail to dump local repositories: %v", err) } log.Printf("Dumping database...") - defer os.Remove("gogs-db.sql") - if err := models.DumpDatabase("gogs-db.sql"); err != nil { + if err := models.DumpDatabase(dbDump); err != nil { log.Fatalf("Fail to dump database: %v", err) } @@ -60,10 +69,10 @@ func runDump(ctx *cli.Context) { } workDir, _ := setting.WorkDir() - if err := z.AddFile("gogs-repo.zip", path.Join(workDir, "gogs-repo.zip")); err !=nil { + if err := z.AddFile("gogs-repo.zip", reposDump); err !=nil { log.Fatalf("Fail to include gogs-repo.zip: %v", err) } - if err := z.AddFile("gogs-db.sql", path.Join(workDir, "gogs-db.sql")); err !=nil { + if err := z.AddFile("gogs-db.sql", dbDump); err !=nil { log.Fatalf("Fail to include gogs-db.sql: %v", err) } if err := z.AddDir("custom", path.Join(workDir, "custom")); err !=nil { @@ -78,5 +87,7 @@ func runDump(ctx *cli.Context) { log.Fatalf("Fail to save %s: %v", fileName, err) } + log.Printf("Removing tmp work dir: %s", TmpWorkDir) + os.RemoveAll(TmpWorkDir) log.Println("Finish dumping!") } From a59b1fcc21f35bafa16b65ac8c722e14d2dc925e Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 15:08:50 +0100 Subject: [PATCH 3/4] Fix dump of log and custom directory in dump cmd Now, the dump cmd uses setting.CustomPath and setting.LogRootPath instead of setting.WorkDir which was kind of broken if the gogs binary was in a different directory than gogs data. Additionally, the backup of setting.CustomPath directory is only done if it exists. --- cmd/dump.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 0c07f36cf..e8354784c 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -68,17 +68,21 @@ func runDump(ctx *cli.Context) { log.Fatalf("Fail to create %s: %v", fileName, err) } - workDir, _ := setting.WorkDir() if err := z.AddFile("gogs-repo.zip", reposDump); err !=nil { log.Fatalf("Fail to include gogs-repo.zip: %v", err) } if err := z.AddFile("gogs-db.sql", dbDump); err !=nil { log.Fatalf("Fail to include gogs-db.sql: %v", err) } - if err := z.AddDir("custom", path.Join(workDir, "custom")); err !=nil { - log.Fatalf("Fail to include custom: %v", err) + customDir, err := os.Stat(setting.CustomPath) + if err == nil && customDir.IsDir() { + if err := z.AddDir("custom", setting.CustomPath); err !=nil { + log.Fatalf("Fail to include custom: %v", err) + } + } else { + log.Printf("Custom dir %s doesn't exist, skipped", setting.CustomPath) } - if err := z.AddDir("log", path.Join(workDir, "log")); err !=nil { + if err := z.AddDir("log", setting.LogRootPath); err !=nil { log.Fatalf("Fail to include log: %v", err) } // FIXME: SSH key file. From 1d7a1b6034e9e9a699812196c04cacba0d6eb957 Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 15:22:10 +0100 Subject: [PATCH 4/4] add name of the dump file in last log message --- cmd/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dump.go b/cmd/dump.go index e8354784c..90082667b 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -93,5 +93,5 @@ func runDump(ctx *cli.Context) { log.Printf("Removing tmp work dir: %s", TmpWorkDir) os.RemoveAll(TmpWorkDir) - log.Println("Finish dumping!") + log.Printf("Finish dumping in file %s", fileName) }