From 0885784f136b9b68653ec33fa733e962f3507f7e Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 12 Aug 2016 00:18:51 +0700 Subject: [PATCH] Wiki mirroring implementation (#3233) * Implement wiki mirroring, add Update mirrors operation to admin dashboard * bindata.go update after merge * Implement checking Git repo endpoint existence, support for BB included * Remove admin dashboard operation Fix bindata.go * Apply gofmt to repo model file * Try to remove bindata from PR * Revert accepted wiki names change in favor of better system * Remove unused imports --- models/repo.go | 101 +++++++++++++++++++++++++++++++-- modules/context/repo.go | 1 + routers/repo/setting.go | 2 +- templates/repo/wiki/start.tmpl | 2 +- templates/repo/wiki/view.tmpl | 2 +- 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/models/repo.go b/models/repo.go index 2902dc84a..8266b8f14 100644 --- a/models/repo.go +++ b/models/repo.go @@ -659,6 +659,51 @@ type MigrateRepoOptions struct { RemoteAddr string } +func isGitRepoURL(repoURL string, timeout time.Duration) bool { + cmd := git.NewCommand("ls-remote") + cmd.AddArguments("-q", "-h", repoURL, "HEAD") + res, err := cmd.RunTimeout(timeout) + if err != nil { + return false + } + if strings.Contains(res, "fatal") || strings.Contains(res, "not found") { + return false + } + return true +} + +func wikiRemoteURL(remote string, timeout time.Duration) string { + wikiRemoteStd := remote + wikiRemoteBitBucket := remote + /* + GitHub, GitLab, Gogs: NAME.wiki.git + BitBucket: NAME.git/wiki + */ + gitSuffixed := strings.HasSuffix(remote, ".git") + if gitSuffixed { + wikiRemoteStd = wikiRemoteStd[:len(wikiRemoteStd)-4] + wikiRemoteBitBucket += "/wiki" + } else { + wikiRemoteBitBucket += ".git/wiki" + } + wikiRemoteStd += ".wiki.git" + isBB := strings.Contains(remote, "bitbucket") + if isBB { + if isGitRepoURL(wikiRemoteBitBucket, timeout) { + return wikiRemoteBitBucket + } else if isGitRepoURL(wikiRemoteStd, timeout) { + return wikiRemoteStd + } + return "" + } + if isGitRepoURL(wikiRemoteStd, timeout) { + return wikiRemoteStd + } else if isGitRepoURL(wikiRemoteBitBucket, timeout) { + return wikiRemoteBitBucket + } + return "" +} + // MigrateRepository migrates a existing repository from other project hosting. func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) { repo, err := CreateRepository(u, CreateRepoOptions{ @@ -676,6 +721,7 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) { os.MkdirAll(tmpDir, os.ModePerm) repoPath := RepoPath(u.Name, opts.Name) + wikiPath := WikiPath(u.Name, opts.Name) if u.IsOrganization() { t, err := u.GetOwnerTeam() @@ -687,15 +733,28 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) { repo.NumWatches = 1 } + gitTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second os.RemoveAll(repoPath) if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, - Timeout: time.Duration(setting.Git.Timeout.Migrate) * time.Second, + Timeout: gitTimeout, }); err != nil { return repo, fmt.Errorf("Clone: %v", err) } + wikiRemotePath := wikiRemoteURL(opts.RemoteAddr, gitTimeout) + if wikiRemotePath != "" { + os.RemoveAll(wikiPath) + if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: gitTimeout, + }); err != nil { + log.Info("Clone wiki failed: %v", err) + } + } + // Check if repository is empty. _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") if err != nil { @@ -735,14 +794,22 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) { return repo, UpdateRepository(repo, false) } - return CleanUpMigrateInfo(repo, repoPath) + return CleanUpMigrateInfo(repo) } // Finish migrating repository with things that don't need to be done for mirrors. -func CleanUpMigrateInfo(repo *Repository, repoPath string) (*Repository, error) { +func CleanUpMigrateInfo(repo *Repository) (*Repository, error) { + repoPath := repo.RepoPath() + hasWiki := repo.HasWiki() + if err := createUpdateHook(repoPath); err != nil { return repo, fmt.Errorf("createUpdateHook: %v", err) } + if hasWiki { + if err := createUpdateHook(repoPath); err != nil { + return repo, fmt.Errorf("createUpdateHook: %v", err) + } + } // Clean up mirror info which prevents "push --all". // This also removes possible user credentials. @@ -755,6 +822,17 @@ func CleanUpMigrateInfo(repo *Repository, repoPath string) (*Repository, error) if err = cfg.SaveToIndent(configPath, "\t"); err != nil { return repo, fmt.Errorf("save config file: %v", err) } + if hasWiki { + wikiConfigPath := filepath.Join(repo.WikiPath(), "config") + cfg, err = ini.Load(wikiConfigPath) + if err != nil { + return repo, fmt.Errorf("open wiki config file: %v", err) + } + cfg.DeleteSection("remote \"origin\"") + if err = cfg.SaveToIndent(wikiConfigPath, "\t"); err != nil { + return repo, fmt.Errorf("save wiki config file: %v", err) + } + } return repo, UpdateRepository(repo, false) } @@ -1695,6 +1773,8 @@ func MirrorUpdate() { } repoPath := m.Repo.RepoPath() + wikiPath := m.Repo.WikiPath() + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second gitArgs := []string{"remote", "update"} if m.EnablePrune { @@ -1702,8 +1782,7 @@ func MirrorUpdate() { } if _, stderr, err := process.ExecDir( - time.Duration(setting.Git.Timeout.Mirror)*time.Second, - repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath), + timeout, repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath), "git", gitArgs...); err != nil { desc := fmt.Sprintf("Fail to update mirror repository(%s): %s", repoPath, stderr) log.Error(4, desc) @@ -1712,6 +1791,18 @@ func MirrorUpdate() { } return nil } + if m.Repo.HasWiki() { + if _, stderr, err := process.ExecDir( + timeout, wikiPath, fmt.Sprintf("MirrorUpdate: %s", wikiPath), + "git", "remote", "update", "--prune"); err != nil { + desc := fmt.Sprintf("Fail to update mirror wiki repository(%s): %s", wikiPath, stderr) + log.Error(4, desc) + if err = CreateRepositoryNotice(desc); err != nil { + log.Error(4, "CreateRepositoryNotice: %v", err) + } + return nil + } + } m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour) mirrors = append(mirrors, m) diff --git a/modules/context/repo.go b/modules/context/repo.go index af1888148..ea3b445e3 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -223,6 +223,7 @@ func RepoAssignment(args ...bool) macaron.Handler { ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner() ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin() ctx.Data["IsRepositoryWriter"] = ctx.Repo.IsWriter() + ctx.Data["IsRepositoryMirror"] = repo.IsMirror ctx.Data["DisableSSH"] = setting.SSH.Disabled ctx.Data["CloneLink"] = repo.CloneLink() diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 08f06dcb3..33ed5b8aa 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -174,7 +174,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { } repo.IsMirror = false - if _, err := models.CleanUpMigrateInfo(repo, models.RepoPath(ctx.Repo.Owner.Name, repo.Name)); err != nil { + if _, err := models.CleanUpMigrateInfo(repo); err != nil { ctx.Handle(500, "CleanUpMigrateInfo", err) return } else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil { diff --git a/templates/repo/wiki/start.tmpl b/templates/repo/wiki/start.tmpl index e74627f84..674199198 100644 --- a/templates/repo/wiki/start.tmpl +++ b/templates/repo/wiki/start.tmpl @@ -6,7 +6,7 @@

{{.i18n.Tr "repo.wiki.welcome"}}

{{.i18n.Tr "repo.wiki.welcome_desc"}}

- {{if .IsRepositoryWriter}} + {{if and .IsRepositoryWriter (not .IsRepositoryMirror)}} {{.i18n.Tr "repo.wiki.create_first_page"}} {{end}} diff --git a/templates/repo/wiki/view.tmpl b/templates/repo/wiki/view.tmpl index 833f372f4..9bc6ece76 100644 --- a/templates/repo/wiki/view.tmpl +++ b/templates/repo/wiki/view.tmpl @@ -46,7 +46,7 @@
{{.title}} - {{if .IsRepositoryWriter}} + {{if and .IsRepositoryWriter (not .IsRepositoryMirror)}}
{{.i18n.Tr "repo.wiki.edit_page_button"}} {{.i18n.Tr "repo.wiki.new_page_button"}}