From 61619c84d0fc511f2532f8c82d98fe971da69447 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 6 Mar 2024 13:09:38 +0800 Subject: [PATCH 01/17] Fix 500 error when adding PR comment (#29622) --- routers/web/repo/pull_review.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/routers/web/repo/pull_review.go b/routers/web/repo/pull_review.go index 64212291e162e..bce807aacdc9f 100644 --- a/routers/web/repo/pull_review.go +++ b/routers/web/repo/pull_review.go @@ -10,6 +10,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" pull_model "code.gitea.io/gitea/models/pull" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -19,6 +20,7 @@ import ( "code.gitea.io/gitea/services/context/upload" "code.gitea.io/gitea/services/forms" pull_service "code.gitea.io/gitea/services/pull" + user_service "code.gitea.io/gitea/services/user" ) const ( @@ -203,6 +205,10 @@ func renderConversation(ctx *context.Context, comment *issues_model.Comment, ori return } ctx.Data["AfterCommitID"] = pullHeadCommitID + ctx.Data["CanBlockUser"] = func(blocker, blockee *user_model.User) bool { + return user_service.CanBlockUser(ctx, ctx.Doer, blocker, blockee) + } + if origin == "diff" { ctx.HTML(http.StatusOK, tplDiffConversation) } else if origin == "timeline" { From 8d32f3cb745124a99a8948ae771bcbb68fa3f297 Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 6 Mar 2024 06:21:39 +0100 Subject: [PATCH 02/17] Update Twitter Logo (#29621) image --- public/assets/img/svg/gitea-twitter.svg | 2 +- web_src/svg/gitea-twitter.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/assets/img/svg/gitea-twitter.svg b/public/assets/img/svg/gitea-twitter.svg index 5d11c6eaec764..5ed1e264cac1f 100644 --- a/public/assets/img/svg/gitea-twitter.svg +++ b/public/assets/img/svg/gitea-twitter.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/web_src/svg/gitea-twitter.svg b/web_src/svg/gitea-twitter.svg index 096b9add2b174..f972d23f90cb5 100644 --- a/web_src/svg/gitea-twitter.svg +++ b/web_src/svg/gitea-twitter.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From da15d6127c8d430dfc069f9815ce783dd9ca35f7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 6 Mar 2024 13:52:48 +0800 Subject: [PATCH 03/17] A small refactor for agit implementation (#29614) This PR made the code simpler, reduced unnecessary database queries and fixed some warnning for the errors.New . --------- Co-authored-by: KN4CK3R --- services/agit/agit.go | 73 +++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/services/agit/agit.go b/services/agit/agit.go index 2233fe854746a..eb3bafa906557 100644 --- a/services/agit/agit.go +++ b/services/agit/agit.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "strconv" "strings" issues_model "code.gitea.io/gitea/models/issues" @@ -21,26 +22,17 @@ import ( // ProcReceive handle proc receive work func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, opts *private.HookOptions) ([]private.HookProcReceiveRefResult, error) { - // TODO: Add more options? - var ( - topicBranch string - title string - description string - forcePush bool - ) - results := make([]private.HookProcReceiveRefResult, 0, len(opts.OldCommitIDs)) - - ownerName := repo.OwnerName - repoName := repo.Name - - topicBranch = opts.GitPushOptions["topic"] - _, forcePush = opts.GitPushOptions["force-push"] + topicBranch := opts.GitPushOptions["topic"] + forcePush, _ := strconv.ParseBool(opts.GitPushOptions["force-push"]) + title := strings.TrimSpace(opts.GitPushOptions["title"]) + description := strings.TrimSpace(opts.GitPushOptions["description"]) // TODO: Add more options? objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + userName := strings.ToLower(opts.UserName) pusher, err := user_model.GetUserByID(ctx, opts.UserID) if err != nil { - return nil, fmt.Errorf("Failed to get user. Error: %w", err) + return nil, fmt.Errorf("failed to get user. Error: %w", err) } for i := range opts.OldCommitIDs { @@ -85,9 +77,6 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. continue } - var headBranch string - userName := strings.ToLower(opts.UserName) - if len(curentTopicBranch) == 0 { curentTopicBranch = topicBranch } @@ -95,6 +84,7 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. // because different user maybe want to use same topic, // So it's better to make sure the topic branch name // has user name prefix + var headBranch string if !strings.HasPrefix(curentTopicBranch, userName+"/") { headBranch = userName + "/" + curentTopicBranch } else { @@ -104,21 +94,26 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. pr, err := issues_model.GetUnmergedPullRequest(ctx, repo.ID, repo.ID, headBranch, baseBranchName, issues_model.PullRequestFlowAGit) if err != nil { if !issues_model.IsErrPullRequestNotExist(err) { - return nil, fmt.Errorf("Failed to get unmerged agit flow pull request in repository: %s/%s Error: %w", ownerName, repoName, err) + return nil, fmt.Errorf("failed to get unmerged agit flow pull request in repository: %s Error: %w", repo.FullName(), err) } - // create a new pull request - if len(title) == 0 { - var has bool - title, has = opts.GitPushOptions["title"] - if !has || len(title) == 0 { - commit, err := gitRepo.GetCommit(opts.NewCommitIDs[i]) - if err != nil { - return nil, fmt.Errorf("Failed to get commit %s in repository: %s/%s Error: %w", opts.NewCommitIDs[i], ownerName, repoName, err) - } - title = strings.Split(commit.CommitMessage, "\n")[0] + var commit *git.Commit + if title == "" || description == "" { + commit, err = gitRepo.GetCommit(opts.NewCommitIDs[i]) + if err != nil { + return nil, fmt.Errorf("failed to get commit %s in repository: %s Error: %w", opts.NewCommitIDs[i], repo.FullName(), err) } - description = opts.GitPushOptions["description"] + } + + // create a new pull request + if title == "" { + title = strings.Split(commit.CommitMessage, "\n")[0] + } + if description == "" { + _, description, _ = strings.Cut(commit.CommitMessage, "\n\n") + } + if description == "" { + description = title } prIssue := &issues_model.Issue{ @@ -160,12 +155,12 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. // update exist pull request if err := pr.LoadBaseRepo(ctx); err != nil { - return nil, fmt.Errorf("Unable to load base repository for PR[%d] Error: %w", pr.ID, err) + return nil, fmt.Errorf("unable to load base repository for PR[%d] Error: %w", pr.ID, err) } oldCommitID, err := gitRepo.GetRefCommitID(pr.GetGitRefName()) if err != nil { - return nil, fmt.Errorf("Unable to get ref commit id in base repository for PR[%d] Error: %w", pr.ID, err) + return nil, fmt.Errorf("unable to get ref commit id in base repository for PR[%d] Error: %w", pr.ID, err) } if oldCommitID == opts.NewCommitIDs[i] { @@ -179,9 +174,11 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. } if !forcePush { - output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1").AddDynamicArguments(oldCommitID, "^"+opts.NewCommitIDs[i]).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: os.Environ()}) + output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1"). + AddDynamicArguments(oldCommitID, "^"+opts.NewCommitIDs[i]). + RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: os.Environ()}) if err != nil { - return nil, fmt.Errorf("Fail to detect force push: %w", err) + return nil, fmt.Errorf("failed to detect force push: %w", err) } else if len(output) > 0 { results = append(results, private.HookProcReceiveRefResult{ OriginalRef: opts.RefFullNames[i], @@ -195,17 +192,13 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. pr.HeadCommitID = opts.NewCommitIDs[i] if err = pull_service.UpdateRef(ctx, pr); err != nil { - return nil, fmt.Errorf("Failed to update pull ref. Error: %w", err) + return nil, fmt.Errorf("failed to update pull ref. Error: %w", err) } pull_service.AddToTaskQueue(ctx, pr) - pusher, err := user_model.GetUserByID(ctx, opts.UserID) - if err != nil { - return nil, fmt.Errorf("Failed to get user. Error: %w", err) - } err = pr.LoadIssue(ctx) if err != nil { - return nil, fmt.Errorf("Failed to load pull issue. Error: %w", err) + return nil, fmt.Errorf("failed to load pull issue. Error: %w", err) } comment, err := pull_service.CreatePushPullComment(ctx, pusher, pr, oldCommitID, opts.NewCommitIDs[i]) if err == nil && comment != nil { From 5cddab4f74bbb307ddf13e458c7ac22f93b9283a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 6 Mar 2024 14:26:32 +0800 Subject: [PATCH 04/17] Make wiki default branch name changable (#29603) Fix #29000 Fix #28685 Fix #18568 Related: #27497 And by the way fix #24036, add a Cancel button there (one line) --- models/migrations/migrations.go | 2 + models/migrations/v1_22/v289.go | 18 ++++++ models/repo/repo.go | 4 ++ modules/git/repo_base_gogit.go | 4 +- modules/git/repo_base_nogogit.go | 4 +- options/locale/locale_en-US.ini | 2 + routers/web/repo/setting/setting.go | 7 +++ routers/web/repo/wiki.go | 60 +++++++++++-------- routers/web/repo/wiki_test.go | 30 ++++++++++ services/forms/repo_form.go | 1 + services/repository/create.go | 2 + services/repository/migrate.go | 88 ++++++++++++++++++---------- services/wiki/wiki.go | 79 ++++++++++++++++++------- services/wiki/wiki_test.go | 10 ++-- templates/repo/settings/options.tmpl | 4 ++ templates/repo/wiki/new.tmpl | 5 +- templates/repo/wiki/pages.tmpl | 1 + 17 files changed, 232 insertions(+), 89 deletions(-) create mode 100644 models/migrations/v1_22/v289.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 9d288ec2bdf17..d40866f3e93b3 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -562,6 +562,8 @@ var migrations = []Migration{ NewMigration("Use Slug instead of ID for Badges", v1_22.UseSlugInsteadOfIDForBadges), // v288 -> v289 NewMigration("Add user_blocking table", v1_22.AddUserBlockingTable), + // v289 -> v290 + NewMigration("Add default_wiki_branch to repository table", v1_22.AddDefaultWikiBranch), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_22/v289.go b/models/migrations/v1_22/v289.go new file mode 100644 index 0000000000000..e2dfc48715a28 --- /dev/null +++ b/models/migrations/v1_22/v289.go @@ -0,0 +1,18 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_22 //nolint + +import "xorm.io/xorm" + +func AddDefaultWikiBranch(x *xorm.Engine) error { + type Repository struct { + ID int64 + DefaultWikiBranch string + } + if err := x.Sync(&Repository{}); err != nil { + return err + } + _, err := x.Exec("UPDATE `repository` SET default_wiki_branch = 'master' WHERE (default_wiki_branch IS NULL) OR (default_wiki_branch = '')") + return err +} diff --git a/models/repo/repo.go b/models/repo/repo.go index f6758f1591ac6..1d17e565aef2c 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -136,6 +136,7 @@ type Repository struct { OriginalServiceType api.GitServiceType `xorm:"index"` OriginalURL string `xorm:"VARCHAR(2048)"` DefaultBranch string + DefaultWikiBranch string NumWatches int NumStars int @@ -285,6 +286,9 @@ func (repo *Repository) AfterLoad() { repo.NumOpenMilestones = repo.NumMilestones - repo.NumClosedMilestones repo.NumOpenProjects = repo.NumProjects - repo.NumClosedProjects repo.NumOpenActionRuns = repo.NumActionRuns - repo.NumClosedActionRuns + if repo.DefaultWikiBranch == "" { + repo.DefaultWikiBranch = setting.Repository.DefaultBranch + } } // LoadAttributes loads attributes of the repository. diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index 3ca5eb36c6ede..0cd07dcdc897e 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -8,11 +8,11 @@ package git import ( "context" - "errors" "path/filepath" gitealog "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" @@ -52,7 +52,7 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { if err != nil { return nil, err } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") + return nil, util.NewNotExistErrorf("no such file or directory") } fs := osfs.New(repoPath) diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 86b6a93567753..7f6512200baeb 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -9,10 +9,10 @@ package git import ( "bufio" "context" - "errors" "path/filepath" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" ) func init() { @@ -54,7 +54,7 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { if err != nil { return nil, err } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") + return nil, util.NewNotExistErrorf("no such file or directory") } // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3fc74959cadac..59aae68cae6bb 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2092,6 +2092,8 @@ settings.branches.add_new_rule = Add New Rule settings.advanced_settings = Advanced Settings settings.wiki_desc = Enable Repository Wiki settings.use_internal_wiki = Use Built-In Wiki +settings.default_wiki_branch_name = Default Wiki Branch Name +settings.failed_to_change_default_wiki_branch = Failed to change the default wiki branch. settings.use_external_wiki = Use External Wiki settings.external_wiki_url = External Wiki URL settings.external_wiki_url_error = The external wiki URL is not a valid URL. diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index 992a980d9e641..e045e3b8dcc09 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -488,6 +488,13 @@ func SettingsPost(ctx *context.Context) { } } + if form.DefaultWikiBranch != "" { + if err := wiki_service.ChangeDefaultWikiBranch(ctx, repo, form.DefaultWikiBranch); err != nil { + log.Error("ChangeDefaultWikiBranch failed, err: %v", err) + ctx.Flash.Warning(ctx.Tr("repo.settings.failed_to_change_default_wiki_branch")) + } + } + if form.EnableIssues && form.EnableExternalTracker && !unit_model.TypeExternalTracker.UnitGlobalDisabled() { if !validation.IsValidExternalURL(form.ExternalTrackerURL) { ctx.Flash.Error(ctx.Tr("repo.settings.external_tracker_url_error")) diff --git a/routers/web/repo/wiki.go b/routers/web/repo/wiki.go index 91cf727e2c5ce..88b63da88d986 100644 --- a/routers/web/repo/wiki.go +++ b/routers/web/repo/wiki.go @@ -93,17 +93,32 @@ func findEntryForFile(commit *git.Commit, target string) (*git.TreeEntry, error) } func findWikiRepoCommit(ctx *context.Context) (*git.Repository, *git.Commit, error) { - wikiRepo, err := gitrepo.OpenWikiRepository(ctx, ctx.Repo.Repository) - if err != nil { - ctx.ServerError("OpenRepository", err) - return nil, nil, err + wikiGitRepo, errGitRepo := gitrepo.OpenWikiRepository(ctx, ctx.Repo.Repository) + if errGitRepo != nil { + ctx.ServerError("OpenRepository", errGitRepo) + return nil, nil, errGitRepo } - commit, err := wikiRepo.GetBranchCommit(wiki_service.DefaultBranch) - if err != nil { - return wikiRepo, nil, err + commit, errCommit := wikiGitRepo.GetBranchCommit(ctx.Repo.Repository.DefaultWikiBranch) + if git.IsErrNotExist(errCommit) { + // if the default branch recorded in database is out of sync, then re-sync it + gitRepoDefaultBranch, errBranch := wikiGitRepo.GetDefaultBranch() + if errBranch != nil { + return wikiGitRepo, nil, errBranch + } + // update the default branch in the database + errDb := repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: ctx.Repo.Repository.ID, DefaultWikiBranch: gitRepoDefaultBranch}, "default_wiki_branch") + if errDb != nil { + return wikiGitRepo, nil, errDb + } + ctx.Repo.Repository.DefaultWikiBranch = gitRepoDefaultBranch + // retry to get the commit from the correct default branch + commit, errCommit = wikiGitRepo.GetBranchCommit(ctx.Repo.Repository.DefaultWikiBranch) } - return wikiRepo, commit, nil + if errCommit != nil { + return wikiGitRepo, nil, errCommit + } + return wikiGitRepo, commit, nil } // wikiContentsByEntry returns the contents of the wiki page referenced by the @@ -316,7 +331,7 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { } // get commit count - wiki revisions - commitsCount, _ := wikiRepo.FileCommitsCount(wiki_service.DefaultBranch, pageFilename) + commitsCount, _ := wikiRepo.FileCommitsCount(ctx.Repo.Repository.DefaultWikiBranch, pageFilename) ctx.Data["CommitCount"] = commitsCount return wikiRepo, entry @@ -368,7 +383,7 @@ func renderRevisionPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) ctx.Data["footerContent"] = "" // get commit count - wiki revisions - commitsCount, _ := wikiRepo.FileCommitsCount(wiki_service.DefaultBranch, pageFilename) + commitsCount, _ := wikiRepo.FileCommitsCount(ctx.Repo.Repository.DefaultWikiBranch, pageFilename) ctx.Data["CommitCount"] = commitsCount // get page @@ -380,7 +395,7 @@ func renderRevisionPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) // get Commit Count commitsHistory, err := wikiRepo.CommitsByFileAndRange( git.CommitsByFileAndRangeOptions{ - Revision: wiki_service.DefaultBranch, + Revision: ctx.Repo.Repository.DefaultWikiBranch, File: pageFilename, Page: page, }) @@ -402,20 +417,17 @@ func renderRevisionPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) func renderEditPage(ctx *context.Context) { wikiRepo, commit, err := findWikiRepoCommit(ctx) - if err != nil { + defer func() { if wikiRepo != nil { - wikiRepo.Close() + _ = wikiRepo.Close() } + }() + if err != nil { if !git.IsErrNotExist(err) { ctx.ServerError("GetBranchCommit", err) } return } - defer func() { - if wikiRepo != nil { - wikiRepo.Close() - } - }() // get requested pagename pageName := wiki_service.WebPathFromRequest(ctx.PathParamRaw("*")) @@ -584,17 +596,15 @@ func WikiPages(ctx *context.Context) { ctx.Data["CanWriteWiki"] = ctx.Repo.CanWrite(unit.TypeWiki) && !ctx.Repo.Repository.IsArchived wikiRepo, commit, err := findWikiRepoCommit(ctx) - if err != nil { - if wikiRepo != nil { - wikiRepo.Close() - } - return - } defer func() { if wikiRepo != nil { - wikiRepo.Close() + _ = wikiRepo.Close() } }() + if err != nil { + ctx.Redirect(ctx.Repo.RepoLink + "/wiki") + return + } entries, err := commit.ListEntries() if err != nil { diff --git a/routers/web/repo/wiki_test.go b/routers/web/repo/wiki_test.go index 719cca304923f..52e216e6a0ac2 100644 --- a/routers/web/repo/wiki_test.go +++ b/routers/web/repo/wiki_test.go @@ -9,6 +9,7 @@ import ( "net/url" "testing" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/git" @@ -221,3 +222,32 @@ func TestWikiRaw(t *testing.T) { } } } + +func TestDefaultWikiBranch(t *testing.T) { + unittest.PrepareTestEnv(t) + + assert.NoError(t, repo_model.UpdateRepositoryCols(db.DefaultContext, &repo_model.Repository{ID: 1, DefaultWikiBranch: "wrong-branch"})) + + ctx, _ := contexttest.MockContext(t, "user2/repo1/wiki") + ctx.SetParams("*", "Home") + contexttest.LoadRepo(t, ctx, 1) + assert.Equal(t, "wrong-branch", ctx.Repo.Repository.DefaultWikiBranch) + Wiki(ctx) // after the visiting, the out-of-sync database record will update the branch name to "master" + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + assert.Equal(t, "master", ctx.Repo.Repository.DefaultWikiBranch) + + // invalid branch name should fail + assert.Error(t, wiki_service.ChangeDefaultWikiBranch(db.DefaultContext, repo, "the bad name")) + repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + assert.Equal(t, "master", repo.DefaultWikiBranch) + + // the same branch name, should succeed (actually a no-op) + assert.NoError(t, wiki_service.ChangeDefaultWikiBranch(db.DefaultContext, repo, "master")) + repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + assert.Equal(t, "master", repo.DefaultWikiBranch) + + // change to another name + assert.NoError(t, wiki_service.ChangeDefaultWikiBranch(db.DefaultContext, repo, "main")) + repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + assert.Equal(t, "main", repo.DefaultWikiBranch) +} diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 8c3e458d2ff6b..e45a2a1695522 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -133,6 +133,7 @@ type RepoSettingForm struct { EnableCode bool EnableWiki bool EnableExternalWiki bool + DefaultWikiBranch string ExternalWikiURL string EnableIssues bool EnableExternalTracker bool diff --git a/services/repository/create.go b/services/repository/create.go index c47ce9c413647..8d8c39197dece 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -173,6 +173,7 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re } repo.DefaultBranch = setting.Repository.DefaultBranch + repo.DefaultWikiBranch = setting.Repository.DefaultBranch if len(opts.DefaultBranch) > 0 { repo.DefaultBranch = opts.DefaultBranch @@ -240,6 +241,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt TrustModel: opts.TrustModel, IsMirror: opts.IsMirror, DefaultBranch: opts.DefaultBranch, + DefaultWikiBranch: setting.Repository.DefaultBranch, ObjectFormatName: opts.ObjectFormatName, } diff --git a/services/repository/migrate.go b/services/repository/migrate.go index 51fdd90f5400e..aae2ddc1200ea 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -25,6 +25,54 @@ import ( "code.gitea.io/gitea/modules/util" ) +func cloneWiki(ctx context.Context, u *user_model.User, opts migration.MigrateOptions, migrateTimeout time.Duration) (string, error) { + wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) + wikiRemotePath := repo_module.WikiRemoteURL(ctx, opts.CloneAddr) + if wikiRemotePath == "" { + return "", nil + } + + if err := util.RemoveAll(wikiPath); err != nil { + return "", fmt.Errorf("failed to remove existing wiki dir %q, err: %w", wikiPath, err) + } + + cleanIncompleteWikiPath := func() { + if err := util.RemoveAll(wikiPath); err != nil { + log.Error("Failed to remove incomplete wiki dir %q, err: %v", wikiPath, err) + } + } + if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + SkipTLSVerify: setting.Migrations.SkipTLSVerify, + }); err != nil { + log.Error("Clone wiki failed, err: %v", err) + cleanIncompleteWikiPath() + return "", err + } + + if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { + cleanIncompleteWikiPath() + return "", err + } + + wikiRepo, err := git.OpenRepository(ctx, wikiPath) + if err != nil { + cleanIncompleteWikiPath() + return "", fmt.Errorf("failed to open wiki repo %q, err: %w", wikiPath, err) + } + defer wikiRepo.Close() + + defaultBranch, err := wikiRepo.GetDefaultBranch() + if err != nil { + cleanIncompleteWikiPath() + return "", fmt.Errorf("failed to get wiki repo default branch for %q, err: %w", wikiPath, err) + } + + return defaultBranch, nil +} + // MigrateRepositoryGitData starts migrating git related data after created migrating repository func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts migration.MigrateOptions, @@ -44,21 +92,20 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - var err error - if err = util.RemoveAll(repoPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", repoPath, err) + if err := util.RemoveAll(repoPath); err != nil { + return repo, fmt.Errorf("failed to remove existing repo dir %q, err: %w", repoPath, err) } - if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ + if err := git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, SkipTLSVerify: setting.Migrations.SkipTLSVerify, }); err != nil { if errors.Is(err, context.DeadlineExceeded) { - return repo, fmt.Errorf("Clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err) + return repo, fmt.Errorf("clone timed out, consider increasing [git.timeout] MIGRATE in app.ini, underlying err: %w", err) } - return repo, fmt.Errorf("Clone: %w", err) + return repo, fmt.Errorf("clone error: %w", err) } if err := git.WriteCommitGraph(ctx, repoPath); err != nil { @@ -66,37 +113,18 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } if opts.Wiki { - wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) - wikiRemotePath := repo_module.WikiRemoteURL(ctx, opts.CloneAddr) - if len(wikiRemotePath) > 0 { - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) - } - - if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - Branch: "master", - SkipTLSVerify: setting.Migrations.SkipTLSVerify, - }); err != nil { - log.Warn("Clone wiki: %v", err) - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) - } - } else { - if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { - return repo, err - } - } + defaultWikiBranch, err := cloneWiki(ctx, u, opts, migrateTimeout) + if err != nil { + return repo, fmt.Errorf("clone wiki error: %w", err) } + repo.DefaultWikiBranch = defaultWikiBranch } if repo.OwnerID == u.ID { repo.Owner = u } - if err = repo_module.CheckDaemonExportOK(ctx, repo); err != nil { + if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { return repo, fmt.Errorf("checkDaemonExportOK: %w", err) } diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index 50d52d3140fe8..6f1ca120b0a27 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -6,18 +6,22 @@ package wiki import ( "context" + "errors" "fmt" "os" "strings" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/sync" + "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" repo_service "code.gitea.io/gitea/services/repository" ) @@ -25,10 +29,7 @@ import ( // TODO: use clustered lock (unique queue? or *abuse* cache) var wikiWorkingPool = sync.NewExclusivePool() -const ( - DefaultRemote = "origin" - DefaultBranch = "master" -) +const DefaultRemote = "origin" // InitWiki initializes a wiki for repository, // it does nothing when repository already has wiki. @@ -41,25 +42,25 @@ func InitWiki(ctx context.Context, repo *repo_model.Repository) error { return fmt.Errorf("InitRepository: %w", err) } else if err = repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { return fmt.Errorf("createDelegateHooks: %w", err) - } else if _, _, err = git.NewCommand(ctx, "symbolic-ref", "HEAD", git.BranchPrefix+DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.WikiPath()}); err != nil { - return fmt.Errorf("unable to set default wiki branch to master: %w", err) + } else if _, _, err = git.NewCommand(ctx, "symbolic-ref", "HEAD").AddDynamicArguments(git.BranchPrefix + repo.DefaultWikiBranch).RunStdString(&git.RunOpts{Dir: repo.WikiPath()}); err != nil { + return fmt.Errorf("unable to set default wiki branch to %q: %w", repo.DefaultWikiBranch, err) } return nil } // prepareGitPath try to find a suitable file path with file name by the given raw wiki name. // return: existence, prepared file path with name, error -func prepareGitPath(gitRepo *git.Repository, wikiPath WebPath) (bool, string, error) { +func prepareGitPath(gitRepo *git.Repository, defaultWikiBranch string, wikiPath WebPath) (bool, string, error) { unescaped := string(wikiPath) + ".md" gitPath := WebPathToGitPath(wikiPath) // Look for both files - filesInIndex, err := gitRepo.LsTree(DefaultBranch, unescaped, gitPath) + filesInIndex, err := gitRepo.LsTree(defaultWikiBranch, unescaped, gitPath) if err != nil { - if strings.Contains(err.Error(), "Not a valid object name master") { - return false, gitPath, nil + if strings.Contains(err.Error(), "Not a valid object name") { + return false, gitPath, nil // branch doesn't exist } - log.Error("%v", err) + log.Error("Wiki LsTree failed, err: %v", err) return false, gitPath, err } @@ -95,7 +96,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("InitWiki: %w", err) } - hasMasterBranch := git.IsBranchExist(ctx, repo.WikiPath(), DefaultBranch) + hasDefaultBranch := git.IsBranchExist(ctx, repo.WikiPath(), repo.DefaultWikiBranch) basePath, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { @@ -112,8 +113,8 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model Shared: true, } - if hasMasterBranch { - cloneOpts.Branch = DefaultBranch + if hasDefaultBranch { + cloneOpts.Branch = repo.DefaultWikiBranch } if err := git.Clone(ctx, repo.WikiPath(), basePath, cloneOpts); err != nil { @@ -128,14 +129,14 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model } defer gitRepo.Close() - if hasMasterBranch { + if hasDefaultBranch { if err := gitRepo.ReadTreeToIndex("HEAD"); err != nil { log.Error("Unable to read HEAD tree to index in: %s %v", basePath, err) return fmt.Errorf("fnable to read HEAD tree to index in: %s %w", basePath, err) } } - isWikiExist, newWikiPath, err := prepareGitPath(gitRepo, newWikiName) + isWikiExist, newWikiPath, err := prepareGitPath(gitRepo, repo.DefaultWikiBranch, newWikiName) if err != nil { return err } @@ -151,7 +152,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model isOldWikiExist := true oldWikiPath := newWikiPath if oldWikiName != newWikiName { - isOldWikiExist, oldWikiPath, err = prepareGitPath(gitRepo, oldWikiName) + isOldWikiExist, oldWikiPath, err = prepareGitPath(gitRepo, repo.DefaultWikiBranch, oldWikiName) if err != nil { return err } @@ -200,7 +201,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model } else { commitTreeOpts.NoGPGSign = true } - if hasMasterBranch { + if hasDefaultBranch { commitTreeOpts.Parents = []string{"HEAD"} } @@ -212,7 +213,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model if err := git.Push(gitRepo.Ctx, basePath, git.PushOptions{ Remote: DefaultRemote, - Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, DefaultBranch), + Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, repo.DefaultWikiBranch), Env: repo_module.FullPushingEnvironment( doer, doer, @@ -269,7 +270,7 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model if err := git.Clone(ctx, repo.WikiPath(), basePath, git.CloneRepoOptions{ Bare: true, Shared: true, - Branch: DefaultBranch, + Branch: repo.DefaultWikiBranch, }); err != nil { log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) return fmt.Errorf("failed to clone repository: %s (%w)", repo.FullName(), err) @@ -287,7 +288,7 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("unable to read HEAD tree to index in: %s %w", basePath, err) } - found, wikiPath, err := prepareGitPath(gitRepo, wikiName) + found, wikiPath, err := prepareGitPath(gitRepo, repo.DefaultWikiBranch, wikiName) if err != nil { return err } @@ -331,7 +332,7 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model if err := git.Push(gitRepo.Ctx, basePath, git.PushOptions{ Remote: DefaultRemote, - Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, DefaultBranch), + Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, repo.DefaultWikiBranch), Env: repo_module.FullPushingEnvironment( doer, doer, @@ -358,3 +359,37 @@ func DeleteWiki(ctx context.Context, repo *repo_model.Repository) error { system_model.RemoveAllWithNotice(ctx, "Delete repository wiki", repo.WikiPath()) return nil } + +func ChangeDefaultWikiBranch(ctx context.Context, repo *repo_model.Repository, newBranch string) error { + if !git.IsValidRefPattern(newBranch) { + return fmt.Errorf("invalid branch name: %s", newBranch) + } + return db.WithTx(ctx, func(ctx context.Context) error { + repo.DefaultWikiBranch = newBranch + if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_wiki_branch"); err != nil { + return fmt.Errorf("unable to update database: %w", err) + } + + gitRepo, err := gitrepo.OpenWikiRepository(ctx, repo) + if errors.Is(err, util.ErrNotExist) { + return nil // no git repo on storage, no need to do anything else + } else if err != nil { + return fmt.Errorf("unable to open repository: %w", err) + } + defer gitRepo.Close() + + oldDefBranch, err := gitRepo.GetDefaultBranch() + if err != nil { + return fmt.Errorf("unable to get default branch: %w", err) + } + if oldDefBranch == newBranch { + return nil + } + + err = gitRepo.RenameBranch(oldDefBranch, newBranch) + if err != nil { + return fmt.Errorf("unable to rename default branch: %w", err) + } + return nil + }) +} diff --git a/services/wiki/wiki_test.go b/services/wiki/wiki_test.go index 59c77060f2c27..0a18cffa25310 100644 --- a/services/wiki/wiki_test.go +++ b/services/wiki/wiki_test.go @@ -170,7 +170,7 @@ func TestRepository_AddWikiPage(t *testing.T) { return } defer gitRepo.Close() - masterTree, err := gitRepo.GetTree(DefaultBranch) + masterTree, err := gitRepo.GetTree(repo.DefaultWikiBranch) assert.NoError(t, err) gitPath := WebPathToGitPath(webPath) entry, err := masterTree.GetTreeEntryByPath(gitPath) @@ -215,7 +215,7 @@ func TestRepository_EditWikiPage(t *testing.T) { // Now need to show that the page has been added: gitRepo, err := gitrepo.OpenWikiRepository(git.DefaultContext, repo) assert.NoError(t, err) - masterTree, err := gitRepo.GetTree(DefaultBranch) + masterTree, err := gitRepo.GetTree(repo.DefaultWikiBranch) assert.NoError(t, err) gitPath := WebPathToGitPath(webPath) entry, err := masterTree.GetTreeEntryByPath(gitPath) @@ -242,7 +242,7 @@ func TestRepository_DeleteWikiPage(t *testing.T) { return } defer gitRepo.Close() - masterTree, err := gitRepo.GetTree(DefaultBranch) + masterTree, err := gitRepo.GetTree(repo.DefaultWikiBranch) assert.NoError(t, err) gitPath := WebPathToGitPath("Home") _, err = masterTree.GetTreeEntryByPath(gitPath) @@ -280,7 +280,7 @@ func TestPrepareWikiFileName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { webPath := UserTitleToWebPath("", tt.arg) - existence, newWikiPath, err := prepareGitPath(gitRepo, webPath) + existence, newWikiPath, err := prepareGitPath(gitRepo, repo.DefaultWikiBranch, webPath) if (err != nil) != tt.wantErr { assert.NoError(t, err) return @@ -312,7 +312,7 @@ func TestPrepareWikiFileName_FirstPage(t *testing.T) { } defer gitRepo.Close() - existence, newWikiPath, err := prepareGitPath(gitRepo, "Home") + existence, newWikiPath, err := prepareGitPath(gitRepo, "master", "Home") assert.False(t, existence) assert.NoError(t, err) assert.EqualValues(t, "Home.md", newWikiPath) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 0de42b34ea898..a699396a84b32 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -335,6 +335,10 @@ +
+ + +
diff --git a/templates/repo/wiki/new.tmpl b/templates/repo/wiki/new.tmpl index ff31df0c32db5..640f8ca9cd607 100644 --- a/templates/repo/wiki/new.tmpl +++ b/templates/repo/wiki/new.tmpl @@ -36,9 +36,8 @@
- + {{ctx.Locale.Tr "cancel"}} +
diff --git a/templates/repo/wiki/pages.tmpl b/templates/repo/wiki/pages.tmpl index 22eb2619f9190..cace0d48e0083 100644 --- a/templates/repo/wiki/pages.tmpl +++ b/templates/repo/wiki/pages.tmpl @@ -10,6 +10,7 @@ {{end}} + {{if .IsRepositoryAdmin}}
{{ctx.Locale.Tr "repo.default_branch"}}: {{.Repository.DefaultWikiBranch}}
{{end}} {{range .Pages}} From a2b0fb1a64b0794b808a013089758a49f56d8915 Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Wed, 6 Mar 2024 16:24:43 +0900 Subject: [PATCH 05/17] Fix wrong line number in code search result (#29260) Fix #29136 Before: The result is a table and all line numbers are all in one row. After: Use a separate table column for the line numbers. --------- Co-authored-by: wxiaoguang --- modules/indexer/code/search.go | 50 +++++++++++++++++++------------ templates/code/searchresults.tmpl | 15 +--------- templates/repo/search.tmpl | 15 +--------- templates/shared/searchfile.tmpl | 14 +++++++++ 4 files changed, 47 insertions(+), 47 deletions(-) create mode 100644 templates/shared/searchfile.tmpl diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index e19e22eea0e1e..2ddc2397fa191 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -16,14 +16,18 @@ import ( // Result a search result to display type Result struct { - RepoID int64 - Filename string - CommitID string - UpdatedUnix timeutil.TimeStamp - Language string - Color string - LineNumbers []int - FormattedLines template.HTML + RepoID int64 + Filename string + CommitID string + UpdatedUnix timeutil.TimeStamp + Language string + Color string + Lines []ResultLine +} + +type ResultLine struct { + Num int + FormattedContent template.HTML } type SearchResultLanguages = internal.SearchResultLanguages @@ -70,7 +74,7 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res var formattedLinesBuffer bytes.Buffer contentLines := strings.SplitAfter(result.Content[startIndex:endIndex], "\n") - lineNumbers := make([]int, len(contentLines)) + lines := make([]ResultLine, 0, len(contentLines)) index := startIndex for i, line := range contentLines { var err error @@ -93,21 +97,29 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res return nil, err } - lineNumbers[i] = startLineNum + i + lines = append(lines, ResultLine{Num: startLineNum + i}) index += len(line) } - highlighted, _ := highlight.Code(result.Filename, "", formattedLinesBuffer.String()) + // we should highlight the whole code block first, otherwise it doesn't work well with multiple line highlighting + hl, _ := highlight.Code(result.Filename, "", formattedLinesBuffer.String()) + highlightedLines := strings.Split(string(hl), "\n") + + // The lines outputted by highlight.Code might not match the original lines, because "highlight" removes the last `\n` + lines = lines[:min(len(highlightedLines), len(lines))] + highlightedLines = highlightedLines[:len(lines)] + for i := 0; i < len(lines); i++ { + lines[i].FormattedContent = template.HTML(highlightedLines[i]) + } return &Result{ - RepoID: result.RepoID, - Filename: result.Filename, - CommitID: result.CommitID, - UpdatedUnix: result.UpdatedUnix, - Language: result.Language, - Color: result.Color, - LineNumbers: lineNumbers, - FormattedLines: highlighted, + RepoID: result.RepoID, + Filename: result.Filename, + CommitID: result.CommitID, + UpdatedUnix: result.UpdatedUnix, + Language: result.Language, + Color: result.Color, + Lines: lines, }, nil } diff --git a/templates/code/searchresults.tmpl b/templates/code/searchresults.tmpl index bb21a5e0dcadf..08bb12951d822 100644 --- a/templates/code/searchresults.tmpl +++ b/templates/code/searchresults.tmpl @@ -22,20 +22,7 @@ {{ctx.Locale.Tr "repo.diff.view_file"}}
-
-
- - - - - - -
- {{range .LineNumbers}} - {{.}} - {{end}} - {{.FormattedLines}}
- + {{template "shared/searchfile" dict "RepoLink" $repo.Link "SearchResult" .}} {{template "shared/searchbottom" dict "root" $ "result" .}} diff --git a/templates/repo/search.tmpl b/templates/repo/search.tmpl index 7b3ad7282ecc4..7513d444cc1c7 100644 --- a/templates/repo/search.tmpl +++ b/templates/repo/search.tmpl @@ -44,20 +44,7 @@ {{ctx.Locale.Tr "repo.diff.view_file"}}
-
- - - - - - - -
- {{range .LineNumbers}} - {{.}} - {{end}} - {{.FormattedLines}}
-
+ {{template "shared/searchfile" dict "RepoLink" $.SourcePath "SearchResult" .}}
{{template "shared/searchbottom" dict "root" $ "result" .}} diff --git a/templates/shared/searchfile.tmpl b/templates/shared/searchfile.tmpl new file mode 100644 index 0000000000000..280584e4d1f9a --- /dev/null +++ b/templates/shared/searchfile.tmpl @@ -0,0 +1,14 @@ +
+ + + {{range .SearchResult.Lines}} + + + + + {{end}} + +
+ {{.Num}} + {{.FormattedContent}}
+
From 5bdf805e054ec4131d8f1451752f251e8807cc73 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 6 Mar 2024 16:47:52 +0800 Subject: [PATCH 06/17] Sync branches to DB immediately when handle git hook calling (#29493) Unlike other async processing in the queue, we should sync branches to the DB immediately when handling git hook calling. If it fails, users can see the error message in the output of the git command. It can avoid potential inconsistency issues, and help #29494. --------- Co-authored-by: Lunny Xiao --- models/git/branch.go | 5 + routers/private/hook_post_receive.go | 66 +++++++++++++- services/repository/branch.go | 117 +++++++++++++++++------- services/repository/push.go | 9 -- tests/integration/git_push_test.go | 131 +++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 45 deletions(-) create mode 100644 tests/integration/git_push_test.go diff --git a/models/git/branch.go b/models/git/branch.go index db02fc9b28bbd..fa0781fed1d57 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -158,6 +158,11 @@ func GetBranch(ctx context.Context, repoID int64, branchName string) (*Branch, e return &branch, nil } +func GetBranches(ctx context.Context, repoID int64, branchNames []string) ([]*Branch, error) { + branches := make([]*Branch, 0, len(branchNames)) + return branches, db.GetEngine(ctx).Where("repo_id=?", repoID).In("name", branchNames).Find(&branches) +} + func AddBranches(ctx context.Context, branches []*Branch) error { for _, branch := range branches { if _, err := db.GetEngine(ctx).Insert(branch); err != nil { diff --git a/routers/private/hook_post_receive.go b/routers/private/hook_post_receive.go index 4eafe3923dfb8..c5504126f865d 100644 --- a/routers/private/hook_post_receive.go +++ b/routers/private/hook_post_receive.go @@ -8,9 +8,11 @@ import ( "net/http" "strconv" + git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" repo_module "code.gitea.io/gitea/modules/repository" @@ -27,6 +29,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { // We don't rely on RepoAssignment here because: // a) we don't need the git repo in this function + // OUT OF DATE: we do need the git repo to sync the branch to the db now. // b) our update function will likely change the repository in the db so we will need to refresh it // c) we don't always need the repo @@ -34,7 +37,11 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { repoName := ctx.Params(":repo") // defer getting the repository at this point - as we should only retrieve it if we're going to call update - var repo *repo_model.Repository + var ( + repo *repo_model.Repository + gitRepo *git.Repository + ) + defer gitRepo.Close() // it's safe to call Close on a nil pointer updates := make([]*repo_module.PushUpdateOptions, 0, len(opts.OldCommitIDs)) wasEmpty := false @@ -87,6 +94,63 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { }) return } + + branchesToSync := make([]*repo_module.PushUpdateOptions, 0, len(updates)) + for _, update := range updates { + if !update.RefFullName.IsBranch() { + continue + } + if repo == nil { + repo = loadRepository(ctx, ownerName, repoName) + if ctx.Written() { + return + } + wasEmpty = repo.IsEmpty + } + + if update.IsDelRef() { + if err := git_model.AddDeletedBranch(ctx, repo.ID, update.RefFullName.BranchName(), update.PusherID); err != nil { + log.Error("Failed to add deleted branch: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to add deleted branch: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + } else { + branchesToSync = append(branchesToSync, update) + } + } + if len(branchesToSync) > 0 { + if gitRepo == nil { + var err error + gitRepo, err = gitrepo.OpenRepository(ctx, repo) + if err != nil { + log.Error("Failed to open repository: %s/%s Error: %v", ownerName, repoName, err) + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to open repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + } + + var ( + branchNames = make([]string, 0, len(branchesToSync)) + commitIDs = make([]string, 0, len(branchesToSync)) + ) + for _, update := range branchesToSync { + branchNames = append(branchNames, update.RefFullName.BranchName()) + commitIDs = append(commitIDs, update.NewCommitID) + } + + if err := repo_service.SyncBranchesToDB(ctx, repo.ID, opts.UserID, branchNames, commitIDs, func(commitID string) (*git.Commit, error) { + return gitRepo.GetCommit(commitID) + }); err != nil { + ctx.JSON(http.StatusInternalServerError, private.HookPostReceiveResult{ + Err: fmt.Sprintf("Failed to sync branch to DB in repository: %s/%s Error: %v", ownerName, repoName, err), + }) + return + } + } } // Handle Push Options diff --git a/services/repository/branch.go b/services/repository/branch.go index 55cedf5d84bfb..402814fb9a343 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -221,44 +221,91 @@ func checkBranchName(ctx context.Context, repo *repo_model.Repository, name stri return err } -// syncBranchToDB sync the branch information in the database. It will try to update the branch first, -// if updated success with affect records > 0, then all are done. Because that means the branch has been in the database. -// If no record is affected, that means the branch does not exist in database. So there are two possibilities. -// One is this is a new branch, then we just need to insert the record. Another is the branches haven't been synced, -// then we need to sync all the branches into database. -func syncBranchToDB(ctx context.Context, repoID, pusherID int64, branchName string, commit *git.Commit) error { - cnt, err := git_model.UpdateBranch(ctx, repoID, pusherID, branchName, commit) - if err != nil { - return fmt.Errorf("git_model.UpdateBranch %d:%s failed: %v", repoID, branchName, err) - } - if cnt > 0 { // This means branch does exist, so it's a normal update. It also means the branch has been synced. - return nil - } +// SyncBranchesToDB sync the branch information in the database. +// It will check whether the branches of the repository have never been synced before. +// If so, it will sync all branches of the repository. +// Otherwise, it will sync the branches that need to be updated. +func SyncBranchesToDB(ctx context.Context, repoID, pusherID int64, branchNames, commitIDs []string, getCommit func(commitID string) (*git.Commit, error)) error { + // Some designs that make the code look strange but are made for performance optimization purposes: + // 1. Sync branches in a batch to reduce the number of DB queries. + // 2. Lazy load commit information since it may be not necessary. + // 3. Exit early if synced all branches of git repo when there's no branch in DB. + // 4. Check the branches in DB if they are already synced. + // + // If the user pushes many branches at once, the Git hook will call the internal API in batches, rather than all at once. + // See https://github.com/go-gitea/gitea/blob/cb52b17f92e2d2293f7c003649743464492bca48/cmd/hook.go#L27 + // For the first batch, it will hit optimization 3. + // For other batches, it will hit optimization 4. + + if len(branchNames) != len(commitIDs) { + return fmt.Errorf("branchNames and commitIDs length not match") + } + + return db.WithTx(ctx, func(ctx context.Context) error { + branches, err := git_model.GetBranches(ctx, repoID, branchNames) + if err != nil { + return fmt.Errorf("git_model.GetBranches: %v", err) + } - // if user haven't visit UI but directly push to a branch after upgrading from 1.20 -> 1.21, - // we cannot simply insert the branch but need to check we have branches or not - hasBranch, err := db.Exist[git_model.Branch](ctx, git_model.FindBranchOptions{ - RepoID: repoID, - IsDeletedBranch: optional.Some(false), - }.ToConds()) - if err != nil { - return err - } - if !hasBranch { - if _, err = repo_module.SyncRepoBranches(ctx, repoID, pusherID); err != nil { - return fmt.Errorf("repo_module.SyncRepoBranches %d:%s failed: %v", repoID, branchName, err) + if len(branches) == 0 { + // if user haven't visit UI but directly push to a branch after upgrading from 1.20 -> 1.21, + // we cannot simply insert the branch but need to check we have branches or not + hasBranch, err := db.Exist[git_model.Branch](ctx, git_model.FindBranchOptions{ + RepoID: repoID, + IsDeletedBranch: optional.Some(false), + }.ToConds()) + if err != nil { + return err + } + if !hasBranch { + if _, err = repo_module.SyncRepoBranches(ctx, repoID, pusherID); err != nil { + return fmt.Errorf("repo_module.SyncRepoBranches %d failed: %v", repoID, err) + } + return nil + } } - return nil - } - // if database have branches but not this branch, it means this is a new branch - return db.Insert(ctx, &git_model.Branch{ - RepoID: repoID, - Name: branchName, - CommitID: commit.ID.String(), - CommitMessage: commit.Summary(), - PusherID: pusherID, - CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), + branchMap := make(map[string]*git_model.Branch, len(branches)) + for _, branch := range branches { + branchMap[branch.Name] = branch + } + + newBranches := make([]*git_model.Branch, 0, len(branchNames)) + + for i, branchName := range branchNames { + commitID := commitIDs[i] + branch, exist := branchMap[branchName] + if exist && branch.CommitID == commitID { + continue + } + + commit, err := getCommit(branchName) + if err != nil { + return fmt.Errorf("get commit of %s failed: %v", branchName, err) + } + + if exist { + if _, err := git_model.UpdateBranch(ctx, repoID, pusherID, branchName, commit); err != nil { + return fmt.Errorf("git_model.UpdateBranch %d:%s failed: %v", repoID, branchName, err) + } + return nil + } + + // if database have branches but not this branch, it means this is a new branch + newBranches = append(newBranches, &git_model.Branch{ + RepoID: repoID, + Name: branchName, + CommitID: commit.ID.String(), + CommitMessage: commit.Summary(), + PusherID: pusherID, + CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), + }) + } + + if len(newBranches) > 0 { + return db.Insert(ctx, newBranches) + } + return nil }) } diff --git a/services/repository/push.go b/services/repository/push.go index 9aaf0e1c9bcab..89a3127902a59 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -11,7 +11,6 @@ import ( "time" "code.gitea.io/gitea/models/db" - git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" @@ -259,10 +258,6 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { commits.Commits = commits.Commits[:setting.UI.FeedMaxCommitNum] } - if err = syncBranchToDB(ctx, repo.ID, opts.PusherID, branch, newCommit); err != nil { - return fmt.Errorf("git_model.UpdateBranch %s:%s failed: %v", repo.FullName(), branch, err) - } - notify_service.PushCommits(ctx, pusher, repo, opts, commits) // Cache for big repository @@ -275,10 +270,6 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { // close all related pulls log.Error("close related pull request failed: %v", err) } - - if err := git_model.AddDeletedBranch(ctx, repo.ID, branch, pusher.ID); err != nil { - return fmt.Errorf("AddDeletedBranch %s:%s failed: %v", repo.FullName(), branch, err) - } } // Even if user delete a branch on a repository which he didn't watch, he will be watch that. diff --git a/tests/integration/git_push_test.go b/tests/integration/git_push_test.go new file mode 100644 index 0000000000000..cb2910b175e8d --- /dev/null +++ b/tests/integration/git_push_test.go @@ -0,0 +1,131 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "fmt" + "net/url" + "testing" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + repo_service "code.gitea.io/gitea/services/repository" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGitPush(t *testing.T) { + onGiteaRun(t, testGitPush) +} + +func testGitPush(t *testing.T, u *url.URL) { + t.Run("Push branches at once", func(t *testing.T) { + runTestGitPush(t, u, func(t *testing.T, gitPath string) (pushed, deleted []string) { + for i := 0; i < 100; i++ { + branchName := fmt.Sprintf("branch-%d", i) + pushed = append(pushed, branchName) + doGitCreateBranch(gitPath, branchName)(t) + } + pushed = append(pushed, "master") + doGitPushTestRepository(gitPath, "origin", "--all")(t) + return pushed, deleted + }) + }) + + t.Run("Push branches one by one", func(t *testing.T) { + runTestGitPush(t, u, func(t *testing.T, gitPath string) (pushed, deleted []string) { + for i := 0; i < 100; i++ { + branchName := fmt.Sprintf("branch-%d", i) + doGitCreateBranch(gitPath, branchName)(t) + doGitPushTestRepository(gitPath, "origin", branchName)(t) + pushed = append(pushed, branchName) + } + return pushed, deleted + }) + }) + + t.Run("Delete branches", func(t *testing.T) { + runTestGitPush(t, u, func(t *testing.T, gitPath string) (pushed, deleted []string) { + doGitPushTestRepository(gitPath, "origin", "master")(t) // make sure master is the default branch instead of a branch we are going to delete + pushed = append(pushed, "master") + + for i := 0; i < 100; i++ { + branchName := fmt.Sprintf("branch-%d", i) + pushed = append(pushed, branchName) + doGitCreateBranch(gitPath, branchName)(t) + } + doGitPushTestRepository(gitPath, "origin", "--all")(t) + + for i := 0; i < 10; i++ { + branchName := fmt.Sprintf("branch-%d", i) + doGitPushTestRepository(gitPath, "origin", "--delete", branchName)(t) + deleted = append(deleted, branchName) + } + return pushed, deleted + }) + }) +} + +func runTestGitPush(t *testing.T, u *url.URL, gitOperation func(t *testing.T, gitPath string) (pushed, deleted []string)) { + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo, err := repo_service.CreateRepository(db.DefaultContext, user, user, repo_service.CreateRepoOptions{ + Name: "repo-to-push", + Description: "test git push", + AutoInit: false, + DefaultBranch: "main", + IsPrivate: false, + }) + require.NoError(t, err) + require.NotEmpty(t, repo) + + gitPath := t.TempDir() + + doGitInitTestRepository(gitPath)(t) + + oldPath := u.Path + oldUser := u.User + defer func() { + u.Path = oldPath + u.User = oldUser + }() + u.Path = repo.FullName() + ".git" + u.User = url.UserPassword(user.LowerName, userPassword) + + doGitAddRemote(gitPath, "origin", u)(t) + + gitRepo, err := git.OpenRepository(git.DefaultContext, gitPath) + require.NoError(t, err) + defer gitRepo.Close() + + pushedBranches, deletedBranches := gitOperation(t, gitPath) + + dbBranches := make([]*git_model.Branch, 0) + require.NoError(t, db.GetEngine(db.DefaultContext).Where("repo_id=?", repo.ID).Find(&dbBranches)) + assert.Equalf(t, len(pushedBranches), len(dbBranches), "mismatched number of branches in db") + dbBranchesMap := make(map[string]*git_model.Branch, len(dbBranches)) + for _, branch := range dbBranches { + dbBranchesMap[branch.Name] = branch + } + + deletedBranchesMap := make(map[string]bool, len(deletedBranches)) + for _, branchName := range deletedBranches { + deletedBranchesMap[branchName] = true + } + + for _, branchName := range pushedBranches { + branch, ok := dbBranchesMap[branchName] + deleted := deletedBranchesMap[branchName] + assert.True(t, ok, "branch %s not found in database", branchName) + assert.Equal(t, deleted, branch.IsDeleted, "IsDeleted of %s is %v, but it's expected to be %v", branchName, branch.IsDeleted, deleted) + commitID, err := gitRepo.GetBranchCommitID(branchName) + require.NoError(t, err) + assert.Equal(t, commitID, branch.CommitID) + } + + require.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user, repo.ID)) +} From a4bcfb8ef1d5b2b522f78c9560d53ddbdbb02218 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 6 Mar 2024 17:56:04 +0800 Subject: [PATCH 07/17] Detect broken git hooks (#29494) Detect broken git hooks by checking if the commit id of branches in DB is the same with the git repo. It can help #29338 #28277 and maybe more issues. Users could complain about actions, webhooks, and activities not working, but they were not aware that it is caused by broken git hooks unless they could see a warning. image It should be merged after #29493. Otherwise, users could see a ephemeral warning after committing and opening the repo home page immediately. And it also waits for #29495, since the doc link (the anchor part) will be updated. --------- Co-authored-by: wxiaoguang Co-authored-by: Giteabot --- options/locale/locale_en-US.ini | 1 + routers/web/repo/view.go | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 59aae68cae6bb..d30c8e521d031 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2643,6 +2643,7 @@ find_file.no_matching = No matching file found error.csv.too_large = Can't render this file because it is too large. error.csv.unexpected = Can't render this file because it contains an unexpected character in line %d and column %d. error.csv.invalid_field_count = Can't render this file because it has a wrong number of fields in line %d. +error.broken_git_hook = Git hooks of this repository seem to be broken. Please follow the documentation to fix them, then push some commits to refresh the status. [graphs] component_loading = Loading %s... diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 4df10fbea1adb..d47c926fa1780 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -998,6 +998,8 @@ func renderHomeCode(ctx *context.Context) { return } + checkOutdatedBranch(ctx) + checkCitationFile(ctx, entry) if ctx.Written() { return @@ -1064,6 +1066,31 @@ func renderHomeCode(ctx *context.Context) { ctx.HTML(http.StatusOK, tplRepoHome) } +func checkOutdatedBranch(ctx *context.Context) { + if !(ctx.Repo.IsAdmin() || ctx.Repo.IsOwner()) { + return + } + + // get the head commit of the branch since ctx.Repo.CommitID is not always the head commit of `ctx.Repo.BranchName` + commit, err := ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.BranchName) + if err != nil { + log.Error("GetBranchCommitID: %v", err) + // Don't return an error page, as it can be rechecked the next time the user opens the page. + return + } + + dbBranch, err := git_model.GetBranch(ctx, ctx.Repo.Repository.ID, ctx.Repo.BranchName) + if err != nil { + log.Error("GetBranch: %v", err) + // Don't return an error page, as it can be rechecked the next time the user opens the page. + return + } + + if dbBranch.CommitID != commit.ID.String() { + ctx.Flash.Warning(ctx.Tr("repo.error.broken_git_hook", "https://docs.gitea.com/help/faq#push-hook--webhook--actions-arent-running"), true) + } +} + // RenderUserCards render a page show users according the input template func RenderUserCards(ctx *context.Context, total int, getter func(opts db.ListOptions) ([]*user_model.User, error), tpl base.TplName) { page := ctx.FormInt("page") From c6cc392b55b71ae1787e86e75b7121c3769adcbe Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Wed, 6 Mar 2024 19:23:27 +0900 Subject: [PATCH 08/17] Fix wrong header of org project view page (#29626) Follow #29248 The project view page still using `user/overview/header.tmpl` Before: ![image](https://github.com/go-gitea/gitea/assets/18380374/9cb638a3-7cc6-4efa-979a-e2592007fd12) After: ![image](https://github.com/go-gitea/gitea/assets/18380374/62b0b2ea-8cb0-459f-b27a-bad3908eb1c0) --- templates/user/overview/header.tmpl | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/templates/user/overview/header.tmpl b/templates/user/overview/header.tmpl index 4fdaa70d879fa..cf5e21fa62184 100644 --- a/templates/user/overview/header.tmpl +++ b/templates/user/overview/header.tmpl @@ -29,19 +29,21 @@ {{end}} - - {{svg "octicon-rss"}} {{ctx.Locale.Tr "user.activity"}} - - {{if not .DisableStars}} - - {{svg "octicon-star"}} {{ctx.Locale.Tr "user.starred"}} - {{if .ContextUser.NumStars}} -
{{.ContextUser.NumStars}}
- {{end}} - - {{else}} - - {{svg "octicon-eye"}} {{ctx.Locale.Tr "user.watched"}} + {{if .ContextUser.IsIndividual}} + + {{svg "octicon-rss"}} {{ctx.Locale.Tr "user.activity"}} + {{if not .DisableStars}} + + {{svg "octicon-star"}} {{ctx.Locale.Tr "user.starred"}} + {{if .ContextUser.NumStars}} +
{{.ContextUser.NumStars}}
+ {{end}} + + {{else}} + + {{svg "octicon-eye"}} {{ctx.Locale.Tr "user.watched"}} + + {{end}} {{end}} From c381343a60b9c9eedb9dbc0931d1b87cb3c1f366 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 6 Mar 2024 19:25:00 +0800 Subject: [PATCH 09/17] Add a link for the recently pushed branch notification (#29627) --- templates/repo/code/recently_pushed_new_branches.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/repo/code/recently_pushed_new_branches.tmpl b/templates/repo/code/recently_pushed_new_branches.tmpl index fedba06fade76..5e603eae81621 100644 --- a/templates/repo/code/recently_pushed_new_branches.tmpl +++ b/templates/repo/code/recently_pushed_new_branches.tmpl @@ -2,7 +2,8 @@
{{$timeSince := TimeSince .CommitTime.AsTime ctx.Locale}} - {{ctx.Locale.Tr "repo.pulls.recently_pushed_new_branches" .Name $timeSince}} + {{$branchLink := HTMLFormat `%s` $.RepoLink (PathEscapeSegments .Name) .Name}} + {{ctx.Locale.Tr "repo.pulls.recently_pushed_new_branches" $branchLink $timeSince}}
{{ctx.Locale.Tr "repo.pulls.compare_changes"}} From 90a3f2d4b7ed3890d9655c0334444f86d89b7b30 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 6 Mar 2024 19:50:39 +0800 Subject: [PATCH 10/17] Avoid unexpected panic in graceful manager (#29629) There is a fundamental design problem of the "manager" and the "wait group". If nothing has started, the "Wait" just panics: sync: WaitGroup is reused before previous Wait has returned There is no clear solution besides a complete rewriting of the "manager" If there are some mistakes in the app.ini, end users would just see the "panic", but not the real error messages. A real case: #27643 This PR is just a quick fix for the annoying panic problem. --- modules/graceful/manager_unix.go | 10 +++++++++- modules/graceful/manager_windows.go | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/graceful/manager_unix.go b/modules/graceful/manager_unix.go index f4af4993d97ea..edf5fc248f99e 100644 --- a/modules/graceful/manager_unix.go +++ b/modules/graceful/manager_unix.go @@ -59,7 +59,15 @@ func (g *Manager) start() { go func() { defer close(startupDone) // Wait till we're done getting all the listeners and then close the unused ones - g.createServerWaitGroup.Wait() + func() { + // FIXME: there is a fundamental design problem of the "manager" and the "wait group". + // If nothing has started, the "Wait" just panics: sync: WaitGroup is reused before previous Wait has returned + // There is no clear solution besides a complete rewriting of the "manager" + defer func() { + _ = recover() + }() + g.createServerWaitGroup.Wait() + }() // Ignore the error here there's not much we can do with it, they're logged in the CloseProvidedListeners function _ = CloseProvidedListeners() g.notify(readyMsg) diff --git a/modules/graceful/manager_windows.go b/modules/graceful/manager_windows.go index 0248dcb24d22b..ecf30af3f3005 100644 --- a/modules/graceful/manager_windows.go +++ b/modules/graceful/manager_windows.go @@ -150,7 +150,15 @@ func (g *Manager) awaitServer(limit time.Duration) bool { c := make(chan struct{}) go func() { defer close(c) - g.createServerWaitGroup.Wait() + func() { + // FIXME: there is a fundamental design problem of the "manager" and the "wait group". + // If nothing has started, the "Wait" just panics: sync: WaitGroup is reused before previous Wait has returned + // There is no clear solution besides a complete rewriting of the "manager" + defer func() { + _ = recover() + }() + g.createServerWaitGroup.Wait() + }() }() if limit > 0 { select { From e308d25f1b2fe24b4735432b05e5e221879a2705 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 6 Mar 2024 20:17:19 +0800 Subject: [PATCH 11/17] Cache repository default branch commit status to reduce query on commit status table (#29444) After repository commit status has been introduced on dashaboard, the most top SQL comes from `GetLatestCommitStatusForPairs`. This PR adds a cache for the repository's default branch's latest combined commit status. When a new commit status updated, the cache will be marked as invalid. image --- routers/api/v1/repo/status.go | 4 +- routers/web/repo/repo.go | 28 ++-- .../repository/commitstatus/commitstatus.go | 135 ++++++++++++++++++ services/repository/files/commit.go | 48 ------- 4 files changed, 145 insertions(+), 70 deletions(-) create mode 100644 services/repository/commitstatus/commitstatus.go diff --git a/routers/api/v1/repo/status.go b/routers/api/v1/repo/status.go index 53711bedebfc5..9e36ea0aed3bf 100644 --- a/routers/api/v1/repo/status.go +++ b/routers/api/v1/repo/status.go @@ -14,7 +14,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/utils" "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/convert" - files_service "code.gitea.io/gitea/services/repository/files" + commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus" ) // NewCommitStatus creates a new CommitStatus @@ -64,7 +64,7 @@ func NewCommitStatus(ctx *context.APIContext) { Description: form.Description, Context: form.Context, } - if err := files_service.CreateCommitStatus(ctx, ctx.Repo.Repository, ctx.Doer, sha, status); err != nil { + if err := commitstatus_service.CreateCommitStatus(ctx, ctx.Repo.Repository, ctx.Doer, sha, status); err != nil { ctx.Error(http.StatusInternalServerError, "CreateCommitStatus", err) return } diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index f0caf199a2979..b54d29c580b4f 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -35,6 +35,7 @@ import ( "code.gitea.io/gitea/services/forms" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" + commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus" ) const ( @@ -634,30 +635,14 @@ func SearchRepo(ctx *context.Context) { return } - // collect the latest commit of each repo - // at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment - repoBranchNames := make(map[int64]string, len(repos)) - for _, repo := range repos { - repoBranchNames[repo.ID] = repo.DefaultBranch - } - - repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames) + latestCommitStatuses, err := commitstatus_service.FindReposLastestCommitStatuses(ctx, repos) if err != nil { - log.Error("FindBranchesByRepoAndBranchName: %v", err) - return - } - - // call the database O(1) times to get the commit statuses for all repos - repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll) - if err != nil { - log.Error("GetLatestCommitStatusForPairs: %v", err) + log.Error("FindReposLastestCommitStatuses: %v", err) return } results := make([]*repo_service.WebSearchRepository, len(repos)) for i, repo := range repos { - latestCommitStatus := git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID]) - results[i] = &repo_service.WebSearchRepository{ Repository: &api.Repository{ ID: repo.ID, @@ -671,8 +656,11 @@ func SearchRepo(ctx *context.Context) { Link: repo.Link(), Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate, }, - LatestCommitStatus: latestCommitStatus, - LocaleLatestCommitStatus: latestCommitStatus.LocaleString(ctx.Locale), + } + + if latestCommitStatuses[i] != nil { + results[i].LatestCommitStatus = latestCommitStatuses[i] + results[i].LocaleLatestCommitStatus = latestCommitStatuses[i].LocaleString(ctx.Locale) } } diff --git a/services/repository/commitstatus/commitstatus.go b/services/repository/commitstatus/commitstatus.go new file mode 100644 index 0000000000000..145fc7d53c4e9 --- /dev/null +++ b/services/repository/commitstatus/commitstatus.go @@ -0,0 +1,135 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package commitstatus + +import ( + "context" + "crypto/sha256" + "fmt" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/automerge" +) + +func getCacheKey(repoID int64, brancheName string) string { + hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName))) + return fmt.Sprintf("commit_status:%x", hashBytes) +} + +func updateCommitStatusCache(ctx context.Context, repoID int64, branchName string, status api.CommitStatusState) error { + c := cache.GetCache() + return c.Put(getCacheKey(repoID, branchName), string(status), 3*24*60) +} + +func deleteCommitStatusCache(ctx context.Context, repoID int64, branchName string) error { + c := cache.GetCache() + return c.Delete(getCacheKey(repoID, branchName)) +} + +// CreateCommitStatus creates a new CommitStatus given a bunch of parameters +// NOTE: All text-values will be trimmed from whitespaces. +// Requires: Repo, Creator, SHA +func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creator *user_model.User, sha string, status *git_model.CommitStatus) error { + repoPath := repo.RepoPath() + + // confirm that commit is exist + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return fmt.Errorf("OpenRepository[%s]: %w", repoPath, err) + } + defer closer.Close() + + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + + commit, err := gitRepo.GetCommit(sha) + if err != nil { + return fmt.Errorf("GetCommit[%s]: %w", sha, err) + } + if len(sha) != objectFormat.FullLength() { + // use complete commit sha + sha = commit.ID.String() + } + + if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{ + Repo: repo, + Creator: creator, + SHA: commit.ID, + CommitStatus: status, + }); err != nil { + return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) + } + + defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return fmt.Errorf("GetBranchCommit[%s]: %w", repo.DefaultBranch, err) + } + + if commit.ID.String() == defaultBranchCommit.ID.String() { // since one commit status updated, the combined commit status should be invalid + if err := deleteCommitStatusCache(ctx, repo.ID, repo.DefaultBranch); err != nil { + log.Error("deleteCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err) + } + } + + if status.State.IsSuccess() { + if err := automerge.MergeScheduledPullRequest(ctx, sha, repo); err != nil { + return fmt.Errorf("MergeScheduledPullRequest[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) + } + } + + return nil +} + +// FindReposLastestCommitStatuses loading repository default branch latest combinded commit status with cache +func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Repository) ([]*git_model.CommitStatus, error) { + results := make([]*git_model.CommitStatus, len(repos)) + c := cache.GetCache() + + for i, repo := range repos { + status, ok := c.Get(getCacheKey(repo.ID, repo.DefaultBranch)).(string) + if ok && status != "" { + results[i] = &git_model.CommitStatus{State: api.CommitStatusState(status)} + } + } + + // collect the latest commit of each repo + // at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment + repoBranchNames := make(map[int64]string, len(repos)) + for i, repo := range repos { + if results[i] == nil { + repoBranchNames[repo.ID] = repo.DefaultBranch + } + } + + repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames) + if err != nil { + return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err) + } + + // call the database O(1) times to get the commit statuses for all repos + repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll) + if err != nil { + return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err) + } + + for i, repo := range repos { + if results[i] == nil { + results[i] = git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID]) + if results[i].State != "" { + if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil { + log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err) + } + } + } + } + + return results, nil +} diff --git a/services/repository/files/commit.go b/services/repository/files/commit.go index 512aec7c814c4..e0dad292732e3 100644 --- a/services/repository/files/commit.go +++ b/services/repository/files/commit.go @@ -5,61 +5,13 @@ package files import ( "context" - "fmt" asymkey_model "code.gitea.io/gitea/models/asymkey" - git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/services/automerge" ) -// CreateCommitStatus creates a new CommitStatus given a bunch of parameters -// NOTE: All text-values will be trimmed from whitespaces. -// Requires: Repo, Creator, SHA -func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creator *user_model.User, sha string, status *git_model.CommitStatus) error { - repoPath := repo.RepoPath() - - // confirm that commit is exist - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return fmt.Errorf("OpenRepository[%s]: %w", repoPath, err) - } - defer closer.Close() - - objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) - - commit, err := gitRepo.GetCommit(sha) - if err != nil { - gitRepo.Close() - return fmt.Errorf("GetCommit[%s]: %w", sha, err) - } else if len(sha) != objectFormat.FullLength() { - // use complete commit sha - sha = commit.ID.String() - } - gitRepo.Close() - - if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{ - Repo: repo, - Creator: creator, - SHA: commit.ID, - CommitStatus: status, - }); err != nil { - return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) - } - - if status.State.IsSuccess() { - if err := automerge.MergeScheduledPullRequest(ctx, sha, repo); err != nil { - return fmt.Errorf("MergeScheduledPullRequest[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) - } - } - - return nil -} - // CountDivergingCommits determines how many commits a branch is ahead or behind the repository's base branch func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, branch string) (*git.DivergeObject, error) { divergence, err := git.GetDivergingCommits(ctx, repo.RepoPath(), repo.DefaultBranch, branch) From 1d2548949adf6046f330d27084efce6e63330e04 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 6 Mar 2024 21:12:44 +0800 Subject: [PATCH 12/17] Avoid issue info panic (#29625) Fix #29624 --- models/activities/action.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/models/activities/action.go b/models/activities/action.go index fcc97e387264d..36205eedd1f03 100644 --- a/models/activities/action.go +++ b/models/activities/action.go @@ -393,10 +393,14 @@ func (a *Action) GetCreate() time.Time { return a.CreatedUnix.AsTime() } -// GetIssueInfos returns a list of issues associated with -// the action. +// GetIssueInfos returns a list of associated information with the action. func (a *Action) GetIssueInfos() []string { - return strings.SplitN(a.Content, "|", 3) + // make sure it always returns 3 elements, because there are some access to the a[1] and a[2] without checking the length + ret := strings.SplitN(a.Content, "|", 3) + for len(ret) < 3 { + ret = append(ret, "") + } + return ret } // GetIssueTitle returns the title of first issue associated with the action. From c996e359585597d8c86e57edf91ada1564e5106c Mon Sep 17 00:00:00 2001 From: Rafael Heard Date: Wed, 6 Mar 2024 09:20:26 -0500 Subject: [PATCH 13/17] Move all login and account creation page labels to be above inputs (#29432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are a few inconsistencies within Gitea and this PR addresses one of them. This PR updates the sign-in page layout, including the register and openID tabs, to match the layout of the settings pages (/user/settings) for more consistency. This PR updates the following routes: `/user/login` `/user/sign_up` `/user/login/openid` `/user/forgot_password` `/user/link_account` `/user/recover_account` **Before** Screenshot 2024-02-05 at 8 27 24 AM **After** Screenshot 2024-02-05 at 8 26 39 AM This PR addresses a revert of the original PR due to this [comment](https://github.com/go-gitea/gitea/pull/28753#issuecomment-1956596817). --------- Co-authored-by: rafh --- templates/user/auth/activate.tmpl | 7 +++---- templates/user/auth/captcha.tmpl | 3 +-- templates/user/auth/change_passwd_inner.tmpl | 7 +++---- templates/user/auth/forgot_passwd.tmpl | 3 +-- templates/user/auth/prohibit_login.tmpl | 2 +- templates/user/auth/reset_passwd.tmpl | 6 ++---- templates/user/auth/signin_inner.tmpl | 15 ++++++--------- templates/user/auth/signin_openid.tmpl | 6 ++---- templates/user/auth/signup_inner.tmpl | 14 ++++++-------- templates/user/auth/signup_openid_register.tmpl | 9 ++++----- templates/user/auth/twofa.tmpl | 5 ++--- templates/user/auth/twofa_scratch.tmpl | 5 ++--- web_src/css/form.css | 10 +--------- 13 files changed, 34 insertions(+), 58 deletions(-) diff --git a/templates/user/auth/activate.tmpl b/templates/user/auth/activate.tmpl index e32a5d8707356..7f8ff0eb5a463 100644 --- a/templates/user/auth/activate.tmpl +++ b/templates/user/auth/activate.tmpl @@ -2,7 +2,7 @@
-
+ {{.CsrfTokenHtml}}

{{ctx.Locale.Tr "auth.active_your_account"}} @@ -10,12 +10,11 @@
{{template "base/alert" .}} {{if .NeedVerifyLocalPassword}} -
+
-
@@ -29,7 +28,7 @@
-
+
{{end}} diff --git a/templates/user/auth/captcha.tmpl b/templates/user/auth/captcha.tmpl index d4d1a82418fc2..0e9c2b9d2245e 100644 --- a/templates/user/auth/captcha.tmpl +++ b/templates/user/auth/captcha.tmpl @@ -1,9 +1,8 @@ {{if .EnableCaptcha}}{{if eq .CaptchaType "image"}}
- {{.Captcha.CreateHTML}}
-
+
diff --git a/templates/user/auth/change_passwd_inner.tmpl b/templates/user/auth/change_passwd_inner.tmpl index cffc798a64a74..01bbf500e5c6a 100644 --- a/templates/user/auth/change_passwd_inner.tmpl +++ b/templates/user/auth/change_passwd_inner.tmpl @@ -5,18 +5,17 @@ {{ctx.Locale.Tr "settings.change_password"}}

- + {{.CsrfTokenHtml}} -
+
-
+
-
diff --git a/templates/user/auth/forgot_passwd.tmpl b/templates/user/auth/forgot_passwd.tmpl index 21a630ec5e9df..55bcf63018393 100644 --- a/templates/user/auth/forgot_passwd.tmpl +++ b/templates/user/auth/forgot_passwd.tmpl @@ -12,13 +12,12 @@ {{if .IsResetSent}}

{{ctx.Locale.Tr "auth.reset_password_mail_sent_prompt" .Email .ResetPwdCodeLives}}

{{else if .IsResetRequest}} -
+
-
{{else if .IsResetDisable}} diff --git a/templates/user/auth/prohibit_login.tmpl b/templates/user/auth/prohibit_login.tmpl index 668aa20e71b97..962ddfa98c03e 100644 --- a/templates/user/auth/prohibit_login.tmpl +++ b/templates/user/auth/prohibit_login.tmpl @@ -2,7 +2,7 @@
-
+

{{ctx.Locale.Tr "auth.prohibit_login"}}

diff --git a/templates/user/auth/reset_passwd.tmpl b/templates/user/auth/reset_passwd.tmpl index 9fee30f554028..4d569e206c0e6 100644 --- a/templates/user/auth/reset_passwd.tmpl +++ b/templates/user/auth/reset_passwd.tmpl @@ -17,13 +17,12 @@
{{end}} {{if .IsResetForm}} -
+
{{if not .user_signed_in}}
-
@@ -42,7 +41,7 @@
{{else}} -
+
@@ -50,7 +49,6 @@ {{end}}
- {{if and .has_two_factor (not .scratch_code)}} {{ctx.Locale.Tr "auth.use_scratch_code"}} diff --git a/templates/user/auth/signin_inner.tmpl b/templates/user/auth/signin_inner.tmpl index 0d0064b02afc2..d7d3649a4d7b3 100644 --- a/templates/user/auth/signin_inner.tmpl +++ b/templates/user/auth/signin_inner.tmpl @@ -9,21 +9,20 @@ {{end}}
- + {{.CsrfTokenHtml}} -
+
{{if or (not .DisablePassword) .LinkAccountMode}} -
+
{{end}} {{if not .LinkAccountMode}}
-
@@ -33,8 +32,7 @@ {{template "user/auth/captcha" .}} -
- +
{{if .ShowRegistrationButton}} -
- +
{{ctx.Locale.Tr "auth.sign_up_now"}}
{{end}} @@ -60,7 +57,7 @@
{{range $provider := .OAuth2Providers}} - diff --git a/templates/user/auth/signin_openid.tmpl b/templates/user/auth/signin_openid.tmpl index 0428026aa8b5d..c1f392dc134df 100644 --- a/templates/user/auth/signin_openid.tmpl +++ b/templates/user/auth/signin_openid.tmpl @@ -8,12 +8,12 @@ OpenID
- + {{.CsrfTokenHtml}}
{{ctx.Locale.Tr "auth.openid_signin_desc"}}
-
+
-
-
diff --git a/templates/user/auth/signup_inner.tmpl b/templates/user/auth/signup_inner.tmpl index e930bd3d158a0..cfd826a0ce210 100644 --- a/templates/user/auth/signup_inner.tmpl +++ b/templates/user/auth/signup_inner.tmpl @@ -7,7 +7,7 @@ {{end}}
-
+ {{.CsrfTokenHtml}} {{if or (not .LinkAccountMode) (and .LinkAccountMode .LinkAccountModeRegister)}} {{template "base/alert" .}} @@ -15,21 +15,21 @@ {{if .DisableRegistration}}

{{ctx.Locale.Tr "auth.disable_register_prompt"}}

{{else}} -
+
-
+
{{if not .DisablePassword}} -
+
-
+
@@ -38,7 +38,6 @@ {{template "user/auth/captcha" .}}
-
diff --git a/templates/user/auth/twofa.tmpl b/templates/user/auth/twofa.tmpl index 5260178d13790..d24523917158b 100644 --- a/templates/user/auth/twofa.tmpl +++ b/templates/user/auth/twofa.tmpl @@ -2,20 +2,19 @@