From d4c2ebafde6466cd112bda078dd8b3aea8dd400d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Tue, 18 May 2021 19:34:05 +0200 Subject: [PATCH 01/17] Inherit submodules from template repository content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Steffen Schröter --- modules/git/submodule.go | 65 +++++++ modules/git/submodule_test.go | 19 ++ modules/git/tests/repos/repo4_submodules/HEAD | 1 + .../git/tests/repos/repo4_submodules/config | 4 + .../tests/repos/repo4_submodules/description | 1 + .../hooks/applypatch-msg.sample | 15 ++ .../repo4_submodules/hooks/commit-msg.sample | 24 +++ .../hooks/fsmonitor-watchman.sample | 173 ++++++++++++++++++ .../repo4_submodules/hooks/post-update.sample | 8 + .../hooks/pre-applypatch.sample | 14 ++ .../repo4_submodules/hooks/pre-commit.sample | 49 +++++ .../hooks/pre-merge-commit.sample | 13 ++ .../repo4_submodules/hooks/pre-push.sample | 53 ++++++ .../repo4_submodules/hooks/pre-rebase.sample | 169 +++++++++++++++++ .../repo4_submodules/hooks/pre-receive.sample | 24 +++ .../hooks/prepare-commit-msg.sample | 42 +++++ .../hooks/push-to-checkout.sample | 78 ++++++++ .../repo4_submodules/hooks/update.sample | 128 +++++++++++++ .../tests/repos/repo4_submodules/info/exclude | 6 + .../97/c3d30df0e6492348292600920a6482feaebb74 | Bin 0 -> 110 bytes .../c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 | Bin 0 -> 112 bytes .../e1/e59caba97193d48862d6809912043871f37437 | 2 + .../repos/repo4_submodules/refs/heads/master | 1 + modules/repository/generate.go | 12 ++ 24 files changed, 901 insertions(+) create mode 100644 modules/git/tests/repos/repo4_submodules/HEAD create mode 100644 modules/git/tests/repos/repo4_submodules/config create mode 100644 modules/git/tests/repos/repo4_submodules/description create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/post-update.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample create mode 100755 modules/git/tests/repos/repo4_submodules/hooks/update.sample create mode 100644 modules/git/tests/repos/repo4_submodules/info/exclude create mode 100644 modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74 create mode 100644 modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 create mode 100644 modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 create mode 100644 modules/git/tests/repos/repo4_submodules/refs/heads/master diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 231827f1e989a..4d8beb664b3cc 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -22,6 +22,11 @@ type SubModule struct { URL string } +type SubModuleCommit struct { + Name string + Commit string +} + // SubModuleFile represents a file with submodule type. type SubModuleFile struct { *Commit @@ -122,3 +127,63 @@ func (sf *SubModuleFile) RefURL(urlPrefix, repoFullName, sshDomain string) strin func (sf *SubModuleFile) RefID() string { return sf.refID } + +func GetSubmoduleCommits(repoPath string) []SubModuleCommit { + var submodules []SubModuleCommit + + submoduleOut, err := NewCommand("config", "-f", ".gitmodules", "--list", "--name-only"). + RunInDir(repoPath) + + if err != nil { + // Command fails if there are no or invalid submodules, just return an empty list + return submodules + } + + for _, line := range strings.Split(strings.TrimSuffix(submoduleOut, "\n"), "\n") { + if len(line) < len("submodule.x.url") || + !strings.HasPrefix(line, "submodule.") || + !strings.HasSuffix(line, ".url") { + + continue + } + + name := line[len("submodule.") : len(line)-len(".url")] + name = strings.TrimSpace(name) + + if len(name) == 0 { + log("Submodule skipped because it has no name") + continue + } + + commit, err := NewCommand("submodule", "status", name). + RunInDir(repoPath) + + // If no commit was found for the module skip it + if err != nil { + log("Submodule %s skipped because it has no commit", name) + continue + } + + if len(commit) > 0 { + commit = commit[1:] + } + + fields := strings.Fields(commit) + + if len(fields) == 0 { + log("Submodule %s skipped because it has no valid commit", name) + continue + } + + commit = fields[0] + + if len(commit) != 40 { + log("Submodule %s skipped due to malformed commit hash", name) + continue + } + + submodules = append(submodules, SubModuleCommit{name, commit}) + } + + return submodules +} diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index ff8dc579f6d51..618203e929a28 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -5,8 +5,10 @@ package git import ( + "path/filepath" "testing" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -41,3 +43,20 @@ func TestGetRefURL(t *testing.T) { assert.EqualValues(t, kase.expect, getRefURL(kase.refURL, kase.prefixURL, kase.parentPath, kase.SSHDomain)) } } + +func TestRepository_GetSubmoduleCommits(t *testing.T) { + bareRepo1Path := filepath.Join(testReposDir, "repo4_submodules") + clonedPath, err := cloneRepo(bareRepo1Path, testReposDir, "repo4_TestRepository_GetSubmoduleCommits") + assert.NoError(t, err) + defer util.RemoveAll(clonedPath) + + submodules := GetSubmoduleCommits(clonedPath) + + assert.EqualValues(t, len(submodules), 2) + + assert.EqualValues(t, submodules[0].Name, "libtest") + assert.EqualValues(t, submodules[0].Commit, "1234567890123456789012345678901234567890") + + assert.EqualValues(t, submodules[1].Name, "<°)))><") + assert.EqualValues(t, submodules[1].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c") +} diff --git a/modules/git/tests/repos/repo4_submodules/HEAD b/modules/git/tests/repos/repo4_submodules/HEAD new file mode 100644 index 0000000000000..cb089cd89a7d7 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/modules/git/tests/repos/repo4_submodules/config b/modules/git/tests/repos/repo4_submodules/config new file mode 100644 index 0000000000000..07d359d07cf1e --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/modules/git/tests/repos/repo4_submodules/description b/modules/git/tests/repos/repo4_submodules/description new file mode 100644 index 0000000000000..498b267a8c781 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample b/modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample new file mode 100755 index 0000000000000..a5d7b84a67345 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +: diff --git a/modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample b/modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample new file mode 100755 index 0000000000000..b58d1184a9d43 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample b/modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample new file mode 100755 index 0000000000000..14ed0aa42de0f --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample @@ -0,0 +1,173 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 2) and last update token +# formatted as a string and outputs to stdout a new update token and +# all files that have been modified since the update token. Paths must +# be relative to the root of the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $last_update_token) = @ARGV; + +# Uncomment for debugging +# print STDERR "$0 $version $last_update_token\n"; + +# Check the hook interface version +if ($version ne 2) { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree = get_working_dir(); + +my $retry = 1; + +my $json_pkg; +eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; +} or do { + require JSON::PP; + $json_pkg = "JSON::PP"; +}; + +launch_watchman(); + +sub launch_watchman { + my $o = watchman_query(); + if (is_work_tree_watched($o)) { + output_result($o->{clock}, @{$o->{files}}); + } +} + +sub output_result { + my ($clockid, @files) = @_; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # binmode $fh, ":utf8"; + # print $fh "$clockid\n@files\n"; + # close $fh; + + binmode STDOUT, ":utf8"; + print $clockid; + print "\0"; + local $, = "\0"; + print @files; +} + +sub watchman_clock { + my $response = qx/watchman clock "$git_work_tree"/; + die "Failed to get clock id on '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + return $json_pkg->new->utf8->decode($response); +} + +sub watchman_query { + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $last_update_token but not from the .git folder. + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + if (substr($last_update_token, 0, 1) eq "c") { + $last_update_token = "\"$last_update_token\""; + } + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $last_update_token, + "fields": ["name"], + "expression": ["not", ["dirname", ".git"]] + }] + END + + # Uncomment for debugging the watchman query + # open (my $fh, ">", ".git/watchman-query.json"); + # print $fh $query; + # close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + # Uncomment for debugging the watch response + # open ($fh, ">", ".git/watchman-response.json"); + # print $fh $response; + # close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + return $json_pkg->new->utf8->decode($response); +} + +sub is_work_tree_watched { + my ($output) = @_; + my $error = $output->{error}; + if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) { + $retry--; + my $response = qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + $output = $json_pkg->new->utf8->decode($response); + $error = $output->{error}; + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # close $fh; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + my $o = watchman_clock(); + $error = $output->{error}; + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + output_result($o->{clock}, ("/")); + $last_update_token = $o->{clock}; + + eval { launch_watchman() }; + return 0; + } + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + return 1; +} + +sub get_working_dir { + my $working_dir; + if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $working_dir = Win32::GetCwd(); + $working_dir =~ tr/\\/\//; + } else { + require Cwd; + $working_dir = Cwd::cwd(); + } + + return $working_dir; +} diff --git a/modules/git/tests/repos/repo4_submodules/hooks/post-update.sample b/modules/git/tests/repos/repo4_submodules/hooks/post-update.sample new file mode 100755 index 0000000000000..ec17ec1939b7c --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample new file mode 100755 index 0000000000000..4142082bcb939 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +precommit="$(git rev-parse --git-path hooks/pre-commit)" +test -x "$precommit" && exec "$precommit" ${1+"$@"} +: diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample new file mode 100755 index 0000000000000..e144712c85c05 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=$(git hash-object -t tree /dev/null) +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --type=bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample new file mode 100755 index 0000000000000..399eab1924e39 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample @@ -0,0 +1,13 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git merge" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message to +# stderr if it wants to stop the merge commit. +# +# To enable this hook, rename this file to "pre-merge-commit". + +. git-sh-setup +test -x "$GIT_DIR/hooks/pre-commit" && + exec "$GIT_DIR/hooks/pre-commit" +: diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample new file mode 100755 index 0000000000000..4ce688d32b753 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample @@ -0,0 +1,53 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +zero=$(git hash-object --stdin &2 "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample new file mode 100755 index 0000000000000..6cbef5c370d8c --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up to date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +<<\DOC_END + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". + +DOC_END diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample new file mode 100755 index 0000000000000..a1fd29ec14823 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to make use of push options. +# The example simply echoes all push options that start with 'echoback=' +# and rejects all pushes when the "reject" push option is used. +# +# To enable this hook, rename this file to "pre-receive". + +if test -n "$GIT_PUSH_OPTION_COUNT" +then + i=0 + while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" + do + eval "value=\$GIT_PUSH_OPTION_$i" + case "$value" in + echoback=*) + echo "echo from the pre-receive-hook: ${value#*=}" >&2 + ;; + reject) + exit 1 + esac + i=$((i + 1)) + done +fi diff --git a/modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample b/modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample new file mode 100755 index 0000000000000..10fa14c5ab013 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample @@ -0,0 +1,42 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first one removes the +# "# Please enter the commit message..." help message. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +COMMIT_MSG_FILE=$1 +COMMIT_SOURCE=$2 +SHA1=$3 + +/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" + +# case "$COMMIT_SOURCE,$SHA1" in +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;; +# *) ;; +# esac + +# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" +# if test -z "$COMMIT_SOURCE" +# then +# /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" +# fi diff --git a/modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample b/modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample new file mode 100755 index 0000000000000..af5a0c0018b5e --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample @@ -0,0 +1,78 @@ +#!/bin/sh + +# An example hook script to update a checked-out tree on a git push. +# +# This hook is invoked by git-receive-pack(1) when it reacts to git +# push and updates reference(s) in its repository, and when the push +# tries to update the branch that is currently checked out and the +# receive.denyCurrentBranch configuration variable is set to +# updateInstead. +# +# By default, such a push is refused if the working tree and the index +# of the remote repository has any difference from the currently +# checked out commit; when both the working tree and the index match +# the current commit, they are updated to match the newly pushed tip +# of the branch. This hook is to be used to override the default +# behaviour; however the code below reimplements the default behaviour +# as a starting point for convenient modification. +# +# The hook receives the commit with which the tip of the current +# branch is going to be updated: +commit=$1 + +# It can exit with a non-zero status to refuse the push (when it does +# so, it must not modify the index or the working tree). +die () { + echo >&2 "$*" + exit 1 +} + +# Or it can make any necessary changes to the working tree and to the +# index to bring them to the desired state when the tip of the current +# branch is updated to the new commit, and exit with a zero status. +# +# For example, the hook can simply run git read-tree -u -m HEAD "$1" +# in order to emulate git fetch that is run in the reverse direction +# with git push, as the two-tree form of git read-tree -u -m is +# essentially the same as git switch or git checkout that switches +# branches while keeping the local changes in the working tree that do +# not interfere with the difference between the branches. + +# The below is a more-or-less exact translation to shell of the C code +# for the default behaviour for git's push-to-checkout hook defined in +# the push_to_deploy() function in builtin/receive-pack.c. +# +# Note that the hook will be executed from the repository directory, +# not from the working tree, so if you want to perform operations on +# the working tree, you will have to adapt your code accordingly, e.g. +# by adding "cd .." or using relative paths. + +if ! git update-index -q --ignore-submodules --refresh +then + die "Up-to-date check failed" +fi + +if ! git diff-files --quiet --ignore-submodules -- +then + die "Working directory has unstaged changes" +fi + +# This is a rough translation of: +# +# head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX +if git cat-file -e HEAD 2>/dev/null +then + head=HEAD +else + head=$(git hash-object -t tree --stdin &2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --type=bool hooks.allowunannotated) +allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch) +denycreatebranch=$(git config --type=bool hooks.denycreatebranch) +allowdeletetag=$(git config --type=bool hooks.allowdeletetag) +allowmodifytag=$(git config --type=bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero=$(git hash-object --stdin &2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/modules/git/tests/repos/repo4_submodules/info/exclude b/modules/git/tests/repos/repo4_submodules/info/exclude new file mode 100644 index 0000000000000..a5196d1be8fb5 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74 b/modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74 new file mode 100644 index 0000000000000000000000000000000000000000..7596090b49fc8304d4c5599e09fa97e65fd86ceb GIT binary patch literal 110 zcmV-!0FnQA0V^p=O;s>7G+;0^FfcPQQP4}zEXmDJDa}bOW;p&J<*nxySLM?ymfqiE ze(9X_7FiQRGXo${usO6rQ&ZEPh6%)`10HMe&nHoGW2LJ#7 literal 0 HcmV?d00001 diff --git a/modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 b/modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 new file mode 100644 index 0000000000000000000000000000000000000000..e3a13c156dce4d306921d33f4414867aea4b58c0 GIT binary patch literal 112 zcmV-$0FVE80ZYosPf{>6HDQP@E=|hKPbtkwRZz;wOe#q&E>Vi*;w(rk$xyIWfQoQ& zmKNmzxfvxT1;tkS`Z@W@i8&eh#U=Vs1$yb3C0xix*&N!Ssi|pa12jtk3ZO>9WZ>cu SqxDO23-n=fVB-M&&@!0EvNJaT literal 0 HcmV?d00001 diff --git a/modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 b/modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 new file mode 100644 index 0000000000000..a8d6e5c17c8f2 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 @@ -0,0 +1,2 @@ +x[ +0E*_$M5tifBk Iŕ7k~9ܘܠ.j O "z`#IirF͹$%|4)?t=:K#[$D^ӒyHU/f?G \ No newline at end of file diff --git a/modules/git/tests/repos/repo4_submodules/refs/heads/master b/modules/git/tests/repos/repo4_submodules/refs/heads/master new file mode 100644 index 0000000000000..102bc34da8ca0 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/refs/heads/master @@ -0,0 +1 @@ +e1e59caba97193d48862d6809912043871f37437 diff --git a/modules/repository/generate.go b/modules/repository/generate.go index 1ba457fb3a551..128807b634040 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -120,6 +120,9 @@ func generateRepoCommit(repo, templateRepo, generateRepo *models.Repository, tmp return fmt.Errorf("git clone: %v", err) } + // Get active submodules from the template + submodules := git.GetSubmoduleCommits(tmpDir) + if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil { return fmt.Errorf("remove git dir: %v", err) } @@ -182,6 +185,15 @@ func generateRepoCommit(repo, templateRepo, generateRepo *models.Repository, tmp return fmt.Errorf("git remote add: %v", err) } + // Reapply the submodules by updating the index + for _, submodule := range submodules { + if stdout, err := git.NewCommand("update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name). + RunInDirWithEnv(tmpDir, env); err != nil { + log.Error("Unable to add %v as submodule to temporary repo %s: stdout %s\nError: %v", submodule.Name, tmpDir, stdout, err) + return fmt.Errorf("git update-index --add --cacheinfo 160000 %s %s: %v", submodule.Commit, submodule.Name, err) + } + } + return initRepoCommit(tmpDir, repo, repo.Owner, templateRepo.DefaultBranch) } From ffcd7da12635718be08f159467723de5915657bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Thu, 24 Jun 2021 22:00:15 +0200 Subject: [PATCH 02/17] Use correct log and fix lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Steffen Schröter --- modules/git/submodule.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 4d8beb664b3cc..a25b6ac911517 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -12,6 +12,8 @@ import ( "path" "regexp" "strings" + + gitea_log "code.gitea.io/gitea/modules/log" ) var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+@)?([a-zA-Z0-9._-]+):(.*)$`) @@ -22,6 +24,7 @@ type SubModule struct { URL string } +// SubModuleCommit submodule name and commit from a repository type SubModuleCommit struct { Name string Commit string @@ -128,6 +131,7 @@ func (sf *SubModuleFile) RefID() string { return sf.refID } +// GetSubmoduleCommits Returns a list of active submodules in the repository func GetSubmoduleCommits(repoPath string) []SubModuleCommit { var submodules []SubModuleCommit @@ -151,7 +155,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { name = strings.TrimSpace(name) if len(name) == 0 { - log("Submodule skipped because it has no name") + gitea_log.Info("Submodule skipped because it has no name") continue } @@ -160,7 +164,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { // If no commit was found for the module skip it if err != nil { - log("Submodule %s skipped because it has no commit", name) + gitea_log.Info("Submodule %s skipped because it has no commit", name) continue } @@ -171,14 +175,14 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { fields := strings.Fields(commit) if len(fields) == 0 { - log("Submodule %s skipped because it has no valid commit", name) + gitea_log.Info("Submodule %s skipped because it has no valid commit", name) continue } commit = fields[0] if len(commit) != 40 { - log("Submodule %s skipped due to malformed commit hash", name) + gitea_log.Info("Submodule %s skipped due to malformed commit hash", name) continue } From 4b338924b98ea4ef191874a3809fd77a3d8943de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Sun, 27 Jun 2021 11:53:18 +0200 Subject: [PATCH 03/17] Changed logging level --- modules/git/submodule.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index a25b6ac911517..3112fa4ce30d8 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -155,7 +155,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { name = strings.TrimSpace(name) if len(name) == 0 { - gitea_log.Info("Submodule skipped because it has no name") + gitea_log.Debug("Submodule skipped because it has no name") continue } @@ -164,7 +164,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { // If no commit was found for the module skip it if err != nil { - gitea_log.Info("Submodule %s skipped because it has no commit", name) + gitea_log.Debug("Submodule %s skipped because it has no commit", name) continue } @@ -175,14 +175,14 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { fields := strings.Fields(commit) if len(fields) == 0 { - gitea_log.Info("Submodule %s skipped because it has no valid commit", name) + gitea_log.Debug("Submodule %s skipped because it has no valid commit", name) continue } commit = fields[0] if len(commit) != 40 { - gitea_log.Info("Submodule %s skipped due to malformed commit hash", name) + gitea_log.Debug("Submodule %s skipped due to malformed commit hash", name) continue } From 7fa0bc1fbe5ee05a82adaeed0196a8c3da8e7f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Sun, 27 Jun 2021 12:39:23 +0200 Subject: [PATCH 04/17] Removed logging workaround --- modules/git/submodule.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 3112fa4ce30d8..8a995e49ee7e0 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -13,7 +13,7 @@ import ( "regexp" "strings" - gitea_log "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/log" ) var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+@)?([a-zA-Z0-9._-]+):(.*)$`) @@ -155,7 +155,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { name = strings.TrimSpace(name) if len(name) == 0 { - gitea_log.Debug("Submodule skipped because it has no name") + log.Debug("Submodule skipped because it has no name") continue } @@ -164,7 +164,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { // If no commit was found for the module skip it if err != nil { - gitea_log.Debug("Submodule %s skipped because it has no commit", name) + log.Debug("Submodule %s skipped because it has no commit", name) continue } @@ -175,14 +175,14 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { fields := strings.Fields(commit) if len(fields) == 0 { - gitea_log.Debug("Submodule %s skipped because it has no valid commit", name) + log.Debug("Submodule %s skipped because it has no valid commit", name) continue } commit = fields[0] if len(commit) != 40 { - gitea_log.Debug("Submodule %s skipped due to malformed commit hash", name) + log.Debug("Submodule %s skipped due to malformed commit hash", name) continue } From 6df3da1daf402b9a75164fd9dcbf1d0d65b2f892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Sun, 27 Jun 2021 14:24:34 +0200 Subject: [PATCH 05/17] Added function to add submodule indexes --- modules/git/submodule.go | 13 +++++++++++++ modules/repository/generate.go | 9 ++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 8a995e49ee7e0..e8cbb81180755 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -191,3 +191,16 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { return submodules } + +// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present. +func AddSubmoduleIndexes(repoPath string, submodules []SubModuleCommit) error { + for _, submodule := range submodules { + if stdout, err := NewCommand("update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name). + RunInDir(repoPath); err != nil { + log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Name, repoPath, stdout, err) + return err + } + } + + return nil +} diff --git a/modules/repository/generate.go b/modules/repository/generate.go index 128807b634040..544ca49de3ac1 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -185,13 +185,8 @@ func generateRepoCommit(repo, templateRepo, generateRepo *models.Repository, tmp return fmt.Errorf("git remote add: %v", err) } - // Reapply the submodules by updating the index - for _, submodule := range submodules { - if stdout, err := git.NewCommand("update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name). - RunInDirWithEnv(tmpDir, env); err != nil { - log.Error("Unable to add %v as submodule to temporary repo %s: stdout %s\nError: %v", submodule.Name, tmpDir, stdout, err) - return fmt.Errorf("git update-index --add --cacheinfo 160000 %s %s: %v", submodule.Commit, submodule.Name, err) - } + if err := git.AddSubmoduleIndexes(tmpDir, submodules); err != nil { + return fmt.Errorf("Failed to add submodules: %v", err) } return initRepoCommit(tmpDir, repo, repo.Owner, templateRepo.DefaultBranch) From 4113314f3715e8b4fded549e79ffe52fe75d8790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Mon, 5 Jul 2021 19:43:05 +0200 Subject: [PATCH 06/17] Use pipeline for GetSubmoduleCommits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Steffen Schröter --- modules/git/submodule.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index e8cbb81180755..566dfb281b877 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -6,7 +6,9 @@ package git import ( + "bufio" "fmt" + "io" "net" "net/url" "path" @@ -133,25 +135,40 @@ func (sf *SubModuleFile) RefID() string { // GetSubmoduleCommits Returns a list of active submodules in the repository func GetSubmoduleCommits(repoPath string) []SubModuleCommit { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("config", "-f", ".gitmodules", "--list", "--name-only").RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + var submodules []SubModuleCommit + bufReader := bufio.NewReader(stdoutReader) - submoduleOut, err := NewCommand("config", "-f", ".gitmodules", "--list", "--name-only"). - RunInDir(repoPath) + for { + line, err := bufReader.ReadString('\n') - if err != nil { - // Command fails if there are no or invalid submodules, just return an empty list - return submodules - } + if err != nil { + break + } - for _, line := range strings.Split(strings.TrimSuffix(submoduleOut, "\n"), "\n") { - if len(line) < len("submodule.x.url") || + if len(line) < len("submodule.x.url\n") || !strings.HasPrefix(line, "submodule.") || - !strings.HasSuffix(line, ".url") { + !strings.HasSuffix(line, ".url\n") { continue } - name := line[len("submodule.") : len(line)-len(".url")] + name := line[len("submodule.") : len(line)-len(".url\n")] name = strings.TrimSpace(name) if len(name) == 0 { From d264e186c1f8a6ca996b855524df3989493c974f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Sun, 13 Feb 2022 11:36:09 +0100 Subject: [PATCH 07/17] Added context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Steffen Schröter --- modules/git/submodule.go | 15 +++++++-------- modules/git/submodule_test.go | 4 ++-- modules/repository/generate.go | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 5bab36aa01b60..c4bbf0268b0c1 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -7,6 +7,7 @@ package git import ( "bufio" + "context" "fmt" "io" "net" @@ -132,7 +133,7 @@ func (sf *SubModuleFile) RefID() string { } // GetSubmoduleCommits Returns a list of active submodules in the repository -func GetSubmoduleCommits(repoPath string) []SubModuleCommit { +func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit { stdoutReader, stdoutWriter := io.Pipe() defer func() { _ = stdoutReader.Close() @@ -141,7 +142,7 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { go func() { stderrBuilder := &strings.Builder{} - err := NewCommand("config", "-f", ".gitmodules", "--list", "--name-only").RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + err := NewCommand(ctx, "config", "-f", ".gitmodules", "--list", "--name-only").RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) if err != nil { _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) } else { @@ -154,7 +155,6 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { for { line, err := bufReader.ReadString('\n') - if err != nil { break } @@ -174,10 +174,9 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { continue } - commit, err := NewCommand("submodule", "status", name). - RunInDir(repoPath) - // If no commit was found for the module skip it + commit, err := NewCommand(ctx, "submodule", "status", name). + RunInDir(repoPath) if err != nil { log.Debug("Submodule %s skipped because it has no commit", name) continue @@ -208,9 +207,9 @@ func GetSubmoduleCommits(repoPath string) []SubModuleCommit { } // AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present. -func AddSubmoduleIndexes(repoPath string, submodules []SubModuleCommit) error { +func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error { for _, submodule := range submodules { - if stdout, err := NewCommand("update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name). + if stdout, err := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name). RunInDir(repoPath); err != nil { log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Name, repoPath, stdout, err) return err diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index ffe592d222f3f..faadae1e84cf1 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -46,11 +46,11 @@ func TestGetRefURL(t *testing.T) { func TestRepository_GetSubmoduleCommits(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo4_submodules") - clonedPath, err := cloneRepo(bareRepo1Path, testReposDir, "repo4_TestRepository_GetSubmoduleCommits") + clonedPath, err := cloneRepo(bareRepo1Path, "repo4_TestRepository_GetSubmoduleCommits") assert.NoError(t, err) defer util.RemoveAll(clonedPath) - submodules := GetSubmoduleCommits(clonedPath) + submodules := GetSubmoduleCommits(DefaultContext, clonedPath) assert.EqualValues(t, len(submodules), 2) diff --git a/modules/repository/generate.go b/modules/repository/generate.go index 45c47eccb58ab..4dd8027c8757d 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -123,7 +123,7 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } // Get active submodules from the template - submodules := git.GetSubmoduleCommits(tmpDir) + submodules := git.GetSubmoduleCommits(ctx, tmpDir) if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil { return fmt.Errorf("remove git dir: %v", err) @@ -187,7 +187,7 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return fmt.Errorf("git remote add: %v", err) } - if err := git.AddSubmoduleIndexes(tmpDir, submodules); err != nil { + if err := git.AddSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { return fmt.Errorf("Failed to add submodules: %v", err) } From 4e1eb81efdaee70e5e20c5815e9796738733197e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Sun, 13 Feb 2022 14:14:27 +0100 Subject: [PATCH 08/17] Empty commit to retrigger CI From 9a23dfa7552607dad9d1fdb9636ddbf1a38c1b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Schr=C3=B6ter?= Date: Tue, 15 Feb 2022 19:52:26 +0100 Subject: [PATCH 09/17] Use RunWithContext --- modules/git/submodule.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index c4bbf0268b0c1..8eed28ad90ab9 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -141,8 +141,14 @@ func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit }() go func() { - stderrBuilder := &strings.Builder{} - err := NewCommand(ctx, "config", "-f", ".gitmodules", "--list", "--name-only").RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + stderrBuilder := strings.Builder{} + err := NewCommand(ctx, "config", "-f", ".gitmodules", "--list", "--name-only").RunWithContext(&RunContext{ + Timeout: -1, + Dir: repoPath, + Stdout: stdoutWriter, + Stderr: &stderrBuilder, + }) + if err != nil { _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) } else { From 68cc6529a22717d41a8642f6b1245ffe25ae3b27 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 31 Dec 2024 20:32:52 +0800 Subject: [PATCH 10/17] remove unnecessary files --- .../tests/repos/repo4_submodules/description | 1 - .../hooks/applypatch-msg.sample | 15 -- .../repo4_submodules/hooks/commit-msg.sample | 24 --- .../hooks/fsmonitor-watchman.sample | 173 ------------------ .../repo4_submodules/hooks/post-update.sample | 8 - .../hooks/pre-applypatch.sample | 14 -- .../repo4_submodules/hooks/pre-commit.sample | 49 ----- .../hooks/pre-merge-commit.sample | 13 -- .../repo4_submodules/hooks/pre-push.sample | 53 ------ .../repo4_submodules/hooks/pre-rebase.sample | 169 ----------------- .../repo4_submodules/hooks/pre-receive.sample | 24 --- .../hooks/prepare-commit-msg.sample | 42 ----- .../hooks/push-to-checkout.sample | 78 -------- .../repo4_submodules/hooks/update.sample | 128 ------------- .../tests/repos/repo4_submodules/info/exclude | 6 - 15 files changed, 797 deletions(-) delete mode 100644 modules/git/tests/repos/repo4_submodules/description delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/post-update.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample delete mode 100755 modules/git/tests/repos/repo4_submodules/hooks/update.sample delete mode 100644 modules/git/tests/repos/repo4_submodules/info/exclude diff --git a/modules/git/tests/repos/repo4_submodules/description b/modules/git/tests/repos/repo4_submodules/description deleted file mode 100644 index 498b267a8c781..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/description +++ /dev/null @@ -1 +0,0 @@ -Unnamed repository; edit this file 'description' to name the repository. diff --git a/modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample b/modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample deleted file mode 100755 index a5d7b84a67345..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# -# An example hook script to check the commit log message taken by -# applypatch from an e-mail message. -# -# The hook should exit with non-zero status after issuing an -# appropriate message if it wants to stop the commit. The hook is -# allowed to edit the commit message file. -# -# To enable this hook, rename this file to "applypatch-msg". - -. git-sh-setup -commitmsg="$(git rev-parse --git-path hooks/commit-msg)" -test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} -: diff --git a/modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample b/modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample deleted file mode 100755 index b58d1184a9d43..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -# -# An example hook script to check the commit log message. -# Called by "git commit" with one argument, the name of the file -# that has the commit message. The hook should exit with non-zero -# status after issuing an appropriate message if it wants to stop the -# commit. The hook is allowed to edit the commit message file. -# -# To enable this hook, rename this file to "commit-msg". - -# Uncomment the below to add a Signed-off-by line to the message. -# Doing this in a hook is a bad idea in general, but the prepare-commit-msg -# hook is more suited to it. -# -# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') -# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" - -# This example catches duplicate Signed-off-by lines. - -test "" = "$(grep '^Signed-off-by: ' "$1" | - sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { - echo >&2 Duplicate Signed-off-by lines. - exit 1 -} diff --git a/modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample b/modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample deleted file mode 100755 index 14ed0aa42de0f..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use IPC::Open2; - -# An example hook script to integrate Watchman -# (https://facebook.github.io/watchman/) with git to speed up detecting -# new and modified files. -# -# The hook is passed a version (currently 2) and last update token -# formatted as a string and outputs to stdout a new update token and -# all files that have been modified since the update token. Paths must -# be relative to the root of the working tree and separated by a single NUL. -# -# To enable this hook, rename this file to "query-watchman" and set -# 'git config core.fsmonitor .git/hooks/query-watchman' -# -my ($version, $last_update_token) = @ARGV; - -# Uncomment for debugging -# print STDERR "$0 $version $last_update_token\n"; - -# Check the hook interface version -if ($version ne 2) { - die "Unsupported query-fsmonitor hook version '$version'.\n" . - "Falling back to scanning...\n"; -} - -my $git_work_tree = get_working_dir(); - -my $retry = 1; - -my $json_pkg; -eval { - require JSON::XS; - $json_pkg = "JSON::XS"; - 1; -} or do { - require JSON::PP; - $json_pkg = "JSON::PP"; -}; - -launch_watchman(); - -sub launch_watchman { - my $o = watchman_query(); - if (is_work_tree_watched($o)) { - output_result($o->{clock}, @{$o->{files}}); - } -} - -sub output_result { - my ($clockid, @files) = @_; - - # Uncomment for debugging watchman output - # open (my $fh, ">", ".git/watchman-output.out"); - # binmode $fh, ":utf8"; - # print $fh "$clockid\n@files\n"; - # close $fh; - - binmode STDOUT, ":utf8"; - print $clockid; - print "\0"; - local $, = "\0"; - print @files; -} - -sub watchman_clock { - my $response = qx/watchman clock "$git_work_tree"/; - die "Failed to get clock id on '$git_work_tree'.\n" . - "Falling back to scanning...\n" if $? != 0; - - return $json_pkg->new->utf8->decode($response); -} - -sub watchman_query { - my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') - or die "open2() failed: $!\n" . - "Falling back to scanning...\n"; - - # In the query expression below we're asking for names of files that - # changed since $last_update_token but not from the .git folder. - # - # To accomplish this, we're using the "since" generator to use the - # recency index to select candidate nodes and "fields" to limit the - # output to file names only. Then we're using the "expression" term to - # further constrain the results. - if (substr($last_update_token, 0, 1) eq "c") { - $last_update_token = "\"$last_update_token\""; - } - my $query = <<" END"; - ["query", "$git_work_tree", { - "since": $last_update_token, - "fields": ["name"], - "expression": ["not", ["dirname", ".git"]] - }] - END - - # Uncomment for debugging the watchman query - # open (my $fh, ">", ".git/watchman-query.json"); - # print $fh $query; - # close $fh; - - print CHLD_IN $query; - close CHLD_IN; - my $response = do {local $/; }; - - # Uncomment for debugging the watch response - # open ($fh, ">", ".git/watchman-response.json"); - # print $fh $response; - # close $fh; - - die "Watchman: command returned no output.\n" . - "Falling back to scanning...\n" if $response eq ""; - die "Watchman: command returned invalid output: $response\n" . - "Falling back to scanning...\n" unless $response =~ /^\{/; - - return $json_pkg->new->utf8->decode($response); -} - -sub is_work_tree_watched { - my ($output) = @_; - my $error = $output->{error}; - if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) { - $retry--; - my $response = qx/watchman watch "$git_work_tree"/; - die "Failed to make watchman watch '$git_work_tree'.\n" . - "Falling back to scanning...\n" if $? != 0; - $output = $json_pkg->new->utf8->decode($response); - $error = $output->{error}; - die "Watchman: $error.\n" . - "Falling back to scanning...\n" if $error; - - # Uncomment for debugging watchman output - # open (my $fh, ">", ".git/watchman-output.out"); - # close $fh; - - # Watchman will always return all files on the first query so - # return the fast "everything is dirty" flag to git and do the - # Watchman query just to get it over with now so we won't pay - # the cost in git to look up each individual file. - my $o = watchman_clock(); - $error = $output->{error}; - - die "Watchman: $error.\n" . - "Falling back to scanning...\n" if $error; - - output_result($o->{clock}, ("/")); - $last_update_token = $o->{clock}; - - eval { launch_watchman() }; - return 0; - } - - die "Watchman: $error.\n" . - "Falling back to scanning...\n" if $error; - - return 1; -} - -sub get_working_dir { - my $working_dir; - if ($^O =~ 'msys' || $^O =~ 'cygwin') { - $working_dir = Win32::GetCwd(); - $working_dir =~ tr/\\/\//; - } else { - require Cwd; - $working_dir = Cwd::cwd(); - } - - return $working_dir; -} diff --git a/modules/git/tests/repos/repo4_submodules/hooks/post-update.sample b/modules/git/tests/repos/repo4_submodules/hooks/post-update.sample deleted file mode 100755 index ec17ec1939b7c..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/post-update.sample +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# An example hook script to prepare a packed repository for use over -# dumb transports. -# -# To enable this hook, rename this file to "post-update". - -exec git update-server-info diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample deleted file mode 100755 index 4142082bcb939..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/pre-applypatch.sample +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# -# An example hook script to verify what is about to be committed -# by applypatch from an e-mail message. -# -# The hook should exit with non-zero status after issuing an -# appropriate message if it wants to stop the commit. -# -# To enable this hook, rename this file to "pre-applypatch". - -. git-sh-setup -precommit="$(git rev-parse --git-path hooks/pre-commit)" -test -x "$precommit" && exec "$precommit" ${1+"$@"} -: diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample deleted file mode 100755 index e144712c85c05..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/pre-commit.sample +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# -# An example hook script to verify what is about to be committed. -# Called by "git commit" with no arguments. The hook should -# exit with non-zero status after issuing an appropriate message if -# it wants to stop the commit. -# -# To enable this hook, rename this file to "pre-commit". - -if git rev-parse --verify HEAD >/dev/null 2>&1 -then - against=HEAD -else - # Initial commit: diff against an empty tree object - against=$(git hash-object -t tree /dev/null) -fi - -# If you want to allow non-ASCII filenames set this variable to true. -allownonascii=$(git config --type=bool hooks.allownonascii) - -# Redirect output to stderr. -exec 1>&2 - -# Cross platform projects tend to avoid non-ASCII filenames; prevent -# them from being added to the repository. We exploit the fact that the -# printable range starts at the space character and ends with tilde. -if [ "$allownonascii" != "true" ] && - # Note that the use of brackets around a tr range is ok here, (it's - # even required, for portability to Solaris 10's /usr/bin/tr), since - # the square bracket bytes happen to fall in the designated range. - test $(git diff --cached --name-only --diff-filter=A -z $against | - LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 -then - cat <<\EOF -Error: Attempt to add a non-ASCII file name. - -This can cause problems if you want to work with people on other platforms. - -To be portable it is advisable to rename the file. - -If you know what you are doing you can disable this check using: - - git config hooks.allownonascii true -EOF - exit 1 -fi - -# If there are whitespace errors, print the offending file names and fail. -exec git diff-index --check --cached $against -- diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample deleted file mode 100755 index 399eab1924e39..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/pre-merge-commit.sample +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# -# An example hook script to verify what is about to be committed. -# Called by "git merge" with no arguments. The hook should -# exit with non-zero status after issuing an appropriate message to -# stderr if it wants to stop the merge commit. -# -# To enable this hook, rename this file to "pre-merge-commit". - -. git-sh-setup -test -x "$GIT_DIR/hooks/pre-commit" && - exec "$GIT_DIR/hooks/pre-commit" -: diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample deleted file mode 100755 index 4ce688d32b753..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/pre-push.sample +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -# An example hook script to verify what is about to be pushed. Called by "git -# push" after it has checked the remote status, but before anything has been -# pushed. If this script exits with a non-zero status nothing will be pushed. -# -# This hook is called with the following parameters: -# -# $1 -- Name of the remote to which the push is being done -# $2 -- URL to which the push is being done -# -# If pushing without using a named remote those arguments will be equal. -# -# Information about the commits which are being pushed is supplied as lines to -# the standard input in the form: -# -# -# -# This sample shows how to prevent push of commits where the log message starts -# with "WIP" (work in progress). - -remote="$1" -url="$2" - -zero=$(git hash-object --stdin &2 "Found WIP commit in $local_ref, not pushing" - exit 1 - fi - fi -done - -exit 0 diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample deleted file mode 100755 index 6cbef5c370d8c..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/pre-rebase.sample +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2006, 2008 Junio C Hamano -# -# The "pre-rebase" hook is run just before "git rebase" starts doing -# its job, and can prevent the command from running by exiting with -# non-zero status. -# -# The hook is called with the following parameters: -# -# $1 -- the upstream the series was forked from. -# $2 -- the branch being rebased (or empty when rebasing the current branch). -# -# This sample shows how to prevent topic branches that are already -# merged to 'next' branch from getting rebased, because allowing it -# would result in rebasing already published history. - -publish=next -basebranch="$1" -if test "$#" = 2 -then - topic="refs/heads/$2" -else - topic=`git symbolic-ref HEAD` || - exit 0 ;# we do not interrupt rebasing detached HEAD -fi - -case "$topic" in -refs/heads/??/*) - ;; -*) - exit 0 ;# we do not interrupt others. - ;; -esac - -# Now we are dealing with a topic branch being rebased -# on top of master. Is it OK to rebase it? - -# Does the topic really exist? -git show-ref -q "$topic" || { - echo >&2 "No such branch $topic" - exit 1 -} - -# Is topic fully merged to master? -not_in_master=`git rev-list --pretty=oneline ^master "$topic"` -if test -z "$not_in_master" -then - echo >&2 "$topic is fully merged to master; better remove it." - exit 1 ;# we could allow it, but there is no point. -fi - -# Is topic ever merged to next? If so you should not be rebasing it. -only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` -only_next_2=`git rev-list ^master ${publish} | sort` -if test "$only_next_1" = "$only_next_2" -then - not_in_topic=`git rev-list "^$topic" master` - if test -z "$not_in_topic" - then - echo >&2 "$topic is already up to date with master" - exit 1 ;# we could allow it, but there is no point. - else - exit 0 - fi -else - not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` - /usr/bin/perl -e ' - my $topic = $ARGV[0]; - my $msg = "* $topic has commits already merged to public branch:\n"; - my (%not_in_next) = map { - /^([0-9a-f]+) /; - ($1 => 1); - } split(/\n/, $ARGV[1]); - for my $elem (map { - /^([0-9a-f]+) (.*)$/; - [$1 => $2]; - } split(/\n/, $ARGV[2])) { - if (!exists $not_in_next{$elem->[0]}) { - if ($msg) { - print STDERR $msg; - undef $msg; - } - print STDERR " $elem->[1]\n"; - } - } - ' "$topic" "$not_in_next" "$not_in_master" - exit 1 -fi - -<<\DOC_END - -This sample hook safeguards topic branches that have been -published from being rewound. - -The workflow assumed here is: - - * Once a topic branch forks from "master", "master" is never - merged into it again (either directly or indirectly). - - * Once a topic branch is fully cooked and merged into "master", - it is deleted. If you need to build on top of it to correct - earlier mistakes, a new topic branch is created by forking at - the tip of the "master". This is not strictly necessary, but - it makes it easier to keep your history simple. - - * Whenever you need to test or publish your changes to topic - branches, merge them into "next" branch. - -The script, being an example, hardcodes the publish branch name -to be "next", but it is trivial to make it configurable via -$GIT_DIR/config mechanism. - -With this workflow, you would want to know: - -(1) ... if a topic branch has ever been merged to "next". Young - topic branches can have stupid mistakes you would rather - clean up before publishing, and things that have not been - merged into other branches can be easily rebased without - affecting other people. But once it is published, you would - not want to rewind it. - -(2) ... if a topic branch has been fully merged to "master". - Then you can delete it. More importantly, you should not - build on top of it -- other people may already want to - change things related to the topic as patches against your - "master", so if you need further changes, it is better to - fork the topic (perhaps with the same name) afresh from the - tip of "master". - -Let's look at this example: - - o---o---o---o---o---o---o---o---o---o "next" - / / / / - / a---a---b A / / - / / / / - / / c---c---c---c B / - / / / \ / - / / / b---b C \ / - / / / / \ / - ---o---o---o---o---o---o---o---o---o---o---o "master" - - -A, B and C are topic branches. - - * A has one fix since it was merged up to "next". - - * B has finished. It has been fully merged up to "master" and "next", - and is ready to be deleted. - - * C has not merged to "next" at all. - -We would want to allow C to be rebased, refuse A, and encourage -B to be deleted. - -To compute (1): - - git rev-list ^master ^topic next - git rev-list ^master next - - if these match, topic has not merged in next at all. - -To compute (2): - - git rev-list master..topic - - if this is empty, it is fully merged to "master". - -DOC_END diff --git a/modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample b/modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample deleted file mode 100755 index a1fd29ec14823..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/pre-receive.sample +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -# -# An example hook script to make use of push options. -# The example simply echoes all push options that start with 'echoback=' -# and rejects all pushes when the "reject" push option is used. -# -# To enable this hook, rename this file to "pre-receive". - -if test -n "$GIT_PUSH_OPTION_COUNT" -then - i=0 - while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" - do - eval "value=\$GIT_PUSH_OPTION_$i" - case "$value" in - echoback=*) - echo "echo from the pre-receive-hook: ${value#*=}" >&2 - ;; - reject) - exit 1 - esac - i=$((i + 1)) - done -fi diff --git a/modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample b/modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample deleted file mode 100755 index 10fa14c5ab013..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/prepare-commit-msg.sample +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# -# An example hook script to prepare the commit log message. -# Called by "git commit" with the name of the file that has the -# commit message, followed by the description of the commit -# message's source. The hook's purpose is to edit the commit -# message file. If the hook fails with a non-zero status, -# the commit is aborted. -# -# To enable this hook, rename this file to "prepare-commit-msg". - -# This hook includes three examples. The first one removes the -# "# Please enter the commit message..." help message. -# -# The second includes the output of "git diff --name-status -r" -# into the message, just before the "git status" output. It is -# commented because it doesn't cope with --amend or with squashed -# commits. -# -# The third example adds a Signed-off-by line to the message, that can -# still be edited. This is rarely a good idea. - -COMMIT_MSG_FILE=$1 -COMMIT_SOURCE=$2 -SHA1=$3 - -/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" - -# case "$COMMIT_SOURCE,$SHA1" in -# ,|template,) -# /usr/bin/perl -i.bak -pe ' -# print "\n" . `git diff --cached --name-status -r` -# if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;; -# *) ;; -# esac - -# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') -# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" -# if test -z "$COMMIT_SOURCE" -# then -# /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" -# fi diff --git a/modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample b/modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample deleted file mode 100755 index af5a0c0018b5e..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/hooks/push-to-checkout.sample +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh - -# An example hook script to update a checked-out tree on a git push. -# -# This hook is invoked by git-receive-pack(1) when it reacts to git -# push and updates reference(s) in its repository, and when the push -# tries to update the branch that is currently checked out and the -# receive.denyCurrentBranch configuration variable is set to -# updateInstead. -# -# By default, such a push is refused if the working tree and the index -# of the remote repository has any difference from the currently -# checked out commit; when both the working tree and the index match -# the current commit, they are updated to match the newly pushed tip -# of the branch. This hook is to be used to override the default -# behaviour; however the code below reimplements the default behaviour -# as a starting point for convenient modification. -# -# The hook receives the commit with which the tip of the current -# branch is going to be updated: -commit=$1 - -# It can exit with a non-zero status to refuse the push (when it does -# so, it must not modify the index or the working tree). -die () { - echo >&2 "$*" - exit 1 -} - -# Or it can make any necessary changes to the working tree and to the -# index to bring them to the desired state when the tip of the current -# branch is updated to the new commit, and exit with a zero status. -# -# For example, the hook can simply run git read-tree -u -m HEAD "$1" -# in order to emulate git fetch that is run in the reverse direction -# with git push, as the two-tree form of git read-tree -u -m is -# essentially the same as git switch or git checkout that switches -# branches while keeping the local changes in the working tree that do -# not interfere with the difference between the branches. - -# The below is a more-or-less exact translation to shell of the C code -# for the default behaviour for git's push-to-checkout hook defined in -# the push_to_deploy() function in builtin/receive-pack.c. -# -# Note that the hook will be executed from the repository directory, -# not from the working tree, so if you want to perform operations on -# the working tree, you will have to adapt your code accordingly, e.g. -# by adding "cd .." or using relative paths. - -if ! git update-index -q --ignore-submodules --refresh -then - die "Up-to-date check failed" -fi - -if ! git diff-files --quiet --ignore-submodules -- -then - die "Working directory has unstaged changes" -fi - -# This is a rough translation of: -# -# head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX -if git cat-file -e HEAD 2>/dev/null -then - head=HEAD -else - head=$(git hash-object -t tree --stdin &2 - echo " (if you want, you could supply GIT_DIR then run" >&2 - echo " $0 )" >&2 - exit 1 -fi - -if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then - echo "usage: $0 " >&2 - exit 1 -fi - -# --- Config -allowunannotated=$(git config --type=bool hooks.allowunannotated) -allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch) -denycreatebranch=$(git config --type=bool hooks.denycreatebranch) -allowdeletetag=$(git config --type=bool hooks.allowdeletetag) -allowmodifytag=$(git config --type=bool hooks.allowmodifytag) - -# check for no description -projectdesc=$(sed -e '1q' "$GIT_DIR/description") -case "$projectdesc" in -"Unnamed repository"* | "") - echo "*** Project description file hasn't been set" >&2 - exit 1 - ;; -esac - -# --- Check types -# if $newrev is 0000...0000, it's a commit to delete a ref. -zero=$(git hash-object --stdin &2 - echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 - exit 1 - fi - ;; - refs/tags/*,delete) - # delete tag - if [ "$allowdeletetag" != "true" ]; then - echo "*** Deleting a tag is not allowed in this repository" >&2 - exit 1 - fi - ;; - refs/tags/*,tag) - # annotated tag - if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 - then - echo "*** Tag '$refname' already exists." >&2 - echo "*** Modifying a tag is not allowed in this repository." >&2 - exit 1 - fi - ;; - refs/heads/*,commit) - # branch - if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then - echo "*** Creating a branch is not allowed in this repository" >&2 - exit 1 - fi - ;; - refs/heads/*,delete) - # delete branch - if [ "$allowdeletebranch" != "true" ]; then - echo "*** Deleting a branch is not allowed in this repository" >&2 - exit 1 - fi - ;; - refs/remotes/*,commit) - # tracking branch - ;; - refs/remotes/*,delete) - # delete tracking branch - if [ "$allowdeletebranch" != "true" ]; then - echo "*** Deleting a tracking branch is not allowed in this repository" >&2 - exit 1 - fi - ;; - *) - # Anything else (is there anything else?) - echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 - exit 1 - ;; -esac - -# --- Finished -exit 0 diff --git a/modules/git/tests/repos/repo4_submodules/info/exclude b/modules/git/tests/repos/repo4_submodules/info/exclude deleted file mode 100644 index a5196d1be8fb5..0000000000000 --- a/modules/git/tests/repos/repo4_submodules/info/exclude +++ /dev/null @@ -1,6 +0,0 @@ -# git ls-files --others --exclude-from=.git/info/exclude -# Lines that start with '#' are comments. -# For a project mostly in C, the following would be a good set of -# exclude patterns (uncomment them if you want to use them): -# *.[oa] -# *~ From 06b9eb3f030b5c2cc499f92fb3768922bfcaa4d5 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 31 Dec 2024 21:04:56 +0800 Subject: [PATCH 11/17] fix build --- modules/git/commit_submodule_file.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/git/commit_submodule_file.go b/modules/git/commit_submodule_file.go index 8da9f273569b3..87c1566188db5 100644 --- a/modules/git/commit_submodule_file.go +++ b/modules/git/commit_submodule_file.go @@ -115,6 +115,12 @@ func (sf *CommitSubModuleFile) RefID() string { return sf.refID } +// SubModuleCommit submodule name and commit from a repository +type SubModuleCommit struct { + Name string + Commit string +} + // GetSubmoduleCommits Returns a list of active submodules in the repository func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit { stdoutReader, stdoutWriter := io.Pipe() @@ -164,8 +170,7 @@ func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit } // If no commit was found for the module skip it - commit, _, err := NewCommand(ctx, "submodule", "status", name). - RunStdString(&RunOpts{Dir: repoPath}) + commit, _, err := NewCommand(ctx, "submodule", "status").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repoPath}) if err != nil { log.Debug("Submodule %s skipped because it has no commit", name) continue @@ -198,7 +203,7 @@ func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit // AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present. func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error { for _, submodule := range submodules { - if stdout, _, err := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name). + if stdout, _, err := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Name). RunStdString(&RunOpts{Dir: repoPath}); err != nil { log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Name, repoPath, stdout, err) return err From 8e2e4918cef1d79ddb8672e94f809c3e48f0bce5 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 31 Dec 2024 22:38:13 +0800 Subject: [PATCH 12/17] fix --- modules/git/batch_reader.go | 6 +- modules/git/commit_submodule_file.go | 103 ----------------- modules/git/commit_submodule_file_test.go | 19 ---- modules/git/parse_nogogit.go | 113 ++++++++++--------- modules/git/pipeline/lfs_nogogit.go | 2 +- modules/git/submodule.go | 63 +++++++++++ modules/git/submodule_test.go | 23 ++++ modules/git/tree.go | 4 +- services/repository/generate.go | 129 +++++++++++----------- 9 files changed, 217 insertions(+), 245 deletions(-) create mode 100644 modules/git/submodule.go create mode 100644 modules/git/submodule_test.go diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index 532dbad9894b5..33e54fe75cb3e 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -242,7 +242,7 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { return out } -// ParseTreeLine reads an entry from a tree in a cat-file --batch stream +// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream // This carefully avoids allocations - except where fnameBuf is too small. // It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations // @@ -250,7 +250,7 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { // SP NUL // // We don't attempt to convert the raw HASH to save a lot of time -func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { +func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { var readBytes []byte // Read the Mode & fname @@ -260,7 +260,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu } idx := bytes.IndexByte(readBytes, ' ') if idx < 0 { - log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes) + log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes) return mode, fname, sha, n, &ErrNotExist{} } diff --git a/modules/git/commit_submodule_file.go b/modules/git/commit_submodule_file.go index 87c1566188db5..bdec35f682987 100644 --- a/modules/git/commit_submodule_file.go +++ b/modules/git/commit_submodule_file.go @@ -5,17 +5,12 @@ package git import ( - "bufio" - "context" "fmt" - "io" "net" "net/url" "path" "regexp" "strings" - - "code.gitea.io/gitea/modules/log" ) var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+@)?([a-zA-Z0-9._-]+):(.*)$`) @@ -114,101 +109,3 @@ func (sf *CommitSubModuleFile) RefURL(urlPrefix, repoFullName, sshDomain string) func (sf *CommitSubModuleFile) RefID() string { return sf.refID } - -// SubModuleCommit submodule name and commit from a repository -type SubModuleCommit struct { - Name string - Commit string -} - -// GetSubmoduleCommits Returns a list of active submodules in the repository -func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit { - stdoutReader, stdoutWriter := io.Pipe() - defer func() { - _ = stdoutReader.Close() - _ = stdoutWriter.Close() - }() - - go func() { - stderrBuilder := strings.Builder{} - err := NewCommand(ctx, "config", "-f", ".gitmodules", "--list", "--name-only").Run(&RunOpts{ - Timeout: -1, - Dir: repoPath, - Stdout: stdoutWriter, - Stderr: &stderrBuilder, - }) - - if err != nil { - _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) - } else { - _ = stdoutWriter.Close() - } - }() - - var submodules []SubModuleCommit - bufReader := bufio.NewReader(stdoutReader) - - for { - line, err := bufReader.ReadString('\n') - if err != nil { - break - } - - if len(line) < len("submodule.x.url\n") || - !strings.HasPrefix(line, "submodule.") || - !strings.HasSuffix(line, ".url\n") { - - continue - } - - name := line[len("submodule.") : len(line)-len(".url\n")] - name = strings.TrimSpace(name) - - if len(name) == 0 { - log.Debug("Submodule skipped because it has no name") - continue - } - - // If no commit was found for the module skip it - commit, _, err := NewCommand(ctx, "submodule", "status").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repoPath}) - if err != nil { - log.Debug("Submodule %s skipped because it has no commit", name) - continue - } - - if len(commit) > 0 { - commit = commit[1:] - } - - fields := strings.Fields(commit) - - if len(fields) == 0 { - log.Debug("Submodule %s skipped because it has no valid commit", name) - continue - } - - commit = fields[0] - - if len(commit) != 40 { - log.Debug("Submodule %s skipped due to malformed commit hash", name) - continue - } - - submodules = append(submodules, SubModuleCommit{name, commit}) - } - - return submodules -} - -// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present. -func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error { - for _, submodule := range submodules { - if stdout, _, err := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Name). - RunStdString(&RunOpts{Dir: repoPath}); err != nil { - log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Name, repoPath, stdout, err) - return err - } - } - - return nil -} diff --git a/modules/git/commit_submodule_file_test.go b/modules/git/commit_submodule_file_test.go index 9efe20b8c194b..473b996b820ab 100644 --- a/modules/git/commit_submodule_file_test.go +++ b/modules/git/commit_submodule_file_test.go @@ -4,10 +4,8 @@ package git import ( - "path/filepath" "testing" - "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -42,20 +40,3 @@ func TestCommitSubModuleFileGetRefURL(t *testing.T) { assert.EqualValues(t, kase.expect, getRefURL(kase.refURL, kase.prefixURL, kase.parentPath, kase.SSHDomain)) } } - -func TestRepository_GetSubmoduleCommits(t *testing.T) { - bareRepo1Path := filepath.Join(testReposDir, "repo4_submodules") - clonedPath, err := cloneRepo(bareRepo1Path, "repo4_TestRepository_GetSubmoduleCommits") - assert.NoError(t, err) - defer util.RemoveAll(clonedPath) - - submodules := GetSubmoduleCommits(DefaultContext, clonedPath) - - assert.EqualValues(t, len(submodules), 2) - - assert.EqualValues(t, submodules[0].Name, "libtest") - assert.EqualValues(t, submodules[0].Commit, "1234567890123456789012345678901234567890") - - assert.EqualValues(t, submodules[1].Name, "<°)))><") - assert.EqualValues(t, submodules[1].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c") -} diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index 546b38be37964..5f8f9d26a9c90 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -23,68 +23,79 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { var sepSpace = []byte{' '} -func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { +func parseLsTreeLine(line []byte) (*TreeEntry, error) { + // expect line to be of the form: + // \t + // \t + var err error + posTab := bytes.IndexByte(line, '\t') + if posTab == -1 { + return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) + } + + entry := new(TreeEntry) + + entryAttrs := line[:posTab] + entryName := line[posTab+1:] + + entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) + _ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type + entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) + if len(entryAttrs) > 0 { + entrySize := entryAttrs // the last field is the space-padded-size + entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) + entry.sized = true + } + + switch string(entryMode) { + case "100644": + entry.entryMode = EntryModeBlob + case "100755": + entry.entryMode = EntryModeExec + case "120000": + entry.entryMode = EntryModeSymlink + case "160000": + entry.entryMode = EntryModeCommit + case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons + entry.entryMode = EntryModeTree + default: + return nil, fmt.Errorf("unknown type: %v", string(entryMode)) + } + + entry.ID, err = NewIDFromString(string(entryObjectID)) + if err != nil { + return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) + } + + if len(entryName) > 0 && entryName[0] == '"' { + entry.name, err = strconv.Unquote(string(entryName)) + if err != nil { + return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) + } + } else { + entry.name = string(entryName) + } + return entry, nil +} + +// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory +func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1) for pos := 0; pos < len(data); { - // expect line to be of the form: - // \t - // \t posEnd := bytes.IndexByte(data[pos:], '\n') if posEnd == -1 { posEnd = len(data) } else { posEnd += pos } - line := data[pos:posEnd] - posTab := bytes.IndexByte(line, '\t') - if posTab == -1 { - return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) - } - - entry := new(TreeEntry) - entry.ptree = ptree - entryAttrs := line[:posTab] - entryName := line[posTab+1:] - - entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) - _ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type - entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) - if len(entryAttrs) > 0 { - entrySize := entryAttrs // the last field is the space-padded-size - entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) - entry.sized = true - } - - switch string(entryMode) { - case "100644": - entry.entryMode = EntryModeBlob - case "100755": - entry.entryMode = EntryModeExec - case "120000": - entry.entryMode = EntryModeSymlink - case "160000": - entry.entryMode = EntryModeCommit - case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons - entry.entryMode = EntryModeTree - default: - return nil, fmt.Errorf("unknown type: %v", string(entryMode)) - } - - entry.ID, err = NewIDFromString(string(entryObjectID)) + line := data[pos:posEnd] + entry, err := parseLsTreeLine(line) if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) - } - - if len(entryName) > 0 && entryName[0] == '"' { - entry.name, err = strconv.Unquote(string(entryName)) - if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) - } - } else { - entry.name = string(entryName) + return nil, err } + entry.ptree = ptree pos = posEnd + 1 entries = append(entries, entry) @@ -100,7 +111,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio. loop: for sz > 0 { - mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf) + mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf) if err != nil { if err == io.EOF { break loop diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index b22805c1327e9..92e35c5a1028a 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err case "tree": var n int64 for n < size { - mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf) + mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf) if err != nil { return nil, err } diff --git a/modules/git/submodule.go b/modules/git/submodule.go new file mode 100644 index 0000000000000..37c3b902d27c0 --- /dev/null +++ b/modules/git/submodule.go @@ -0,0 +1,63 @@ +package git + +import ( + "bufio" + "context" + "fmt" + "os" + + "code.gitea.io/gitea/modules/log" +) + +// SubModuleCommit submodule name and commit from a repository +type SubModuleCommit struct { + Path string + Commit string +} + +// GetSubmoduleCommits returns a list of submodules paths and their commits from a repository +// This function is only for generating new repos based on existing template, the template couldn't be too large. +func GetSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []SubModuleCommit, _ error) { + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return nil, err + } + opts := &RunOpts{ + Dir: repoPath, + Stdout: stdoutWriter, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + defer stdoutReader.Close() + + scanner := bufio.NewScanner(stdoutReader) + for scanner.Scan() { + entry, err := parseLsTreeLine(scanner.Bytes()) + if err != nil { + cancel() + return err + } + if entry.IsSubModule() { + submoduleCommits = append(submoduleCommits, SubModuleCommit{Path: entry.Name(), Commit: entry.ID.String()}) + } + } + return scanner.Err() + }, + } + err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts) + if err != nil { + return nil, fmt.Errorf("GetSubmoduleCommits: error running git ls-tree: %v", err) + } + return submoduleCommits, nil +} + +// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present. +func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error { + for _, submodule := range submodules { + cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path) + if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil { + log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err) + return err + } + } + return nil +} diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go new file mode 100644 index 0000000000000..c62e8c453ee88 --- /dev/null +++ b/modules/git/submodule_test.go @@ -0,0 +1,23 @@ +package git + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRepository_GetSubmoduleCommits(t *testing.T) { + testRepoPath := filepath.Join(testReposDir, "repo4_submodules") + submodules, err := GetSubmoduleCommits(DefaultContext, testRepoPath) + require.NoError(t, err) + + assert.EqualValues(t, len(submodules), 2) + + assert.EqualValues(t, submodules[0].Path, "<°)))><") + assert.EqualValues(t, submodules[0].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c") + + assert.EqualValues(t, submodules[1].Path, "libtest") + assert.EqualValues(t, submodules[1].Commit, "1234567890123456789012345678901234567890") +} diff --git a/modules/git/tree.go b/modules/git/tree.go index d35dc58d8d0ed..5a644f6c87aa1 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -17,7 +17,7 @@ func NewTree(repo *Repository, id ObjectID) *Tree { } } -// SubTree get a sub tree by the sub dir path +// SubTree get a subtree by the sub dir path func (t *Tree) SubTree(rpath string) (*Tree, error) { if len(rpath) == 0 { return t, nil @@ -63,7 +63,7 @@ func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error return filelist, err } -// GetTreePathLatestCommitID returns the latest commit of a tree path +// GetTreePathLatestCommit returns the latest commit of a tree path func (repo *Repository) GetTreePathLatestCommit(refName, treePath string) (*Commit, error) { stdout, _, err := NewCommand(repo.Ctx, "rev-list", "-1"). AddDynamicArguments(refName).AddDashesAndList(treePath). diff --git a/services/repository/generate.go b/services/repository/generate.go index 28c3b55df13ba..090e79552d7fc 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "os" - "path" "path/filepath" "regexp" "strconv" @@ -123,7 +122,7 @@ func (gt *GiteaTemplate) Globs() []glob.Glob { return gt.globs } -func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { +func readGiteaTemplateFile(tmpDir string) (*GiteaTemplate, error) { gtPath := filepath.Join(tmpDir, ".gitea", "template") if _, err := os.Stat(gtPath); os.IsNotExist(err) { return nil, nil @@ -136,12 +135,55 @@ func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { return nil, err } - gt := &GiteaTemplate{ - Path: gtPath, - Content: content, + return &GiteaTemplate{Path: gtPath, Content: content}, nil +} + +func processGiteaTemplateFile(tmpDir string, templateRepo, generateRepo *repo_model.Repository, giteaTemplateFile *GiteaTemplate) error { + if err := util.Remove(giteaTemplateFile.Path); err != nil { + return fmt.Errorf("remove .giteatemplate: %w", err) } + if len(giteaTemplateFile.Globs()) == 0 { + return nil // Avoid walking tree if there are no globs + } + tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" + return filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if d.IsDir() { + return nil + } + + base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash) + for _, g := range giteaTemplateFile.Globs() { + if g.Match(base) { + content, err := os.ReadFile(path) + if err != nil { + return err + } + + generatedContent := []byte(generateExpansion(string(content), templateRepo, generateRepo, false)) + if err := os.WriteFile(path, generatedContent, 0o644); err != nil { + return err + } + + substPath := filepath.FromSlash(filepath.Join(tmpDirSlash, generateExpansion(base, templateRepo, generateRepo, true))) - return gt, nil + // Create parent subdirectories if needed or continue silently if it exists + if err = os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil { + return err + } + + // Substitute filename variables + if err = os.Rename(path, substPath); err != nil { + return err + } + break + } + } + return nil + }) // end: WalkDir } func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error { @@ -168,85 +210,40 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } // Get active submodules from the template - submodules := git.GetSubmoduleCommits(ctx, tmpDir) + submodules, err := git.GetSubmoduleCommits(ctx, tmpDir) + if err != nil { + return fmt.Errorf("GetSubmoduleCommits: %w", err) + } - if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil { + if err = util.RemoveAll(filepath.Join(tmpDir, ".git")); err != nil { return fmt.Errorf("remove git dir: %w", err) } // Variable expansion - gt, err := checkGiteaTemplate(tmpDir) + giteaTemplateFile, err := readGiteaTemplateFile(tmpDir) if err != nil { - return fmt.Errorf("checkGiteaTemplate: %w", err) + return fmt.Errorf("readGiteaTemplateFile: %w", err) } - if gt != nil { - if err := util.Remove(gt.Path); err != nil { - return fmt.Errorf("remove .giteatemplate: %w", err) - } - - // Avoid walking tree if there are no globs - if len(gt.Globs()) > 0 { - tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" - if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - - if d.IsDir() { - return nil - } - - base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash) - for _, g := range gt.Globs() { - if g.Match(base) { - content, err := os.ReadFile(path) - if err != nil { - return err - } - - if err := os.WriteFile(path, - []byte(generateExpansion(string(content), templateRepo, generateRepo, false)), - 0o644); err != nil { - return err - } - - substPath := filepath.FromSlash(filepath.Join(tmpDirSlash, - generateExpansion(base, templateRepo, generateRepo, true))) - - // Create parent subdirectories if needed or continue silently if it exists - if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil { - return err - } - - // Substitute filename variables - if err := os.Rename(path, substPath); err != nil { - return err - } - - break - } - } - return nil - }); err != nil { - return err - } + if giteaTemplateFile != nil { + err = processGiteaTemplateFile(tmpDir, templateRepo, generateRepo, giteaTemplateFile) + if err != nil { + return err } } - if err := git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil { + if err = git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil { return err } - repoPath := repo.RepoPath() - if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repoPath). + if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repo.RepoPath()). RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil { log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err) return fmt.Errorf("git remote add: %w", err) } - if err := git.AddSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { - return fmt.Errorf("Failed to add submodules: %v", err) + if err = git.AddSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { + return fmt.Errorf("failed to add submodules: %v", err) } // set default branch based on whether it's specified in the newly generated repo or not From 0918e758e06c8be1b113726bc18ed016f3777543 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 31 Dec 2024 22:44:50 +0800 Subject: [PATCH 13/17] fix --- modules/git/submodule.go | 16 ++++++++-------- modules/git/submodule_test.go | 2 +- services/repository/generate.go | 6 +++--- templates/repo/view_list.tmpl | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 37c3b902d27c0..260c226157409 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -9,15 +9,14 @@ import ( "code.gitea.io/gitea/modules/log" ) -// SubModuleCommit submodule name and commit from a repository -type SubModuleCommit struct { +type TemplateSubmoduleCommit struct { Path string Commit string } -// GetSubmoduleCommits returns a list of submodules paths and their commits from a repository +// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository // This function is only for generating new repos based on existing template, the template couldn't be too large. -func GetSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []SubModuleCommit, _ error) { +func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) { stdoutReader, stdoutWriter, err := os.Pipe() if err != nil { return nil, err @@ -37,7 +36,7 @@ func GetSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits return err } if entry.IsSubModule() { - submoduleCommits = append(submoduleCommits, SubModuleCommit{Path: entry.Name(), Commit: entry.ID.String()}) + submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name(), Commit: entry.ID.String()}) } } return scanner.Err() @@ -45,13 +44,14 @@ func GetSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits } err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts) if err != nil { - return nil, fmt.Errorf("GetSubmoduleCommits: error running git ls-tree: %v", err) + return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err) } return submoduleCommits, nil } -// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present. -func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error { +// AddTemplateSubmoduleIndexes Adds the given submodules to the git index. +// It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir. +func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error { for _, submodule := range submodules { cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path) if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil { diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index c62e8c453ee88..8b60d064aa4ae 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -10,7 +10,7 @@ import ( func TestRepository_GetSubmoduleCommits(t *testing.T) { testRepoPath := filepath.Join(testReposDir, "repo4_submodules") - submodules, err := GetSubmoduleCommits(DefaultContext, testRepoPath) + submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath) require.NoError(t, err) assert.EqualValues(t, len(submodules), 2) diff --git a/services/repository/generate.go b/services/repository/generate.go index 090e79552d7fc..ef9a8dc94065b 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -210,9 +210,9 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } // Get active submodules from the template - submodules, err := git.GetSubmoduleCommits(ctx, tmpDir) + submodules, err := git.GetTemplateSubmoduleCommits(ctx, tmpDir) if err != nil { - return fmt.Errorf("GetSubmoduleCommits: %w", err) + return fmt.Errorf("GetTemplateSubmoduleCommits: %w", err) } if err = util.RemoveAll(filepath.Join(tmpDir, ".git")); err != nil { @@ -242,7 +242,7 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return fmt.Errorf("git remote add: %w", err) } - if err = git.AddSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { + if err = git.AddTemplateSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { return fmt.Errorf("failed to add submodules: %v", err) } diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index 01bb70e06ff43..c8e97d2617fd1 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -19,9 +19,9 @@ {{svg "octicon-file-submodule"}} {{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}} {{/* FIXME: the usage of AppUrl seems incorrect, it would be fixed in the future, use AppSubUrl instead */}} {{if $refURL}} - {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} + {{$entry.Name}} @ {{ShortSha $subModuleFile.RefID}} {{else}} - {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} + {{$entry.Name}} @ {{ShortSha $subModuleFile.RefID}} {{end}} {{else}} {{if $entry.IsDir}} From df6ac1a81812ef8a40611526b020be086366ea7d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 31 Dec 2024 22:53:07 +0800 Subject: [PATCH 14/17] fix lint --- modules/git/parse.go | 69 +++++++++++++++++++++++++++++++++++ modules/git/parse_nogogit.go | 60 ------------------------------ modules/git/submodule.go | 3 ++ modules/git/submodule_test.go | 13 ++++--- 4 files changed, 80 insertions(+), 65 deletions(-) create mode 100644 modules/git/parse.go diff --git a/modules/git/parse.go b/modules/git/parse.go new file mode 100644 index 0000000000000..b1ce6a4e91283 --- /dev/null +++ b/modules/git/parse.go @@ -0,0 +1,69 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "bytes" + "fmt" + "strconv" + "strings" +) + +var sepSpace = []byte{' '} + +func parseLsTreeLine(line []byte) (*TreeEntry, error) { + // expect line to be of the form: + // \t + // \t + + var err error + posTab := bytes.IndexByte(line, '\t') + if posTab == -1 { + return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) + } + + entry := new(TreeEntry) + + entryAttrs := line[:posTab] + entryName := line[posTab+1:] + + entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) + _ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type + entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) + if len(entryAttrs) > 0 { + entrySize := entryAttrs // the last field is the space-padded-size + entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) + entry.sized = true + } + + switch string(entryMode) { + case "100644": + entry.entryMode = EntryModeBlob + case "100755": + entry.entryMode = EntryModeExec + case "120000": + entry.entryMode = EntryModeSymlink + case "160000": + entry.entryMode = EntryModeCommit + case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons + entry.entryMode = EntryModeTree + default: + return nil, fmt.Errorf("unknown type: %v", string(entryMode)) + } + + entry.ID, err = NewIDFromString(string(entryObjectID)) + if err != nil { + return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) + } + + if len(entryName) > 0 && entryName[0] == '"' { + entry.name, err = strconv.Unquote(string(entryName)) + if err != nil { + return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) + } + } else { + entry.name = string(entryName) + } + return entry, nil +} diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index 5f8f9d26a9c90..bf74821429bdf 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -10,8 +10,6 @@ import ( "bytes" "fmt" "io" - "strconv" - "strings" "code.gitea.io/gitea/modules/log" ) @@ -21,64 +19,6 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { return parseTreeEntries(data, nil) } -var sepSpace = []byte{' '} - -func parseLsTreeLine(line []byte) (*TreeEntry, error) { - // expect line to be of the form: - // \t - // \t - - var err error - posTab := bytes.IndexByte(line, '\t') - if posTab == -1 { - return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) - } - - entry := new(TreeEntry) - - entryAttrs := line[:posTab] - entryName := line[posTab+1:] - - entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) - _ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type - entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) - if len(entryAttrs) > 0 { - entrySize := entryAttrs // the last field is the space-padded-size - entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) - entry.sized = true - } - - switch string(entryMode) { - case "100644": - entry.entryMode = EntryModeBlob - case "100755": - entry.entryMode = EntryModeExec - case "120000": - entry.entryMode = EntryModeSymlink - case "160000": - entry.entryMode = EntryModeCommit - case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons - entry.entryMode = EntryModeTree - default: - return nil, fmt.Errorf("unknown type: %v", string(entryMode)) - } - - entry.ID, err = NewIDFromString(string(entryObjectID)) - if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) - } - - if len(entryName) > 0 && entryName[0] == '"' { - entry.name, err = strconv.Unquote(string(entryName)) - if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) - } - } else { - entry.name = string(entryName) - } - return entry, nil -} - // parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1) diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 260c226157409..1490e872af981 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -1,3 +1,6 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + package git import ( diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index 8b60d064aa4ae..052aab69bd572 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -1,3 +1,6 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + package git import ( @@ -13,11 +16,11 @@ func TestRepository_GetSubmoduleCommits(t *testing.T) { submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath) require.NoError(t, err) - assert.EqualValues(t, len(submodules), 2) + assert.Len(t, submodules, 2) - assert.EqualValues(t, submodules[0].Path, "<°)))><") - assert.EqualValues(t, submodules[0].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c") + assert.EqualValues(t, "<°)))><", submodules[0].Path) + assert.EqualValues(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit) - assert.EqualValues(t, submodules[1].Path, "libtest") - assert.EqualValues(t, submodules[1].Commit, "1234567890123456789012345678901234567890") + assert.EqualValues(t, "libtest", submodules[1].Path) + assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit) } From 764931acc18c079e3c1084d580042293eb11df3b Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 31 Dec 2024 23:08:49 +0800 Subject: [PATCH 15/17] fix lint --- modules/git/parse.go | 31 ++++++++++++++++++++----------- modules/git/parse_nogogit.go | 12 +++++++++--- modules/git/submodule.go | 4 ++-- modules/git/tree_blob_nogogit.go | 1 - modules/git/tree_entry_nogogit.go | 12 +++--------- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/modules/git/parse.go b/modules/git/parse.go index b1ce6a4e91283..eb26632cc0e5c 100644 --- a/modules/git/parse.go +++ b/modules/git/parse.go @@ -8,11 +8,20 @@ import ( "fmt" "strconv" "strings" + + "code.gitea.io/gitea/modules/optional" ) var sepSpace = []byte{' '} -func parseLsTreeLine(line []byte) (*TreeEntry, error) { +type LsTreeEntry struct { + ID ObjectID + EntryMode EntryMode + Name string + Size optional.Option[int64] +} + +func parseLsTreeLine(line []byte) (*LsTreeEntry, error) { // expect line to be of the form: // \t // \t @@ -23,7 +32,7 @@ func parseLsTreeLine(line []byte) (*TreeEntry, error) { return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) } - entry := new(TreeEntry) + entry := new(LsTreeEntry) entryAttrs := line[:posTab] entryName := line[posTab+1:] @@ -33,21 +42,21 @@ func parseLsTreeLine(line []byte) (*TreeEntry, error) { entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) if len(entryAttrs) > 0 { entrySize := entryAttrs // the last field is the space-padded-size - entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) - entry.sized = true + size, _ := strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) + entry.Size = optional.Some(size) } switch string(entryMode) { case "100644": - entry.entryMode = EntryModeBlob + entry.EntryMode = EntryModeBlob case "100755": - entry.entryMode = EntryModeExec + entry.EntryMode = EntryModeExec case "120000": - entry.entryMode = EntryModeSymlink + entry.EntryMode = EntryModeSymlink case "160000": - entry.entryMode = EntryModeCommit + entry.EntryMode = EntryModeCommit case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons - entry.entryMode = EntryModeTree + entry.EntryMode = EntryModeTree default: return nil, fmt.Errorf("unknown type: %v", string(entryMode)) } @@ -58,12 +67,12 @@ func parseLsTreeLine(line []byte) (*TreeEntry, error) { } if len(entryName) > 0 && entryName[0] == '"' { - entry.name, err = strconv.Unquote(string(entryName)) + entry.Name, err = strconv.Unquote(string(entryName)) if err != nil { return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) } } else { - entry.name = string(entryName) + entry.Name = string(entryName) } return entry, nil } diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index bf74821429bdf..676bb3c76c09f 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -31,12 +31,18 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { } line := data[pos:posEnd] - entry, err := parseLsTreeLine(line) + lsTreeLine, err := parseLsTreeLine(line) if err != nil { return nil, err } - entry.ptree = ptree - + entry := &TreeEntry{ + ptree: ptree, + ID: lsTreeLine.ID, + entryMode: lsTreeLine.EntryMode, + name: lsTreeLine.Name, + size: lsTreeLine.Size.Value(), + sized: lsTreeLine.Size.Has(), + } pos = posEnd + 1 entries = append(entries, entry) } diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 1490e872af981..017b644052b93 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -38,8 +38,8 @@ func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submodul cancel() return err } - if entry.IsSubModule() { - submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name(), Commit: entry.ID.String()}) + if entry.EntryMode == EntryModeCommit { + submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name, Commit: entry.ID.String()}) } } return scanner.Err() diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go index 92d3d107a7425..b7bcf40edd2a9 100644 --- a/modules/git/tree_blob_nogogit.go +++ b/modules/git/tree_blob_nogogit.go @@ -17,7 +17,6 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { ptree: t, ID: t.ID, name: "", - fullName: "", entryMode: EntryModeTree, }, nil } diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index 1c3bcd197a01d..81fb638d56fbe 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -9,23 +9,17 @@ import "code.gitea.io/gitea/modules/log" // TreeEntry the leaf in the git tree type TreeEntry struct { - ID ObjectID - + ID ObjectID ptree *Tree entryMode EntryMode name string - - size int64 - sized bool - fullName string + size int64 + sized bool } // Name returns the name of the entry func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } return te.name } From 47d8be6ffc425fd054c16eac34e88d63626f3360 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 1 Jan 2025 01:33:08 +0800 Subject: [PATCH 16/17] improve tests --- modules/git/submodule_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index 052aab69bd572..fe43b361b4817 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -4,6 +4,8 @@ package git import ( + "context" + "os" "path/filepath" "testing" @@ -11,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestRepository_GetSubmoduleCommits(t *testing.T) { +func TestGetTemplateSubmoduleCommits(t *testing.T) { testRepoPath := filepath.Join(testReposDir, "repo4_submodules") submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath) require.NoError(t, err) @@ -24,3 +26,23 @@ func TestRepository_GetSubmoduleCommits(t *testing.T) { assert.EqualValues(t, "libtest", submodules[1].Path) assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit) } + +func TestAddTemplateSubmoduleIndexes(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + var err error + _, _, err = NewCommand(ctx, "init").RunStdString(&RunOpts{Dir: tmpDir}) + require.NoError(t, err) + _ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0755) + err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}}) + require.NoError(t, err) + _, _, err = NewCommand(ctx, "add", "--all").RunStdString(&RunOpts{Dir: tmpDir}) + require.NoError(t, err) + _, _, err = NewCommand(ctx, "-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(&RunOpts{Dir: tmpDir}) + require.NoError(t, err) + submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir) + require.NoError(t, err) + assert.Len(t, submodules, 1) + assert.EqualValues(t, "new-dir", submodules[0].Path) + assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[0].Commit) +} From e36a690d3355e21edf960059f46b10d8d5899358 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 1 Jan 2025 09:48:20 +0800 Subject: [PATCH 17/17] fix lint --- modules/git/submodule_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index fe43b361b4817..d53946a27d40f 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -33,7 +33,7 @@ func TestAddTemplateSubmoduleIndexes(t *testing.T) { var err error _, _, err = NewCommand(ctx, "init").RunStdString(&RunOpts{Dir: tmpDir}) require.NoError(t, err) - _ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0755) + _ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0o755) err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}}) require.NoError(t, err) _, _, err = NewCommand(ctx, "add", "--all").RunStdString(&RunOpts{Dir: tmpDir})