Skip to content

Commit 061cf49

Browse files
030030
committed
fix: [#418] Add more examples regex usage.
1 parent a86e9a1 commit 061cf49

File tree

10 files changed

+73
-9
lines changed

10 files changed

+73
-9
lines changed

‎.github/workflows/docker.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ jobs:
66
runs-on: ubuntu-20.04
77
steps:
88
- uses: actions/[email protected]
9-
- uses: schubergphilis/mcvs-docker-action@v0.2.1
9+
- uses: 030/mcvs-docker-action@18-trivyignore-validation
1010
with:
1111
dockle-accept-key: libcrypto3,libssl3
1212
token: ${{ secrets.GITHUB_TOKEN }}

‎.trivyignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Accept the risk until 2023-01-01
2+
CVE-2019-14697 exp:2024-01-01
3+
CVE-2019-14697 exp:2023-01-01

‎cmd/n3dr/count.go

+5
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ Examples:
2727
2828
# Return the number of artifacts, write them to a /tmp/helloworld.csv and sort it:
2929
n3dr count --csv /tmp/helloworld --sort
30+
31+
# Return the number of specific artifacts using regex, write them to a /tmp/helloworld.csv and sort it:
32+
n3dr count --regex="some" --csv /tmp/helloworld --sort
3033
`,
3134
Run: func(cmd *cobra.Command, args []string) {
3235
n := connection.Nexus3{
@@ -43,6 +46,7 @@ Examples:
4346
FQDN: n3drURL,
4447
HTTPS: &https,
4548
Pass: n3drPass,
49+
Regex: regex,
4650
RepoName: n3drRepo,
4751
SkipErrors: skipErrors,
4852
User: n3drUser,
@@ -62,6 +66,7 @@ Examples:
6266

6367
func init() {
6468
countCmd.Flags().StringVar(&csv, "csv", "", "write to a csvFile")
69+
countCmd.Flags().StringVarP(&regex, "regex", "x", ".*", "only count artifacts that match a regular expression, e.g. 'some/group42'")
6570
countCmd.Flags().BoolVar(&sort, "sort", false, "sort the csvFile")
6671

6772
rootCmd.AddCommand(countCmd)

‎cmd/n3dr/repositoriesV2.go

+1
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ Examples:
8383
FQDN: n3drURL,
8484
HTTPS: &https,
8585
Pass: n3drPass,
86+
Regex: regex,
8687
RepoName: n3drRepo,
8788
SkipErrors: skipErrors,
8889
User: n3drUser,

‎examples/README.md

+2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Examples
22

3+
regex
4+
35
## repositoriesV2
46

57
Use the [basePathPrefix](./repositoriesV2/BASE_PATH_PREFIX.md) subcommand if

‎internal/app/n3dr/artifactsv2/count/artifacts.go

+15
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"os"
77
"path/filepath"
8+
"regexp"
89
"sort"
910
"sync"
1011
"time"
@@ -70,6 +71,8 @@ func (n *Nexus3) write(asset *models.AssetXO, repositoriesTotalArtifacts *int) {
7071
for _, checksum := range checksums {
7172
if value, ok := asset.Checksum[checksum]; ok {
7273
record = append(record, value.(string))
74+
} else {
75+
record = append(record, "unknown")
7376
}
7477
}
7578

@@ -180,6 +183,18 @@ func (n *Nexus3) Artifacts() error {
180183
var wg sync.WaitGroup
181184
fmt.Printf("COUNT\t\tFORMAT\t\tTYPE\t\tNAME\n")
182185
for _, repo := range repos {
186+
187+
// skip count of artifact if it does not match the regex
188+
repoName := repo.Name
189+
r, err := regexp.Compile(n.Regex)
190+
if err != nil {
191+
panic(err)
192+
}
193+
if !r.MatchString(repoName) {
194+
log.Debugf("repo: '%s' skipped as it does not match regex: '%s'", repoName, n.Regex)
195+
continue
196+
}
197+
183198
wg.Add(1)
184199
go func(repo *models.AbstractAPIRepository) {
185200
defer wg.Done()

‎internal/app/n3dr/artifactsv2/download.go

+22-4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"net/http"
77
"os"
88
"path/filepath"
9+
"regexp"
910
"sync"
1011
"time"
1112

@@ -86,7 +87,7 @@ func (n *Nexus3) download(checksum, downloadedFileChecksum string, asset *models
8687
return nil
8788
}
8889

89-
func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) {
90+
func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) error {
9091
shaType, checksum := artifacts.Checksum(asset)
9192

9293
log.WithFields(log.Fields{
@@ -101,6 +102,17 @@ func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) {
101102
}
102103
if !filesToBeSkipped {
103104
file := filepath.Join(n.DownloadDirName, repo, assetPath)
105+
106+
// skip download of artifact if it does not match the regex
107+
r, err := regexp.Compile(n.Regex)
108+
if err != nil {
109+
return err
110+
}
111+
if !r.MatchString(file) {
112+
log.Debugf("file: '%s' skipped as it does not match regex: '%s'", file, n.Regex)
113+
return nil
114+
}
115+
104116
downloadedFileChecksum, err := artifacts.ChecksumLocalFile(file, shaType)
105117
if err != nil {
106118
panic(err)
@@ -110,6 +122,8 @@ func (n *Nexus3) downloadSingleArtifact(asset *models.AssetXO, repo string) {
110122
panic(err)
111123
}
112124
}
125+
126+
return nil
113127
}
114128

115129
func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo string) error {
@@ -129,13 +143,17 @@ func (n *Nexus3) downloadIfChecksumMismatchLocalFile(continuationToken, repo str
129143
for _, item := range resp.GetPayload().Items {
130144
for _, asset := range item.Assets {
131145
if n.WithoutWaitGroups || n.WithoutWaitGroupArtifacts {
132-
n.downloadSingleArtifact(asset, repo)
146+
if err := n.downloadSingleArtifact(asset, repo); err != nil {
147+
return err
148+
}
133149
} else {
134150
wg.Add(1)
135151
go func(asset *models.AssetXO) {
136152
defer wg.Done()
137153

138-
n.downloadSingleArtifact(asset, repo)
154+
if err := n.downloadSingleArtifact(asset, repo); err != nil {
155+
panic(err)
156+
}
139157
}(asset)
140158
}
141159
}
@@ -199,7 +217,7 @@ func (n *Nexus3) repository(repo *models.AbstractAPIRepository) {
199217
func (n *Nexus3) Backup() error {
200218
var wg sync.WaitGroup
201219

202-
cn := connection.Nexus3{BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS, DockerHost: n.DockerHost, DockerPort: n.DockerPort, DockerPortSecure: n.DockerPortSecure}
220+
cn := connection.Nexus3{BasePathPrefix: n.BasePathPrefix, FQDN: n.FQDN, DownloadDirName: n.DownloadDirName, Pass: n.Pass, User: n.User, HTTPS: n.HTTPS, DockerHost: n.DockerHost, DockerPort: n.DockerPort, DockerPortSecure: n.DockerPortSecure, Regex: n.Regex}
203221
a := artifacts.Nexus3{Nexus3: &cn}
204222
repos, err := a.Repos()
205223
if err != nil {

‎internal/app/n3dr/artifactsv2/upload/maven2/snapshot/upload.go

+12-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ import (
1414
)
1515

1616
type Nexus3 struct {
17-
HTTPS, SkipErrors bool
18-
DownloadDirName, FQDN, Pass, RepoFormat, RepoName, User string
17+
HTTPS, SkipErrors bool
18+
DownloadDirName, FQDN, Pass, Regex, RepoFormat, RepoName, User string
1919
}
2020

2121
func (n *Nexus3) statusCode(resp *http.Response) error {
@@ -39,6 +39,16 @@ func (n *Nexus3) statusCode(resp *http.Response) error {
3939
}
4040

4141
func (n *Nexus3) readRetryAndUpload(path string) error {
42+
// skip upload of artifact if it does not match the regex
43+
r, err := regexp.Compile(n.Regex)
44+
if err != nil {
45+
return err
46+
}
47+
if !r.MatchString(path) {
48+
log.Debugf("file: '%s' skipped as it does not match regex: '%s'", path, n.Regex)
49+
return nil
50+
}
51+
4252
log.Debugf("reading path: '%s' and uploading it", path)
4353
f, err := os.Open(filepath.Clean(path))
4454
if err != nil {

‎internal/app/n3dr/artifactsv2/upload/upload.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,16 @@ func (n *Nexus3) ReadLocalDirAndUploadArtifacts(localDiskRepoHome, localDiskRepo
652652
return err
653653
}
654654

655+
// skip upload of artifact if it does not match the regex
656+
r, err := regexp.Compile(n.Regex)
657+
if err != nil {
658+
return err
659+
}
660+
if !r.MatchString(path) {
661+
log.Debugf("file: '%s' skipped as it does not match regex: '%s'", path, n.Regex)
662+
return nil
663+
}
664+
655665
filesToBeSkipped, err := artifacts.FilesToBeSkipped(filepath.Ext(path))
656666
if err != nil {
657667
return err
@@ -704,7 +714,7 @@ func (n *Nexus3) maven2SnapshotsUpload(localDiskRepo string) {
704714
log.Tracef("VersionPolicy: '%s'", vp)
705715

706716
if strings.EqualFold(vp, "snapshot") {
707-
s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User}
717+
s := snapshot.Nexus3{DownloadDirName: n.DownloadDirName, FQDN: n.FQDN, HTTPS: *n.HTTPS, Pass: n.Pass, RepoFormat: "maven2", Regex: n.Regex, RepoName: localDiskRepo, SkipErrors: n.SkipErrors, User: n.User}
708718

709719
if err := s.Upload(); err != nil {
710720
uploaded, errRegex := regexp.MatchString("bad status: 400 Repository does not allow updating assets", err.Error())

‎internal/app/n3dr/connection/connection.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
)
1010

1111
type Nexus3 struct {
12-
AwsBucket, AwsID, AwsRegion, AwsSecret, BasePathPrefix, DockerHost, DownloadDirName, DownloadDirNameZip, Pass, RepoName, User string
12+
AwsBucket, AwsID, AwsRegion, AwsSecret, BasePathPrefix, DockerHost, DownloadDirName, DownloadDirNameZip, Pass, Regex, RepoName, User string
1313
DockerPort int32
1414
DockerPortSecure, SkipErrors, StrictContentTypeValidation, WithoutWaitGroups, WithoutWaitGroupArtifacts, WithoutWaitGroupRepositories, ZIP bool
1515
HTTPS *bool `validate:"required"`

0 commit comments

Comments
 (0)