-
Notifications
You must be signed in to change notification settings - Fork 119
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add best-effort coverage reports per test type (#1915)
Ensure that each executed test type marks some content as covered, so we can have an idea of the files covered by tests, and the test types executed. Before we reported if a test type has been executed by marking a line in a manifest as covered, but this fails if the manifest doesn't have enough lines. At the moment the following files are fully or partially marked as covered if an specific type of test is executed: Pipeline tests: - No changes in this PR, we have coverage based on ES stats. Asset tests: - All files under the kibana directory. Policy tests: - All manifests. - All template files in the executed data stream. Static tests: - Sample events. System tests - All manifests. - Fields files. The rest of non-development files are marked as not covered.
- Loading branch information
Showing
20 changed files
with
445 additions
and
385 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,229 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package testrunner | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "io/fs" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/elastic/elastic-package/internal/files" | ||
| ) | ||
|
|
||
| // GenerateBasePackageCoverageReport generates a coverage report where all files under the root path are | ||
| // marked as not covered. It ignores files under _dev directories. | ||
| func GenerateBasePackageCoverageReport(pkgName, rootPath, format string) (CoverageReport, error) { | ||
| repoPath, err := files.FindRepositoryRootDirectory() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to find repository root directory: %w", err) | ||
| } | ||
|
|
||
| var coverage CoverageReport | ||
| err = filepath.WalkDir(rootPath, func(match string, d fs.DirEntry, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if d.IsDir() { | ||
| if d.Name() == "_dev" { | ||
| return fs.SkipDir | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| fileCoverage, err := generateBaseFileCoverageReport(repoPath, pkgName, match, format, false) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to generate base coverage for \"%s\": %w", match, err) | ||
| } | ||
| if coverage == nil { | ||
| coverage = fileCoverage | ||
| return nil | ||
| } | ||
|
|
||
| err = coverage.Merge(fileCoverage) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot merge coverages: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| }) | ||
| // If the directory is not found, give it as valid, will return an empty coverage. This is also useful for mocked tests. | ||
| if err != nil && !errors.Is(err, os.ErrNotExist) { | ||
| return nil, fmt.Errorf("failed to walk package directory %s: %w", rootPath, err) | ||
| } | ||
| return coverage, nil | ||
| } | ||
|
|
||
| // GenerateBaseFileCoverageReport generates a coverage report for a given file, where all the file is marked as covered or uncovered. | ||
| func GenerateBaseFileCoverageReport(pkgName, path, format string, covered bool) (CoverageReport, error) { | ||
| repoPath, err := files.FindRepositoryRootDirectory() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to find repository root directory: %w", err) | ||
| } | ||
|
|
||
| return generateBaseFileCoverageReport(repoPath, pkgName, path, format, covered) | ||
| } | ||
|
|
||
| // GenerateBaseFileCoverageReport generates a coverage report for all the files matching any of the given patterns. The complete | ||
| // files are marked as fully covered or uncovered depending on the given value. | ||
| func GenerateBaseFileCoverageReportGlob(pkgName string, patterns []string, format string, covered bool) (CoverageReport, error) { | ||
| repoPath, err := files.FindRepositoryRootDirectory() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to find repository root directory: %w", err) | ||
| } | ||
|
|
||
| var coverage CoverageReport | ||
| for _, pattern := range patterns { | ||
| matches, err := filepath.Glob(pattern) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| for _, match := range matches { | ||
| fileCoverage, err := generateBaseFileCoverageReport(repoPath, pkgName, match, format, covered) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to generate base coverage for \"%s\": %w", match, err) | ||
| } | ||
| if coverage == nil { | ||
| coverage = fileCoverage | ||
| continue | ||
| } | ||
|
|
||
| err = coverage.Merge(fileCoverage) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("cannot merge coverages: %w", err) | ||
| } | ||
| } | ||
| } | ||
| return coverage, nil | ||
| } | ||
|
|
||
| func generateBaseFileCoverageReport(repoPath, pkgName, path, format string, covered bool) (CoverageReport, error) { | ||
| switch format { | ||
| case "cobertura": | ||
| return generateBaseCoberturaFileCoverageReport(repoPath, pkgName, path, covered) | ||
| case "generic": | ||
| return generateBaseGenericFileCoverageReport(repoPath, pkgName, path, covered) | ||
| default: | ||
| return nil, fmt.Errorf("unknwon coverage format %s", format) | ||
| } | ||
| } | ||
|
|
||
| func generateBaseCoberturaFileCoverageReport(repoPath, pkgName, path string, covered bool) (*CoberturaCoverage, error) { | ||
| coveragePath, err := filepath.Rel(repoPath, path) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to obtain path inside repository for %s", path) | ||
| } | ||
| ext := filepath.Ext(path) | ||
| class := CoberturaClass{ | ||
| Name: pkgName + "." + strings.TrimSuffix(filepath.Base(path), ext), | ||
| Filename: coveragePath, | ||
| } | ||
| pkg := CoberturaPackage{ | ||
| Name: pkgName, | ||
| Classes: []*CoberturaClass{ | ||
| &class, | ||
| }, | ||
| } | ||
| coverage := CoberturaCoverage{ | ||
| Sources: []*CoberturaSource{ | ||
| { | ||
| Path: path, | ||
| }, | ||
| }, | ||
| Packages: []*CoberturaPackage{ | ||
| &pkg, | ||
| }, | ||
| Timestamp: time.Now().UnixNano(), | ||
| } | ||
|
|
||
| f, err := os.Open(path) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to open file: %v", err) | ||
| } | ||
| defer f.Close() | ||
|
|
||
| hits := int64(0) | ||
| if covered { | ||
| hits = 1 | ||
| } | ||
| lines, err := countReaderLines(f) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to count lines in file: %w", err) | ||
| } | ||
| for i := range lines { | ||
| line := CoberturaLine{ | ||
| Number: i + 1, | ||
| Hits: hits, | ||
| } | ||
| class.Lines = append(class.Lines, &line) | ||
| } | ||
| coverage.LinesValid = int64(lines) | ||
| coverage.LinesCovered = int64(lines) * hits | ||
|
|
||
| return &coverage, nil | ||
| } | ||
|
|
||
| func generateBaseGenericFileCoverageReport(repoPath, _, path string, covered bool) (*GenericCoverage, error) { | ||
| coveragePath, err := filepath.Rel(repoPath, path) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to obtain path inside repository for %s", path) | ||
| } | ||
| file := GenericFile{ | ||
| Path: coveragePath, | ||
| } | ||
| coverage := GenericCoverage{ | ||
| Version: 1, | ||
| Timestamp: time.Now().UnixNano(), | ||
| TestType: fmt.Sprintf("Coverage for %s", coveragePath), | ||
| Files: []*GenericFile{ | ||
| &file, | ||
| }, | ||
| } | ||
|
|
||
| f, err := os.Open(path) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to open file: %v", err) | ||
| } | ||
| defer f.Close() | ||
|
|
||
| lines, err := countReaderLines(f) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to count lines in file: %w", err) | ||
| } | ||
| for i := range lines { | ||
| line := GenericLine{ | ||
| LineNumber: int64(i) + 1, | ||
| Covered: covered, | ||
| } | ||
| file.Lines = append(file.Lines, &line) | ||
| } | ||
|
|
||
| return &coverage, nil | ||
| } | ||
|
|
||
| func countReaderLines(r io.Reader) (int, error) { | ||
| count := 0 | ||
| buffered := bufio.NewReader(r) | ||
| for { | ||
| c, _, err := buffered.ReadRune() | ||
| if errors.Is(err, io.EOF) { | ||
| break | ||
| } | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to read rune: %w", err) | ||
| } | ||
| if c != '\n' { | ||
| continue | ||
| } | ||
| count += 1 | ||
| } | ||
| return count, nil | ||
| } |
Oops, something went wrong.