Skip to content

Commit

Permalink
feat(github): add relation
Browse files Browse the repository at this point in the history 
also add pr fields
Closes #1256
  • Loading branch information
@warren830
warren830 committed Feb 27, 2022
1 parent 6280962 commit 34da896
Show file tree
Hide file tree
Showing 24 changed files with 354 additions and 205 deletions.
1 change: 0 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ require (
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.1.4
github.com/emirpasic/gods v1.12.0 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions models/domainlayer/code/pull_request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,8 @@ type PullRequest struct {
Type string
Component string
MergeCommitSha string `gorm:"type:char(40)"`
HeadRef string
BaseRef string
BaseCommitSha string
HeadCommitSha string
}
8 changes: 8 additions & 0 deletions models/domainlayer/crossdomain/ref_bug_stats.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package crossdomain

type RefBugStats struct {
NewRefName string `gorm:"primaryKey;type:varchar(255)"`
OldRefName string `gorm:"primaryKey;type:varchar(255)"`
IssueCount int
IssueNumber string
}
1 change: 1 addition & 0 deletions models/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func migrateDB() {
&crossdomain.BoardRepo{},
&code.PullRequestIssue{},
&crossdomain.IssueCommit{},
&crossdomain.RefBugStats{},
)
if err != nil {
panic(err)
Expand Down
142 changes: 82 additions & 60 deletions plugins/github/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@ package main // must be main for plugin entry point
import (
"context"
"fmt"
"github.com/merico-dev/lake/errors"
"os"
"strings"

"github.com/merico-dev/lake/errors"

"github.com/merico-dev/lake/config"
"github.com/merico-dev/lake/logger" // A pseudo type for Plugin Interface implementation
lakeModels "github.com/merico-dev/lake/models"
Expand Down Expand Up @@ -83,9 +82,10 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
"collectPullRequests": true,
"collectPullRequestReviews": true,
"collectPullRequestCommits": true,
"collectPullRequestComments": true,
"enrichIssues": true,
"enrichPullRequests": true,
"enrichIssueComments": true,
"enrichPullRequestComments": true,
"enrichGithubPullRequestIssues": true,
"convertRepos": true,
"convertIssues": true,
Expand All @@ -97,6 +97,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
"convertPullRequestLabels": true,
"convertNotes": true,
"convertUsers": true,
"createRefBugStats": true,
}
} else {
for _, task := range op.Tasks {
Expand Down Expand Up @@ -148,7 +149,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
if tasksToRun["collectCommitsStat"] {
progress <- 0.11
fmt.Println("INFO >>> starting commits stat collection")
err = tasks.CollectCommitsStat(op.Owner, op.Repo, repoId, apiClient, rateLimitPerSecondInt, ctx)
err = tasks.CollectCommitsStat(ctx, op.Owner, op.Repo, repoId, apiClient, rateLimitPerSecondInt)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("Could not collect commits: %v", err).Error(),
Expand All @@ -157,7 +158,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["collectIssues"] {
progress <- 0.19
progress <- 0.15
fmt.Println("INFO >>> starting issues collection")
err = tasks.CollectIssues(op.Owner, op.Repo, repoId, apiClient)
if err != nil {
Expand All @@ -168,7 +169,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["collectIssueEvents"] {
progress <- 0.2
progress <- 0.18
fmt.Println("INFO >>> starting Issue Events collection")
err = tasks.CollectIssueEvents(op.Owner, op.Repo, repoId, apiClient)
if err != nil {
Expand All @@ -180,7 +181,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}

if tasksToRun["collectIssueComments"] {
progress <- 0.3
progress <- 0.21
fmt.Println("INFO >>> starting Issue Comments collection")
err = tasks.CollectIssueComments(op.Owner, op.Repo, repoId, apiClient)
if err != nil {
Expand All @@ -191,7 +192,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["collectPullRequests"] {
progress <- 0.4
progress <- 0.28
fmt.Println("INFO >>> collecting PR collection")
err = tasks.CollectPullRequests(op.Owner, op.Repo, repoId, apiClient)
if err != nil {
Expand All @@ -203,9 +204,9 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}

if tasksToRun["collectPullRequestReviews"] {
progress <- 0.5
progress <- 0.4
fmt.Println("INFO >>> collecting PR Reviews collection")
err = tasks.CollectPullRequestReviews(op.Owner, op.Repo, 0, apiClient, rateLimitPerSecondInt, ctx)
err = tasks.CollectPullRequestReviews(ctx, op.Owner, op.Repo, 0, apiClient, rateLimitPerSecondInt)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("Could not collect PR Reviews: %v", err).Error(),
Expand All @@ -215,9 +216,9 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}

if tasksToRun["collectPullRequestCommits"] {
progress <- 0.7
progress <- 0.5
fmt.Println("INFO >>> starting PR Commits collection")
err = tasks.CollectPullRequestCommits(op.Owner, op.Repo, repoId, rateLimitPerSecondInt, apiClient, ctx)
err = tasks.CollectPullRequestCommits(ctx, op.Owner, op.Repo, repoId, rateLimitPerSecondInt, apiClient)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("Could not collect PR Commits: %v", err).Error(),
Expand All @@ -226,20 +227,8 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}

if tasksToRun["collectPullRequestComments"] {
progress <- 0.8
fmt.Println("INFO >>> starting PR Comments collection")
err = tasks.CollectPullRequestComments(op.Owner, op.Repo, apiClient, rateLimitPerSecondInt, ctx)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("Could not collect PR Comments: %v", err).Error(),
SubTaskName: "collectPullRequestComments",
}
}
}

if tasksToRun["enrichIssues"] {
progress <- 0.91
progress <- 0.6
fmt.Println("INFO >>> Enriching Issues")
err = tasks.EnrichGithubIssues(ctx, repoId)
if err != nil {
Expand All @@ -250,7 +239,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["enrichPullRequests"] {
progress <- 0.92
progress <- 0.65
fmt.Println("INFO >>> Enriching PullRequests")
err = tasks.EnrichGithubPullRequests(ctx, repoId)
if err != nil {
Expand All @@ -261,9 +250,9 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["enrichGithubPullRequestIssues"] {
progress <- 0.92
fmt.Println("INFO >>> Enriching PullRequests")
err = tasks.EnrichGithubPullRequestIssue(ctx, repoId)
progress <- 0.68
fmt.Println("INFO >>> Enriching PullRequestIssues")
err = tasks.EnrichGithubPullRequestIssue(ctx, repoId, op.Owner, op.Repo)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("could not enrich PullRequests: %v", err).Error(),
Expand All @@ -272,18 +261,29 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["enrichIssueComments"] {
progress <- 0.92
progress <- 0.72
fmt.Println("INFO >>> Enriching Issue Comments")
err = tasks.EnrichIssueComments(repoId, ctx)
err = tasks.EnrichIssueComments(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("could not enrich PullRequests: %v", err).Error(),
Message: fmt.Errorf("could not enrich Issue Comments: %v", err).Error(),
SubTaskName: "enrichIssueComments",
}
}
}
if tasksToRun["enrichPullRequestComments"] {
progress <- 0.75
fmt.Println("INFO >>> Enriching Pr Comments")
err = tasks.EnrichPullRequestComments(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("could not enrich PullRequest Comments: %v", err).Error(),
SubTaskName: "enrichPullRequestComments",
}
}
}
if tasksToRun["convertRepos"] {
progress <- 0.93
progress <- 0.80
err = tasks.ConvertRepos(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -293,7 +293,7 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertIssues"] {
progress <- 0.94
progress <- 0.85
err = tasks.ConvertIssues(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -303,7 +303,8 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertIssueLabels"] {
progress <- 0.94
progress <- 0.90
fmt.Println("INFO >>> starting convertIssueLabels")
err = tasks.ConvertIssueLabels(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -313,7 +314,8 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertPullRequests"] {
progress <- 0.95
progress <- 0.91
fmt.Println("INFO >>> starting convertPullRequests")
err = tasks.ConvertPullRequests(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -323,14 +325,16 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertPullRequestLabels"] {
progress <- 0.96
progress <- 0.92
fmt.Println("INFO >>> starting convertPullRequestLabels")
err = tasks.ConvertPullRequestLabels(ctx, repoId)
if err != nil {
return err
}
}
if tasksToRun["convertCommits"] {
progress <- 0.96
progress <- 0.93
fmt.Println("INFO >>> starting convertCommits")
err = tasks.ConvertCommits(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -340,7 +344,8 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertPullRequestCommits"] {
progress <- 0.97
progress <- 0.94
fmt.Println("INFO >>> starting convertPullRequestCommits")
err = tasks.PrCommitConvertor(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -350,8 +355,8 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertPullRequestIssues"] {
progress <- 0.97
fmt.Println("INFO >>> starting Issue Comments collection")
progress <- 0.95
fmt.Println("INFO >>> starting convertPullRequestIssues")
err = tasks.ConvertPullRequestIssues(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -361,7 +366,8 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertNotes"] {
progress <- 0.98
progress <- 0.96
fmt.Println("INFO >>> starting convertNotes")
err = tasks.ConvertNotes(ctx, repoId)
if err != nil {
return &errors.SubTaskError{
Expand All @@ -371,7 +377,9 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}
if tasksToRun["convertUsers"] {
progress <- 0.99
progress <- 0.97
fmt.Println("INFO >>> starting convertUsers")

err = tasks.ConvertUsers(ctx, repoId)

if err != nil {
Expand All @@ -382,6 +390,20 @@ func (plugin Github) Execute(options map[string]interface{}, progress chan<- flo
}
}

if tasksToRun["createRefBugStats"] {
progress <- 0.98
fmt.Println("INFO >>> starting createRefBugStats")

err = tasks.CreateRepoBugList(ctx, repoId)

if err != nil {
return &errors.SubTaskError{
Message: fmt.Errorf("could not Create Bug List: %v", err).Error(),
SubTaskName: "createRepoBugList",
}
}
}

progress <- 1
return nil
}
Expand Down Expand Up @@ -412,8 +434,8 @@ var PluginEntry Github //nolint
// standalone mode for debugging
func main() {
args := os.Args[1:]
owner := "tikv"
repo := "pd"
owner := "pingcap"
repo := "tidb"
if len(args) > 0 {
owner = args[0]
}
Expand Down Expand Up @@ -442,29 +464,29 @@ func main() {
"owner": owner,
"repo": repo,
"tasks": []string{
"collectRepo",
////"collectCommits",
////"collectCommitsStat",
//"collectRepo",
//"collectCommitsStat",
//"collectIssues",
//"collectPullRequests",
////"collectIssueEvents",
////"collectIssueComments",
////"collectPullRequestReviews",
//"collectIssueEvents",
"collectIssueComments",
//"collectPullRequestReviews",
//"collectPullRequestCommits",
////"collectPullRequestComments",
//"enrichIssues",
//"enrichPullRequests",
"enrichIssueComments",
"enrichPullRequestComments",
//"enrichGithubPullRequestIssues",
//"convertRepos",
//"convertIssues",
////"convertIssueLabels",
//"convertPullRequests",
////"convertPullRequestLabels",
////"enrichGithubPullRequestIssues",
////"convertCommits",
"convertIssueLabels",
"convertPullRequests",
"convertPullRequestLabels",
"convertPullRequestCommits",
//"convertPullRequestIssues",
//"convertNotes",
//"convertUsers",
"convertPullRequestIssues",
"convertNotes",
"convertUsers",
"createRefBugStats",
},
},
progress,
Expand Down
1 change: 0 additions & 1 deletion plugins/github/models/github_issue_comment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ type GithubIssueComment struct {
GithubId int `gorm:"primaryKey"`
IssueId int `gorm:"index;comment:References the Issue"`
IssueNumber int `gorm:"index;comment:References the Issue Number"`
RepoId int `gorm:"index;comment:References the RepoIdr"`
Body string
AuthorUsername string
GithubCreatedAt time.Time
Expand Down
Loading

0 comments on commit 34da896

Please sign in to comment.