diff --git a/backend/core/models/domainlayer/code/pull_request.go b/backend/core/models/domainlayer/code/pull_request.go index 0c3be7f8d77..1c63df50fd9 100644 --- a/backend/core/models/domainlayer/code/pull_request.go +++ b/backend/core/models/domainlayer/code/pull_request.go @@ -19,9 +19,10 @@ package code import ( "fmt" - "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "time" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/models/domainlayer" ) @@ -43,6 +44,8 @@ type PullRequest struct { AuthorName string `gorm:"type:varchar(100)"` //User domainUser.User `gorm:"foreignKey:AuthorId"` AuthorId string `gorm:"type:varchar(100)"` + MergedByName string `gorm:"type:varchar(100)"` + MergedById string `gorm:"type:varchar(100)"` ParentPrId string `gorm:"index;type:varchar(100)"` PullRequestKey int CreatedDate time.Time diff --git a/backend/core/models/migrationscripts/20240710_add_merge_by_to_pr.go b/backend/core/models/migrationscripts/20240710_add_merge_by_to_pr.go new file mode 100644 index 00000000000..ab24d9ecb6d --- /dev/null +++ b/backend/core/models/migrationscripts/20240710_add_merge_by_to_pr.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addMergedByToPr)(nil) + +type pr20240710 struct { + MergedByName string `gorm:"type:varchar(100)"` + MergedById string `gorm:"type:varchar(100)"` +} + +func (pr20240710) TableName() string { + return "pull_requests" +} + +type addMergedByToPr struct{} + +func (*addMergedByToPr) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&pr20240710{}); err != nil { + return err + } + return nil +} + +func (*addMergedByToPr) Version() uint64 { + return 20240710142101 +} + +func (*addMergedByToPr) Name() string { + return "add merged by to pull_requests" +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index 6caa89677a8..0ce4680ce3a 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -125,5 +125,6 @@ func All() []plugin.MigrationScript { new(addPullRequestIdIndexToPullRequestComments), new(initIncidentRelatedTables), new(renameProjectIssueMetrics), + new(addMergedByToPr), } } diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 9f2ca7ad528..89a004239cd 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -89,11 +89,7 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err } - cmd, err := g.buildCloneCommand(ctx, localDir, since) - if err != nil { - return err - } - err = g.execCloneCommand(cmd) + err := g.execGitCloneCommand(ctx, localDir, since) if err != nil { return err } @@ -101,13 +97,11 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err if since != nil { // fixes error described on https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1 // It might be casued by the commit which being deepen has mulitple parent(e.g. a merge commit), not sure. - repackCmd := exec.CommandContext(ctx.GetContext(), "git", "-C", localDir, "repack", "-d") - if err := repackCmd.Run(); err != nil { + if err := g.execGitCommand(ctx, "-C", localDir, "repack", "-d"); err != nil { return errors.Default.Wrap(err, "failed to repack the repo") } - deepenCmd := exec.CommandContext(ctx.GetContext(), "git", "-C", localDir, "fetch", "--deepen=1") // deepen would fail on a EMPTY repo, ignore the error - if err := deepenCmd.Run(); err != nil { + if err := g.execGitCommand(ctx, "-C", localDir, "fetch", "--deepen=1"); err != nil { g.logger.Error(err, "failed to deepen the cloned repo") } } @@ -119,9 +113,22 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err return nil } -func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir string, since *time.Time) (*exec.Cmd, errors.Error) { +func (g *GitcliCloner) execGitCloneCommand(ctx plugin.SubTaskContext, localDir string, since *time.Time) errors.Error { taskData := ctx.GetData().(*GitExtractorTaskData) args := []string{"clone", taskData.Options.Url, localDir, "--bare", "--progress"} + if since != nil { + args = append(args, fmt.Sprintf("--shallow-since=%s", since.Format(time.RFC3339))) + } + // support time after and diff sync + // support skipping blobs collection + if *taskData.Options.SkipCommitStat { + args = append(args, "--filter=blob:none") + } + return g.execGitCommand(ctx, args...) +} + +func (g *GitcliCloner) execGitCommand(ctx plugin.SubTaskContext, args ...string) errors.Error { + taskData := ctx.GetData().(*GitExtractorTaskData) env := []string{} // support proxy if taskData.ParsedURL.Scheme == "http" || taskData.ParsedURL.Scheme == "https" { @@ -136,7 +143,7 @@ func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir str if taskData.Options.Proxy != "" { parsedProxyURL, e := url.Parse(taskData.Options.Proxy) if e != nil { - return nil, errors.BadInput.Wrap(e, "failed to parse the proxy URL") + return errors.BadInput.Wrap(e, "failed to parse the proxy URL") } proxyCommand := "corkscrew" sshCmdArgs = append(sshCmdArgs, "-o", fmt.Sprintf(`ProxyCommand="%s %s %s %%h %%p"`, proxyCommand, parsedProxyURL.Hostname(), parsedProxyURL.Port())) @@ -146,16 +153,16 @@ func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir str pkFile, err := os.CreateTemp("", "gitext-pk") if err != nil { g.logger.Error(err, "create temp private key file error") - return nil, errors.Default.New("failed to handle the private key") + return errors.Default.New("failed to handle the private key") } if _, e := pkFile.WriteString(taskData.Options.PrivateKey + "\n"); e != nil { g.logger.Error(err, "write private key file error") - return nil, errors.Default.New("failed to write the private key") + return errors.Default.New("failed to write the private key") } pkFile.Close() if e := os.Chmod(pkFile.Name(), 0600); e != nil { g.logger.Error(err, "chmod private key file error") - return nil, errors.Default.New("failed to modify the private key") + return errors.Default.New("failed to modify the private key") } if taskData.Options.Passphrase != "" { @@ -169,7 +176,7 @@ func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir str if ppout, pperr := pp.CombinedOutput(); pperr != nil { g.logger.Error(pperr, "change private key passphrase error") g.logger.Info(string(ppout)) - return nil, errors.Default.New("failed to decrypt the private key") + return errors.Default.New("failed to decrypt the private key") } } defer os.Remove(pkFile.Name()) @@ -179,22 +186,13 @@ func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir str env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=ssh %s", strings.Join(sshCmdArgs, " "))) } } - // support time after and diff sync - if since != nil { - args = append(args, fmt.Sprintf("--shallow-since=%s", since.Format(time.RFC3339))) - } - // support skipping blobs collection - if *taskData.Options.SkipCommitStat { - args = append(args, "--filter=blob:none") - } - // fmt.Printf("args: %v\n", args) g.logger.Debug("git %v", args) cmd := exec.CommandContext(ctx.GetContext(), "git", args...) cmd.Env = env - return cmd, nil + return g.execCommand(cmd) } -func (g *GitcliCloner) execCloneCommand(cmd *exec.Cmd) errors.Error { +func (g *GitcliCloner) execCommand(cmd *exec.Cmd) errors.Error { stdout, err := cmd.StdoutPipe() if err != nil { g.logger.Error(err, "stdout pipe error") diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go b/backend/plugins/gitextractor/parser/repo_gogit.go index d98ec3d95ed..baaa4e42b17 100644 --- a/backend/plugins/gitextractor/parser/repo_gogit.go +++ b/backend/plugins/gitextractor/parser/repo_gogit.go @@ -301,6 +301,20 @@ func (r *GogitRepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) (e default: } commitSha := commit.Hash.String() + + if commit.NumParents() != 0 { + _, err := commit.Parents().Next() + if err != nil { + if err == plumbing.ErrObjectNotFound { + // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. + // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting + // the correct addition/deletion data in the database with an absurdly large addition number. + r.logger.Info("skip commit %s because it has no parent commit", commitSha) + return nil + } + return err + } + } codeCommit := &code.Commit{ Sha: commitSha, Message: commit.Message, diff --git a/backend/plugins/gitextractor/parser/repo_libgit2.go b/backend/plugins/gitextractor/parser/repo_libgit2.go index f4371204ff4..78451feb017 100644 --- a/backend/plugins/gitextractor/parser/repo_libgit2.go +++ b/backend/plugins/gitextractor/parser/repo_libgit2.go @@ -279,6 +279,17 @@ func (r *Libgit2RepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) if commit == nil { return nil } + var parent *git.Commit + if commit.ParentCount() > 0 { + parent = commit.Parent(0) + // Skip calculating commit statistics when there are parent commits, but the first one cannot be fetched from the ODB. + // This usually happens during a shallow clone for incremental collection. Otherwise, we might end up overwriting + // the correct addition/deletion data in the database with an absurdly large addition number. + if parent == nil { + r.logger.Info("skip commit %s because it has no parent commit", commit.Id().String()) + return nil + } + } commitSha := commit.Id().String() r.logger.Debug("process commit: %s", commitSha) c := &code.Commit{ @@ -303,10 +314,6 @@ func (r *Libgit2RepoCollector) CollectCommits(subtaskCtx plugin.SubTaskContext) if err != nil { return err } - var parent *git.Commit - if commit.ParentCount() > 0 { - parent = commit.Parent(0) - } if !*taskOpts.SkipCommitStat { var stats *git.DiffStats diff --git a/backend/plugins/github/e2e/pr_review_test.go b/backend/plugins/github/e2e/pr_review_test.go index 10c364e0619..043b8a93678 100644 --- a/backend/plugins/github/e2e/pr_review_test.go +++ b/backend/plugins/github/e2e/pr_review_test.go @@ -71,9 +71,10 @@ func TestPrReviewDataFlow(t *testing.T) { "./snapshot_tables/_tool_github_reviewers.csv", []string{ "connection_id", - "github_id", + "reviewer_id", "pull_request_id", - "login", + "username", + "name", "_raw_data_params", "_raw_data_table", "_raw_data_id", diff --git a/backend/plugins/github/e2e/snapshot_tables/_tool_github_reviewers.csv b/backend/plugins/github/e2e/snapshot_tables/_tool_github_reviewers.csv index 6f34bb15afc..359da5a2665 100644 --- a/backend/plugins/github/e2e/snapshot_tables/_tool_github_reviewers.csv +++ b/backend/plugins/github/e2e/snapshot_tables/_tool_github_reviewers.csv @@ -1,7 +1,7 @@ -connection_id,github_id,pull_request_id,login,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,2813260,325179595,KevinBaiSg,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,39, -1,7496278,308859272,panjf2000,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,21, -1,7496278,316337433,panjf2000,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,22, -1,7496278,325179595,panjf2000,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,38, -1,8923413,308859272,choleraehyq,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,20, -1,8923413,316337433,choleraehyq,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,24, +connection_id,reviewer_id,pull_request_id,username,name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,2813260,325179595,KevinBaiSg,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,39, +1,7496278,308859272,panjf2000,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,21, +1,7496278,316337433,panjf2000,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,22, +1,7496278,325179595,panjf2000,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,38, +1,8923413,308859272,choleraehyq,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,20, +1,8923413,316337433,choleraehyq,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_pull_request_reviews,24, diff --git a/backend/plugins/github/models/migrationscripts/20240710_add_merge_by_to_pr.go b/backend/plugins/github/models/migrationscripts/20240710_add_merge_by_to_pr.go new file mode 100644 index 00000000000..6c65eadb018 --- /dev/null +++ b/backend/plugins/github/models/migrationscripts/20240710_add_merge_by_to_pr.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addMergedByToPr)(nil) + +type pr20240710 struct { + MergedByName string `gorm:"type:varchar(100)"` + MergedById int +} + +func (pr20240710) TableName() string { + return "_tool_github_pull_requests" +} + +type addMergedByToPr struct{} + +func (*addMergedByToPr) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&pr20240710{}); err != nil { + return err + } + return nil +} + +func (*addMergedByToPr) Version() uint64 { + return 20240710142100 +} + +func (*addMergedByToPr) Name() string { + return "add merged by to _tool_github_pull_requests" +} diff --git a/backend/plugins/github/models/migrationscripts/20240711_restruct_github_reviewers.go b/backend/plugins/github/models/migrationscripts/20240711_restruct_github_reviewers.go new file mode 100644 index 00000000000..af5a95a34ed --- /dev/null +++ b/backend/plugins/github/models/migrationscripts/20240711_restruct_github_reviewers.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + coreArchived "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/github/models/migrationscripts/archived" +) + +var _ plugin.MigrationScript = (*restructReviewer)(nil) + +type reviewer20240711 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ReviewerId int `gorm:"primaryKey"` + PullRequestId int `gorm:"primaryKey"` + Name string `gorm:"type:varchar(255)"` + Username string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + WebUrl string `gorm:"type:varchar(255)"` + coreArchived.NoPKModel +} + +func (reviewer20240711) TableName() string { + return "_tool_github_reviewers" +} + +type restructReviewer struct{} + +func (*restructReviewer) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.DropTables(&archived.GithubReviewer{}); err != nil { + return err + } + if err := db.AutoMigrate(&reviewer20240711{}); err != nil { + return err + } + return nil +} + +func (*restructReviewer) Version() uint64 { + return 20240710142104 +} + +func (*restructReviewer) Name() string { + return "restruct reviewer table" +} diff --git a/backend/plugins/github/models/migrationscripts/register.go b/backend/plugins/github/models/migrationscripts/register.go index 496f54544fd..4a4b9506975 100644 --- a/backend/plugins/github/models/migrationscripts/register.go +++ b/backend/plugins/github/models/migrationscripts/register.go @@ -50,5 +50,7 @@ func All() []plugin.MigrationScript { new(addWorkflowDisplayTitle), new(addReleaseTable), new(addReleaseCommitSha), + new(addMergedByToPr), + new(restructReviewer), } } diff --git a/backend/plugins/github/models/pr.go b/backend/plugins/github/models/pr.go index bd2e15d0d19..b8e8fa78a89 100644 --- a/backend/plugins/github/models/pr.go +++ b/backend/plugins/github/models/pr.go @@ -18,8 +18,9 @@ limitations under the License. package models import ( - "github.com/apache/incubator-devlake/core/models/common" "time" + + "github.com/apache/incubator-devlake/core/models/common" ) type GithubPullRequest struct { @@ -52,6 +53,8 @@ type GithubPullRequest struct { Url string `gorm:"type:varchar(255)"` AuthorName string `gorm:"type:varchar(100)"` AuthorId int + MergedByName string `gorm:"type:varchar(100)"` + MergedById int common.NoPKModel } diff --git a/backend/plugins/github/models/reviewer.go b/backend/plugins/github/models/reviewer.go index c8af4e181ee..7d82a88e649 100644 --- a/backend/plugins/github/models/reviewer.go +++ b/backend/plugins/github/models/reviewer.go @@ -23,9 +23,13 @@ import ( type GithubReviewer struct { ConnectionId uint64 `gorm:"primaryKey"` - GithubId int `gorm:"primaryKey"` - Login string `gorm:"type:varchar(255)"` + ReviewerId int `gorm:"primaryKey"` PullRequestId int `gorm:"primaryKey"` + Name string `gorm:"type:varchar(255)"` + Username string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + WebUrl string `gorm:"type:varchar(255)"` common.NoPKModel } diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go index 6f725012740..da935c300cb 100644 --- a/backend/plugins/github/tasks/pr_convertor.go +++ b/backend/plugins/github/tasks/pr_convertor.go @@ -102,6 +102,8 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { BaseCommitSha: pr.BaseCommitSha, HeadRef: pr.HeadRef, HeadCommitSha: pr.HeadCommitSha, + MergedByName: pr.MergedByName, + MergedById: accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById), } if pr.State == "open" || pr.State == "OPEN" { domainPr.Status = code.OPEN diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go index c14a179e8f2..2fba9a9551d 100644 --- a/backend/plugins/github/tasks/pr_review_extractor.go +++ b/backend/plugins/github/tasks/pr_review_extractor.go @@ -108,8 +108,8 @@ func ExtractApiPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error { } if apiPullRequestReview.User != nil { - githubReviewer.GithubId = apiPullRequestReview.User.Id - githubReviewer.Login = apiPullRequestReview.User.Login + githubReviewer.ReviewerId = apiPullRequestReview.User.Id + githubReviewer.Username = apiPullRequestReview.User.Login githubPrReview.AuthorUserId = apiPullRequestReview.User.Id githubPrReview.AuthorUsername = apiPullRequestReview.User.Login diff --git a/backend/plugins/github/tasks/review_convertor.go b/backend/plugins/github/tasks/review_convertor.go new file mode 100644 index 00000000000..d4d10104256 --- /dev/null +++ b/backend/plugins/github/tasks/review_convertor.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" +) + +func init() { + RegisterSubtaskMeta(&ConvertReviewsMeta) +} + +var ConvertReviewsMeta = plugin.SubTaskMeta{ + Name: "Convert Reviews", + EntryPoint: ConvertReviews, + EnabledByDefault: true, + Description: "ConvertReviews data from Github api", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, + DependencyTables: []string{ + models.GithubReviewer{}.TableName(), // cursor and id generator + models.GithubPullRequest{}.TableName(), // cursor and id generator + models.GithubAccount{}.TableName(), // id generator + }, + ProductTables: []string{code.PullRequestReviewer{}.TableName()}, +} + +func ConvertReviews(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Options.GithubId + + cursor, err := db.Cursor( + dal.From(&models.GithubReviewer{}), + dal.Join("left join _tool_github_pull_requests "+ + "on _tool_github_pull_requests.github_id = _tool_github_reviewers.pull_request_id"), + dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + InputRowType: reflect.TypeOf(models.GithubReviewer{}), + Input: cursor, + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + Table: RAW_PR_REVIEW_TABLE, + }, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + githubReview := inputRow.(*models.GithubReviewer) + domainReview := &code.PullRequestReviewer{ + PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubReview.PullRequestId), + ReviewerId: accountIdGen.Generate(data.Options.ConnectionId, githubReview.ReviewerId), + Name: githubReview.Name, + UserName: githubReview.Username, + } + return []interface{}{ + domainReview, + }, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go index 8045cd14539..66433699c3c 100644 --- a/backend/plugins/github_graphql/impl/impl.go +++ b/backend/plugins/github_graphql/impl/impl.go @@ -125,6 +125,7 @@ func (p GithubGraphql) SubTaskMetas() []plugin.SubTaskMeta { githubTasks.ConvertIssueAssigneeMeta, githubTasks.ConvertIssueCommentsMeta, githubTasks.ConvertPullRequestCommentsMeta, + githubTasks.ConvertReviewsMeta, githubTasks.ConvertMilestonesMeta, githubTasks.ConvertAccountsMeta, diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go index 903ce6c5bda..45fdbf5d0a9 100644 --- a/backend/plugins/github_graphql/tasks/pr_collector.go +++ b/backend/plugins/github_graphql/tasks/pr_collector.go @@ -85,6 +85,31 @@ type GraphqlQueryPr struct { TotalCount graphql.Int Nodes []GraphqlQueryReview `graphql:"nodes"` } `graphql:"reviews(first: 100)"` + MergedBy *GraphqlInlineAccountQuery + ReviewRequests struct { + Nodes []ReviewRequestNode `graphql:"nodes"` + } `graphql:"reviewRequests(first: 10)"` +} + +type ReviewRequestNode struct { + RequestedReviewer RequestedReviewer `graphql:"requestedReviewer"` +} + +type RequestedReviewer struct { + User User `graphql:"... on User"` + Team Team `graphql:"... on Team"` +} + +type User struct { + Id int `graphql:"databaseId"` + Login string `graphql:"login"` + Name string `graphql:"name"` +} + +type Team struct { + Id int `graphql:"databaseId"` + Name string `graphql:"name"` + Slug string `graphql:"slug"` } type GraphqlQueryReview struct { diff --git a/backend/plugins/github_graphql/tasks/pr_extractor.go b/backend/plugins/github_graphql/tasks/pr_extractor.go index c82975fc827..cc26d0cc94c 100644 --- a/backend/plugins/github_graphql/tasks/pr_extractor.go +++ b/backend/plugins/github_graphql/tasks/pr_extractor.go @@ -121,6 +121,15 @@ func ExtractPrs(taskCtx plugin.SubTaskContext) errors.Error { results = append(results, githubPrReview) } } + for _, apiReviewRequests := range rawL.ReviewRequests.Nodes { + githubReviewRequests := &models.GithubReviewer{ + ConnectionId: data.Options.ConnectionId, + PullRequestId: githubPr.GithubId, + ReviewerId: apiReviewRequests.RequestedReviewer.User.Id, + Username: apiReviewRequests.RequestedReviewer.User.Login, + } + results = append(results, githubReviewRequests) + } for _, apiPullRequestCommit := range rawL.Commits.Nodes { githubCommit, err := convertPullRequestCommit(apiPullRequestCommit) @@ -137,9 +146,6 @@ func ExtractPrs(taskCtx plugin.SubTaskContext) errors.Error { CommitAuthorEmail: githubCommit.AuthorEmail, CommitAuthoredDate: githubCommit.AuthoredDate, } - if err != nil { - return nil, err - } results = append(results, githubPullRequestCommit) extractGraphqlPreAccount(&results, apiPullRequestCommit.Commit.Author.User, data.Options.GithubId, data.Options.ConnectionId) } @@ -174,6 +180,10 @@ func convertGithubPullRequest(pull GraphqlQueryPr, connId uint64, repoId int) (* HeadRef: pull.HeadRefName, HeadCommitSha: pull.HeadRefOid, } + if pull.MergedBy != nil { + githubPull.MergedByName = pull.MergedBy.Login + githubPull.MergedById = pull.MergedBy.Id + } if pull.MergeCommit != nil { githubPull.MergeCommitSha = pull.MergeCommit.Oid } diff --git a/grafana/dashboards/EngineeringOverview.json b/grafana/dashboards/EngineeringOverview.json index 3ebdfe55be1..0fc94d8df60 100644 --- a/grafana/dashboards/EngineeringOverview.json +++ b/grafana/dashboards/EngineeringOverview.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 16, + "id": 40, "links": [], "liveNow": false, "panels": [ @@ -34,7 +34,6 @@ "y": 0 }, "id": 32, - "links": [], "options": { "code": { "language": "plaintext", @@ -44,7 +43,7 @@ "content": "- Use Cases: This dashboard is to overview the Git and project management metrics.\n- Data Source Required: Jira + GitHub, or Jira + GitLab.", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": { @@ -93,7 +92,6 @@ "y": 3 }, "id": 8, - "links": [], "options": { "colorMode": "value", "graphMode": "area", @@ -106,10 +104,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", @@ -119,7 +119,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n count(distinct i.id)\nfrom\n issues i\n\tjoin board_issues bi on i.id = bi.issue_id\n\tjoin boards b on bi.board_id = b.id\n\tjoin project_mapping pm on b.id = pm.row_id\nwhere\n pm.project_name in (${project}) and\n i.priority in (${priority}) and\n i.type = 'BUG' and\n date(i.created_date) between STR_TO_DATE('$month','%Y-%m-%d') and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY;", + "rawSql": "select\n count(distinct i.id)\nfrom\n issues i\n\tjoin board_issues bi on i.id = bi.issue_id\n\tjoin boards b on bi.board_id = b.id\n\tjoin project_mapping pm on b.id = pm.row_id\nwhere\n pm.project_name in (${project}) and\n i.priority in (${priority}) and\n i.type = 'BUG' and\n date(i.created_date) between STR_TO_DATE('$month','%Y-%m-%d') and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH;", "refId": "A", "select": [ [ @@ -160,7 +160,7 @@ ] } ], - "title": "Critical Defects Identified", + "title": "Critical Defects Identified [in $month]", "type": "stat" }, { @@ -171,6 +171,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -215,14 +216,15 @@ "y": 3 }, "id": 22, - "links": [], "options": { "barRadius": 0, "barWidth": 0.46, "fullHighlight": false, "groupWidth": 0.7, "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -234,6 +236,7 @@ "valueSize": 12 }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -318,7 +321,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 6, "x": 0, "y": 11 @@ -343,18 +346,21 @@ "fields": "/^value$/", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", + "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\n\tjoin project_mapping pm on b.id = pm.row_id\r\nwhere \r\n pm.project_name in (${project}) and\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)", + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\n\tjoin project_mapping pm on b.id = pm.row_id\r\nwhere \r\n pm.project_name in (${project}) and\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and i.resolution_date between STR_TO_DATE('$month','%Y-%m-%d') and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH;", "refId": "A", "select": [ [ @@ -366,6 +372,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "table": "ca_analysis", "timeColumn": "create_time", "timeColumnType": "timestamp", @@ -378,7 +401,7 @@ ] } ], - "title": "Mean Issue Lead Time in Days [Issues Resolved in Select Time Range]", + "title": "Mean Issue Lead Time in Days [Issues Resolved in $month]", "type": "stat" }, { @@ -390,6 +413,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Lead Time(days)", @@ -429,7 +453,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 18, "x": 6, "y": 11 @@ -449,7 +473,9 @@ "fullHighlight": false, "groupWidth": 0.7, "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -461,6 +487,7 @@ "valueSize": 12 }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -534,10 +561,9 @@ "h": 8, "w": 6, "x": 0, - "y": 17 + "y": 19 }, "id": 14, - "links": [], "options": { "colorMode": "value", "graphMode": "area", @@ -550,19 +576,22 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", + "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n count(distinct author_name)\nfrom\n commits c\n join repo_commits rc on c.sha = rc.commit_sha\n join project_mapping pm on rc.repo_id = pm.row_id\nwhere\n date(authored_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY and\n pm.project_name in (${project});", + "rawSql": "select\n count(distinct author_name)\nfrom\n commits c\n join repo_commits rc on c.sha = rc.commit_sha\n join project_mapping pm on rc.repo_id = pm.row_id and pm.table = 'repos' \nwhere\n date(authored_date) between STR_TO_DATE('$month','%Y-%m-%d') and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH\n and pm.project_name in (${project})", "refId": "A", "select": [ [ @@ -574,6 +603,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "table": "_devlake_blueprints", "timeColumn": "created_at", "timeColumnType": "timestamp", @@ -586,7 +632,7 @@ ] } ], - "title": "Number of Developers", + "title": "Number of Developers [in $month]", "type": "stat" }, { @@ -597,6 +643,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -638,17 +685,18 @@ "h": 8, "w": 18, "x": 6, - "y": 17 + "y": 19 }, "id": 24, - "links": [], "options": { "barRadius": 0, "barWidth": 0.5, "fullHighlight": false, "groupWidth": 0.7, "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -660,6 +708,7 @@ "valueSize": 12 }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -670,12 +719,13 @@ "targets": [ { "datasource": "mysql", + "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "with _developers as(\n select\n DATE_ADD(date(c.authored_date), INTERVAL -DAY(date(c.authored_date))+1 DAY) as time,\n count(distinct author_name) as developer_count\n from\n commits c\n join repo_commits rc on c.sha = rc.commit_sha\n join project_mapping pm on rc.repo_id = pm.row_id\n where\n $__timeFilter(c.authored_date)\n and c.authored_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\n and pm.project_name in (${project})\n group by time\n)\n\nselect\n date_format(time,'%M %Y') as month,\n developer_count\nfrom _developers\norder by time asc", + "rawSql": "with _developers as(\n select\n DATE_ADD(date(c.authored_date), INTERVAL -DAY(date(c.authored_date))+1 DAY) as time,\n count(distinct author_name) as developer_count\n from\n commits c\n join repo_commits rc on c.sha = rc.commit_sha\n join project_mapping pm on rc.repo_id = pm.row_id and pm.table = 'repos' \n where\n $__timeFilter(c.authored_date)\n and c.authored_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\n and pm.project_name in (${project})\n group by time\n)\n\nselect\n date_format(time,'%M %Y') as month,\n developer_count\nfrom _developers\norder by time asc", "refId": "A", "select": [ [ @@ -687,6 +737,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "table": "_devlake_blueprints", "timeColumn": "created_at", "timeColumnType": "timestamp", @@ -736,7 +803,7 @@ "h": 8, "w": 6, "x": 0, - "y": 25 + "y": 27 }, "id": 6, "links": [ @@ -758,10 +825,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", @@ -771,7 +840,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "with _num_issues_with_sprint_updated as (\n select\n NULLIF(COUNT(DISTINCT i.id), 0) AS num_issues_with_sprint_updated\n from\n issues i\n join board_issues bi on i.id = bi.issue_id\n\t join boards b on bi.board_id = b.id\n\t join project_mapping pm on b.id = pm.row_id\n join issue_changelogs c on i.id = c.issue_id\n where\n pm.project_name in (${project}) and\n c.field_name = 'Sprint' and\n c.original_from_value != '' and\n c.original_to_value != '' and\n date(i.created_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY\n),\n\n_total_num_issues as (\n select\n NULLIF(COUNT(distinct i.id), 0) as total_num_issues\n from\n issues i\n join board_issues bi on i.id = bi.issue_id\n\t join boards b on bi.board_id = b.id\n\t join project_mapping pm on b.id = pm.row_id\n where\n pm.project_name in (${project}) and\n date(i.created_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY\n)\n\nselect\n now() as time,\n 100 - 100 * (select 1.0 * num_issues_with_sprint_updated from _num_issues_with_sprint_updated) / (select total_num_issues from _total_num_issues) as ratio;", + "rawSql": "with _num_issues_with_sprint_updated as (\n select\n NULLIF(COUNT(DISTINCT i.id), 0) AS num_issues_with_sprint_updated\n from\n issues i\n join board_issues bi on i.id = bi.issue_id\n\t join boards b on bi.board_id = b.id\n\t join project_mapping pm on b.id = pm.row_id\n join issue_changelogs c on i.id = c.issue_id\n where\n pm.project_name in (${project}) and\n c.field_name = 'Sprint' and\n c.original_from_value != '' and\n c.original_to_value != '' and\n date(i.created_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH\n),\n\n_total_num_issues as (\n select\n NULLIF(COUNT(distinct i.id), 0) as total_num_issues\n from\n issues i\n join board_issues bi on i.id = bi.issue_id\n\t join boards b on bi.board_id = b.id\n\t join project_mapping pm on b.id = pm.row_id\n where\n pm.project_name in (${project}) and\n date(i.created_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH\n)\n\nselect\n now() as time,\n 100 - 100 * (select 1.0 * num_issues_with_sprint_updated from _num_issues_with_sprint_updated) / (select total_num_issues from _total_num_issues) as ratio;", "refId": "A", "select": [ [ @@ -812,7 +881,7 @@ ] } ], - "title": "On-time Delivery", + "title": "On-time Delivery [in $month]", "type": "stat" }, { @@ -824,6 +893,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -837,6 +907,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 10, @@ -875,7 +946,7 @@ "h": 8, "w": 18, "x": 6, - "y": 25 + "y": 27 }, "id": 25, "links": [ @@ -887,12 +958,15 @@ ], "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -964,8 +1038,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "green", @@ -980,7 +1053,7 @@ "h": 8, "w": 6, "x": 0, - "y": 33 + "y": 35 }, "id": 4, "links": [ @@ -1002,10 +1075,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", @@ -1015,7 +1090,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n count(distinct pr.id)\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pr.merged_date is not null \n and date(pr.merged_date) between\n STR_TO_DATE('$month','%Y-%m-%d')\n and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY \n and pm.project_name in (${project});", + "rawSql": "select\n count(distinct pr.id)\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pr.merged_date is not null \n and date(pr.merged_date) between\n STR_TO_DATE('$month','%Y-%m-%d')\n and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH\n and pm.project_name in (${project});", "refId": "A", "select": [ [ @@ -1056,7 +1131,7 @@ ] } ], - "title": "PRs merged", + "title": "PRs merged [in $month]", "type": "stat" }, { @@ -1068,6 +1143,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1093,8 +1169,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1109,7 +1184,7 @@ "h": 8, "w": 18, "x": 6, - "y": 33 + "y": 35 }, "id": 26, "links": [ @@ -1125,7 +1200,9 @@ "fullHighlight": false, "groupWidth": 0.7, "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true @@ -1137,6 +1214,7 @@ "valueSize": 12 }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" }, @@ -1209,8 +1287,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "orange", @@ -1230,7 +1307,7 @@ "h": 8, "w": 6, "x": 0, - "y": 41 + "y": 43 }, "id": 16, "links": [ @@ -1252,10 +1329,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", @@ -1265,7 +1344,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select \n 100*count(distinct case when pr.id in (select pull_request_id from pull_request_issues) then pr.id else null end)/count(distinct pr.id) as unlinked_pr_rate\nfrom pull_requests pr\njoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere pm.project_name in (${project})", + "rawSql": "select \n 100*count(distinct case when pr.id in (select pull_request_id from pull_request_issues) then pr.id else null end)/count(distinct pr.id) as unlinked_pr_rate\nfrom pull_requests pr\njoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere \n pm.project_name in (${project})\n and date(pr.created_date) between STR_TO_DATE('$month','%Y-%m-%d') and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH", "refId": "A", "select": [ [ @@ -1306,7 +1385,7 @@ ] } ], - "title": "Unlinked PRs %", + "title": "Unlinked PRs % [in $month]", "type": "stat" }, { @@ -1317,6 +1396,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1330,6 +1410,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 10, @@ -1351,8 +1432,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1368,7 +1448,7 @@ "h": 8, "w": 18, "x": 6, - "y": 41 + "y": 43 }, "id": 28, "links": [ @@ -1380,12 +1460,15 @@ ], "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1456,8 +1539,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "green", @@ -1473,10 +1555,9 @@ "h": 8, "w": 6, "x": 0, - "y": 49 + "y": 51 }, "id": 12, - "links": [], "options": { "colorMode": "value", "graphMode": "area", @@ -1489,10 +1570,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", @@ -1502,7 +1585,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "with _commits_groupby_name_and_date as (\n select\n author_name,\n date(authored_date) as _day,\n count(distinct c.sha)\n from\n commits c\n join repo_commits rc on c.sha = rc.commit_sha\n join project_mapping pm on rc.repo_id = pm.row_id\n where\n pm.project_name in (${project}) and\n WEEKDAY(authored_date) between 0 and 4 and\n date(authored_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY\n group by\n author_name, date(authored_date)\n)\n\nselect 100 * count(*) / (count(distinct author_name) * count(distinct _day))\nfrom _commits_groupby_name_and_date;", + "rawSql": "with _commits_groupby_name_and_date as (\n select\n author_name,\n date(authored_date) as _day,\n count(distinct c.sha)\n from\n commits c\n join repo_commits rc on c.sha = rc.commit_sha\n join project_mapping pm on rc.repo_id = pm.row_id\n where\n pm.project_name in (${project}) and\n WEEKDAY(authored_date) between 0 and 4 and\n date(authored_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH\n group by\n author_name, date(authored_date)\n)\n\nselect 100 * count(*) / (count(distinct author_name) * count(distinct _day))\nfrom _commits_groupby_name_and_date;", "refId": "A", "select": [ [ @@ -1543,7 +1626,7 @@ ] } ], - "title": "Coding Days %", + "title": "Coding Days % [in $month]", "type": "stat" }, { @@ -1554,6 +1637,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1567,6 +1651,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 10, @@ -1588,8 +1673,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1605,18 +1689,20 @@ "h": 8, "w": 18, "x": 6, - "y": 49 + "y": 51 }, "id": 29, - "links": [], "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1689,8 +1775,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1706,7 +1791,7 @@ "h": 8, "w": 6, "x": 0, - "y": 57 + "y": 59 }, "id": 2, "links": [ @@ -1728,19 +1813,22 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": "mysql", + "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n AVG(TIMESTAMPDIFF(MINUTE, pr.created_date, pr.merged_date) / 1440)\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pm.project_name in (${project}) and\n pr.merged_date is not null\n and date(pr.created_date) between\n STR_TO_DATE('$month','%Y-%m-%d') \n and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY", + "rawSql": "select\n AVG(TIMESTAMPDIFF(MINUTE, pr.created_date, pr.merged_date) / 1440)\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pm.project_name in (${project}) and\n pr.merged_date is not null\n and date(pr.created_date) between\n STR_TO_DATE('$month','%Y-%m-%d') \n and STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH", "refId": "A", "select": [ [ @@ -1752,6 +1840,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "table": "_raw_github_api_repositories", "timeColumn": "created_at", "timeColumnType": "timestamp", @@ -1764,7 +1869,7 @@ ] } ], - "title": "Mean Time to Merge (From PR creation to merge)", + "title": "Mean Time to Merge [in $month]", "type": "stat" }, { @@ -1776,6 +1881,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1789,6 +1895,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 10, @@ -1811,8 +1918,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1828,7 +1934,7 @@ "h": 8, "w": 18, "x": 6, - "y": 57 + "y": 59 }, "id": 30, "links": [ @@ -1840,12 +1946,15 @@ ], "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1912,41 +2021,13 @@ "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 68, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "d" + "mappings": [] }, "overrides": [] }, @@ -1954,43 +2035,47 @@ "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 67 }, - "id": 20, - "links": [], + "id": 21, "options": { - "barRadius": 0, - "barWidth": 0.27, - "fullHighlight": false, - "groupWidth": 0.7, + "displayLabels": [ + "name", + "percent" + ], "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "text": { - "valueSize": 12 + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + } }, "targets": [ { "datasource": "mysql", + "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n i.priority as 'Priority',\n AVG(TIMESTAMPDIFF(MINUTE, i.created_date, NOW()) / 1440) as 'Average Age'\nfrom\n issues i\n join board_issues bi on i.id = bi.issue_id\n\tjoin boards b on bi.board_id = b.id\n\tjoin project_mapping pm on b.id = pm.row_id\nwhere\n pm.project_name in (${project}) and\n i.status = 'TODO'\n and i.type = 'BUG'\n and i.priority in (${priority})\ngroup by\n i.priority", + "rawSql": "select\n count(distinct case when i.type = 'BUG' then i.id else null end) as 'Bug',\n count(distinct case when i.type != 'BUG' and epic_key != '' then i.id else null end) as 'Strategic',\n count(distinct case when i.type != 'BUG' and epic_key = '' then i.id else null end) as 'Non-Strategic'\nfrom\n issues i\n join board_issues bi on i.id = bi.issue_id\n\tjoin boards b on bi.board_id = b.id\n\tjoin project_mapping pm on b.id = pm.row_id\nwhere\n pm.project_name in (${project}) and\n i.resolution_date is not null and\n date(resolution_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH", "refId": "A", "select": [ [ @@ -2002,6 +2087,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "table": "_devlake_blueprints", "timeColumn": "created_at", "timeColumnType": "timestamp", @@ -2014,8 +2116,8 @@ ] } ], - "title": "Average Age of Critical Outstanding Defects by Priority", - "type": "barchart" + "title": "Work Done % [in $month]", + "type": "piechart" }, { "datasource": "mysql", @@ -2025,13 +2127,41 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 68, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" } }, - "mappings": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" }, "overrides": [] }, @@ -2039,47 +2169,45 @@ "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 67 }, - "id": 21, - "links": [], + "id": 20, "options": { - "displayLabels": [ - "name", - "percent" - ], + "barRadius": 0, + "barWidth": 0.27, + "fullHighlight": false, + "groupWidth": 0.7, "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { "calcs": [ - "lastNotNull" + "mean" ], - "fields": "", - "values": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" - } + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, "targets": [ { "datasource": "mysql", - "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n count(distinct case when i.type = 'BUG' then i.id else null end) as 'Bug',\n count(distinct case when i.type != 'BUG' and epic_key != '' then i.id else null end) as 'Strategic',\n count(distinct case when i.type != 'BUG' and epic_key = '' then i.id else null end) as 'Non-Strategic'\nfrom\n issues i\n join board_issues bi on i.id = bi.issue_id\n\tjoin boards b on bi.board_id = b.id\n\tjoin project_mapping pm on b.id = pm.row_id\nwhere\n pm.project_name in (${project}) and\n i.resolution_date is not null and\n date(resolution_date) between\n STR_TO_DATE('$month','%Y-%m-%d') and\n STR_TO_DATE('$month','%Y-%m-%d') + INTERVAL 1 MONTH - INTERVAL 1 DAY", + "rawSql": "select\n i.priority as 'Priority',\n AVG(TIMESTAMPDIFF(MINUTE, i.created_date, NOW()) / 1440) as 'Average Age'\nfrom\n issues i\n join board_issues bi on i.id = bi.issue_id\n\tjoin boards b on bi.board_id = b.id\n\tjoin project_mapping pm on b.id = pm.row_id\nwhere\n pm.project_name in (${project}) and\n i.status = 'TODO'\n and i.type = 'BUG'\n and i.priority in (${priority})\ngroup by\n i.priority", "refId": "A", "select": [ [ @@ -2091,23 +2219,6 @@ } ] ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, "table": "_devlake_blueprints", "timeColumn": "created_at", "timeColumnType": "timestamp", @@ -2120,8 +2231,8 @@ ] } ], - "title": "Work Done % [Selected Month]", - "type": "piechart" + "title": "Average Age of Critical Outstanding Defects by Priority", + "type": "barchart" }, { "datasource": { @@ -2132,7 +2243,7 @@ "h": 2, "w": 24, "x": 0, - "y": 73 + "y": 75 }, "id": 34, "options": { @@ -2144,7 +2255,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "11.0.0", "targets": [ { "datasource": { @@ -2159,8 +2270,7 @@ } ], "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [ "Engineering Leads Dashboard" ], @@ -2168,9 +2278,13 @@ "list": [ { "current": { - "selected": false, - "text": "All", - "value": "$__all" + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] }, "datasource": "mysql", "definition": "select distinct name from projects", @@ -2187,6 +2301,27 @@ "sort": 0, "type": "query" }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "datasource": "mysql", + "definition": "select\n distinct(concat(date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m') , ':', date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m-%d'))) as month\nfrom\n issues i\norder by month desc", + "hide": 0, + "includeAll": false, + "label": "Selected Month", + "multi": false, + "name": "month", + "options": [], + "query": "select\n distinct(concat(date_format(month_timestamp, '%Y-%m') , ':', date_format(month_timestamp, '%Y-%m-%d'))) as month\nfrom\n calendar_months\nwhere \n month_timestamp <= curdate()\norder by month desc\nlimit 12", + "refresh": 1, + "regex": "/^(?.*):(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, { "current": { "selected": true, @@ -2233,27 +2368,6 @@ "skipUrlSync": false, "sort": 0, "type": "query" - }, - { - "current": { - "selected": false, - "text": "2024-01", - "value": "2024-01-01" - }, - "datasource": "mysql", - "definition": "select\n distinct(concat(date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m') , ':', date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m-%d'))) as month\nfrom\n issues i\norder by month desc", - "hide": 0, - "includeAll": false, - "label": "Selected Month", - "multi": false, - "name": "month", - "options": [], - "query": "select\n distinct(concat(date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m') , ':', date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m-%d'))) as month\nfrom\n issues i\norder by month desc", - "refresh": 1, - "regex": "/^(?.*):(?.*)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, @@ -2261,10 +2375,11 @@ "from": "now-6M", "to": "now" }, + "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, "timezone": "", "title": "Engineering Overview", "uid": "ZF6abXX7z", - "version": 2, + "version": 15, "weekStart": "" } \ No newline at end of file