Create JSON files instead of CSV from Collectors

.github/workflows/nextjs.yml

@@ -31,6 +31,26 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v4
+        with:
+          go-version: ^1.19
+      - name: Get dependencies
+        run: |
+          cd backend
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              dep ensure
+          fi
+      - name: Build Go
+        run: make build
+      # - name: Test
+      #   run: make test
+      - name: Run Go program and save output
+        id: run
+        run: |
+          $PWD/backend/bin/metrics
       - name: Detect package manager
         id: detect-package-manager
         run: |

README.md

@@ -73,7 +73,14 @@ Issue Project  [here](https://github.com/github/SI-skills-based-volunteering/iss
 
 <!-- TODO: Add min requirements and deployment steps -->
 ### Backend
-Run the following command to run the action locally
+
+To update the repository data.
+
+1. Generate a [new GitHub Token](https://github.com/settings/tokens) with the ability to read repo and projects.
+2. Run the following command from the root of the repository
 ```
-gh act -W .github/workflows/graphql.yml  --artifact-server-path ./tmp/ --env-file dev.vscode.env
+make build
+./backend/bin/metrics
 ```
+
+This will generate a new `data.json` file in the UI directory which can be imported directly as part of the static build.

backend/business/core/collectors/github/fetchers/contribution_info.go

@@ -12,13 +12,18 @@ type ContributionInfoFetcher struct {
 	orgName string
 }
 
+type ContributionInfoResult struct {
+	RepositoryName string `json:"repositoryName"`
+}
+
 func NewContributionInfoFetcher(client *githubv4.Client, orgName string) *ContributionInfoFetcher {
 	return &ContributionInfoFetcher{client: client, orgName: orgName}
 }
 
 type contributionInfo struct {
 	Node struct {
 		NameWithOwner githubv4.String
+		Name          githubv4.String
 	}
 }
 
@@ -34,7 +39,7 @@ type contributionInfoQuery struct {
 	} `graphql:"organization(login:$organizationLogin)"`
 }
 
-func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (string, error) {
+func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (*map[string]ContributionInfoResult, error) {
 	variables := map[string]interface{}{
 		"organizationLogin": githubv4.String(c.orgName),
 		"reposCursor":       (*githubv4.String)(nil), // Null after argument to get first page.
@@ -47,7 +52,7 @@ func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (string, error) {
 	for {
 		err := c.client.Query(ctx, &q, variables)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 		allRepos = append(allRepos, q.Organization.Repositories.Edges...)
 		if !q.Organization.Repositories.PageInfo.HasNextPage {
@@ -56,14 +61,12 @@ func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (string, error) {
 		variables["reposCursor"] = githubv4.NewString(q.Organization.Repositories.PageInfo.EndCursor)
 	}
 
-	csvString, err := c.formatCSV(allRepos)
-	if err != nil {
-		return "", err
-	}
-	return csvString, nil
+	result := c.buildResult(allRepos)
+	return result, nil
 }
 
-func (f *ContributionInfoFetcher) formatCSV(data []contributionInfo) (string, error) {
+// FormatCSV formats the given data as CSV
+func (f *ContributionInfoFetcher) FormatCSV(data []contributionInfo) (string, error) {
 	csvString := "repo_name,issues_opened\n"
 	for _, edge := range data {
 		csvString += fmt.Sprintf("%s\n",
@@ -76,3 +79,24 @@ func (f *ContributionInfoFetcher) formatCSV(data []contributionInfo) (string, er
 	}
 	return csvString, nil
 }
+
+func (f *ContributionInfoFetcher) buildResult(data []contributionInfo) *map[string]ContributionInfoResult {
+	result := make(map[string]ContributionInfoResult)
+	for _, edge := range data {
+		result[string(edge.Node.NameWithOwner)] = ContributionInfoResult{
+			RepositoryName: string(edge.Node.Name),
+			// CollaboratorsCount: edge.Node.Collaborators.TotalCount,
+			// ProjectsCount:      edge.Node.Projects.TotalCount + edge.Node.ProjectsV2.TotalCount,
+			// DiscussionsCount:   edge.Node.Discussions.TotalCount,
+			// ForksCount:         edge.Node.Forks.TotalCount,
+			// IssuesCount:        edge.Node.Issues.TotalCount,
+			// OpenIssuesCount:    edge.Node.OpenIssues.TotalCount,
+			// ClosedIssuesCount:  edge.Node.ClosedIssues.TotalCount,
+			// OpenPRsCount:       edge.Node.OpenPullRequests.TotalCount,
+			// MergedPRsCount:     edge.Node.MergedPullRequests.TotalCount,
+			// LicenseName:        edge.Node.LicenseInfo.Name,
+			// WatchersCount:      edge.Node.Watchers.TotalCount,
+		}
+	}
+	return &result
+}

backend/business/core/collectors/github/fetchers/org_info.go

@@ -7,11 +7,23 @@ import (
 	"github.com/shurcooL/githubv4"
 )
 
-type OrgInfo struct {
+type OrgInfoFetcher struct {
 	client  *githubv4.Client
 	orgName string
 }
 
+type OrgInfoResult struct {
+	Login                string `json:"login"`
+	Name                 string `json:"name"`
+	Description          string `json:"description"`
+	CreatedAt            string `json:"createdAt"`
+	MembersWithRoleCount int    `json:"membersWithRoleCount"`
+	ProjectsCount        int    `json:"projectsCount"`
+	ProjectsV2Count      int    `json:"projectsV2Count"`
+	RepositoriesCount    int    `json:"repositoriesCount"`
+	TeamsCount           int    `json:"teamsCount"`
+}
+
 type orgInfo struct {
 	Login           githubv4.String
 	Name            githubv4.String
@@ -38,31 +50,31 @@ type orgInfoQuery struct {
 	Organization orgInfo `graphql:"organization(login:$organizationLogin)"`
 }
 
-func NewOrgInfo(client *githubv4.Client, orgName string) *OrgInfo {
-	return &OrgInfo{client: client, orgName: orgName}
+func NewOrgInfo(client *githubv4.Client, orgName string) *OrgInfoFetcher {
+	return &OrgInfoFetcher{client: client, orgName: orgName}
 }
 
-func (o *OrgInfo) Fetch(ctx context.Context) (string, error) {
+func (oif *OrgInfoFetcher) Fetch(ctx context.Context) (*OrgInfoResult, error) {
 	variables := map[string]interface{}{
-		"organizationLogin": githubv4.String(o.orgName),
+		"organizationLogin": githubv4.String(oif.orgName),
 	}
 
 	var q orgInfoQuery
 
-	err := o.client.Query(ctx, &q, variables)
+	err := oif.client.Query(ctx, &q, variables)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	csvString, err := o.formatCSV(q.Organization)
+	result := oif.buildResult(q.Organization)
 
 	if err != nil {
-		return "", err
+		return result, err
 	}
-	return csvString, nil
+	return result, nil
 }
 
-func (f *OrgInfo) formatCSV(data orgInfo) (string, error) {
+func (f *OrgInfoFetcher) formatCSV(data orgInfo) (string, error) {
 	csvString := "login,name,description,createdAt,totalMembers,totalProjects,totalRepositories,totalTeams\n"
 	csvString += fmt.Sprintf("%s,%s,%s,%s,%d,%d,%d,%d\n",
 		data.Login,
@@ -76,3 +88,17 @@ func (f *OrgInfo) formatCSV(data orgInfo) (string, error) {
 	)
 	return csvString, nil
 }
+
+func (f *OrgInfoFetcher) buildResult(data orgInfo) *OrgInfoResult {
+	return &OrgInfoResult{
+		Login:                string(data.Login),
+		Name:                 string(data.Name),
+		Description:          string(data.Description),
+		CreatedAt:            data.CreatedAt.Format("2006-01-02T15:04:05-0700"),
+		MembersWithRoleCount: int(data.MembersWithRole.TotalCount),
+		ProjectsCount:        int(data.Projects.TotalCount),
+		ProjectsV2Count:      int(data.ProjectsV2.TotalCount),
+		RepositoriesCount:    int(data.Repositories.TotalCount),
+		TeamsCount:           int(data.Teams.TotalCount),
+	}
+}

backend/business/core/collectors/github/fetchers/repos_info_fetcher.go

@@ -7,6 +7,21 @@ import (
 	"github.com/shurcooL/githubv4"
 )
 
+type RepoInfoResult struct {
+	RepoName                string `json:"repoName"`
+	CollaboratorsCount      int    `json:"collaboratorsCount"`
+	ProjectsCount           int    `json:"projectsCount"`
+	DiscussionsCount        int    `json:"discussionsCount"`
+	ForksCount              int    `json:"forksCount"`
+	IssuesCount             int    `json:"issuesCount"`
+	OpenIssuesCount         int    `json:"openIssuesCount"`
+	ClosedIssuesCount       int    `json:"closedIssuesCount"`
+	OpenPullRequestsCount   int    `json:"openPullRequestsCount"`
+	MergedPullRequestsCount int    `json:"mergedPullRequestsCount"`
+	LicenseName             string `json:"licenseName"`
+	WatchersCount           int    `json:"watchersCount"`
+}
+
 type ReposInfoFetcher struct {
 	client  *githubv4.Client
 	orgName string
@@ -37,7 +52,7 @@ type repoInfo struct {
 		Issues struct {
 			TotalCount githubv4.Int
 		}
-		OpenIssues    struct {
+		OpenIssues struct {
 			TotalCount githubv4.Int
 		} `graphql:"openIssues: issues(states: OPEN)"`
 		ClosedIssues struct {
@@ -66,11 +81,12 @@ type reposInfoQuery struct {
 				EndCursor   githubv4.String
 				HasNextPage bool
 			}
-		} `graphql:"repositories(first: 100, privacy: PUBLIC, after: $reposCursor)"`
+		} `graphql:"repositories(first: 20, privacy: PUBLIC, after: $reposCursor)"`
 	} `graphql:"organization(login:$organizationLogin)"`
 }
 
-func (f *ReposInfoFetcher) Fetch(ctx context.Context) (string, error) {
+// Fetch fetches the repos info for the given organization
+func (f *ReposInfoFetcher) Fetch(ctx context.Context) (*map[string]RepoInfoResult, error) {
 	variables := map[string]interface{}{
 		"organizationLogin": githubv4.String(f.orgName),
 		"reposCursor":       (*githubv4.String)(nil), // Null after argument to get first page.
@@ -82,8 +98,9 @@ func (f *ReposInfoFetcher) Fetch(ctx context.Context) (string, error) {
 
 	for {
 		err := f.client.Query(ctx, &q, variables)
+
 		if err != nil {
-			return "", err
+			return nil, fmt.Errorf("failed to fetch repos info for %s: %s", f.orgName, err.Error())
 		}
 		allRepos = append(allRepos, q.Organization.Repositories.Edges...)
 		if !q.Organization.Repositories.PageInfo.HasNextPage {
@@ -92,14 +109,12 @@ func (f *ReposInfoFetcher) Fetch(ctx context.Context) (string, error) {
 		variables["reposCursor"] = githubv4.NewString(q.Organization.Repositories.PageInfo.EndCursor)
 	}
 
-	csvString, err := f.formatCSV(allRepos)
-	if err != nil {
-		return "", err
-	}
-	return csvString, nil
+	result := f.buildJSON(allRepos)
+	return result, nil
 }
 
-func (f *ReposInfoFetcher) formatCSV(data []repoInfo) (string, error) {
+// FormatCSV formats the given data as CSV file
+func (f *ReposInfoFetcher) FormatCSV(data []repoInfo) (string, error) {
 	csvString := "repo_name,collaborators_count,projects_count,discussions_count,forks_count,issues_count,open_issues_count,closed_issues_count,open_pull_requests_count,merged_pull_requests_count,license_name,watchers_count\n"
 	for _, edge := range data {
 		csvString += fmt.Sprintf("%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%s,%d\n",
@@ -119,3 +134,25 @@ func (f *ReposInfoFetcher) formatCSV(data []repoInfo) (string, error) {
 	}
 	return csvString, nil
 }
+
+func (f *ReposInfoFetcher) buildJSON(data []repoInfo) *map[string]RepoInfoResult {
+	holder := make(map[string]RepoInfoResult)
+	for _, edge := range data {
+		holder[string(edge.Node.NameWithOwner)] = RepoInfoResult{
+			RepoName:                string(edge.Node.NameWithOwner),
+			CollaboratorsCount:      int(edge.Node.Collaborators.TotalCount),
+			ProjectsCount:           int(edge.Node.Projects.TotalCount + edge.Node.ProjectsV2.TotalCount),
+			DiscussionsCount:        int(edge.Node.Discussions.TotalCount),
+			ForksCount:              int(edge.Node.Forks.TotalCount),
+			IssuesCount:             int(edge.Node.Issues.TotalCount),
+			OpenIssuesCount:         int(edge.Node.OpenIssues.TotalCount),
+			ClosedIssuesCount:       int(edge.Node.ClosedIssues.TotalCount),
+			OpenPullRequestsCount:   int(edge.Node.OpenPullRequests.TotalCount),
+			MergedPullRequestsCount: int(edge.Node.MergedPullRequests.TotalCount),
+			LicenseName:             string(edge.Node.LicenseInfo.Name),
+			WatchersCount:           int(edge.Node.Watchers.TotalCount),
+		}
+	}
+
+	return &holder
+}