Create JSON files instead of CSV from Collectors

.github/workflows/nextjs.yml

@@ -18,6 +18,9 @@ permissions:
   pages: write
   id-token: write
 
+env:
+  GRAPHQL_TOKEN: ${{ secrets.GRAPHQL_TOKEN }}
+  ORGANIZATION_NAME: WorldHealthOrganization
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
 # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
 concurrency:
@@ -31,6 +34,26 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v4
+        with:
+          go-version: ^1.19
+      - name: Get dependencies
+        run: |
+          cd backend
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              dep ensure
+          fi
+      - name: Build Go
+        run: make build
+      # - name: Test
+      #   run: make test
+      - name: Run Go program and save output
+        id: run
+        run: |
+          $PWD/backend/bin/metrics
       - name: Detect package manager
         id: detect-package-manager
         run: |

README.md

@@ -73,7 +73,15 @@ Issue Project  [here](https://github.com/github/SI-skills-based-volunteering/iss
 
 <!-- TODO: Add min requirements and deployment steps -->
 ### Backend
-Run the following command to run the action locally
+
+To update the repository data.
+
+1. Generate a [new GitHub Token](https://github.com/settings/tokens) with the ability to read repo, read org, and  read projects scopes.
+1. Set the `GRAPHQL_TOKEN` environment variable to be the value of your newly created token.
+1. Run the following command from the root of the repository:
 ```
-gh act -W .github/workflows/graphql.yml  --artifact-server-path ./tmp/ --env-file dev.vscode.env
+make build
+./backend/bin/metrics
 ```
+
+This will generate a new `data.json` file in the UI directory which can be imported directly as part of the static build.

backend/business/core/collectors/github/fetchers/contribution_info.go

@@ -2,7 +2,6 @@ package fetchers
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/shurcooL/githubv4"
 )
@@ -12,13 +11,18 @@ type ContributionInfoFetcher struct {
 	orgName string
 }
 
+type ContributionInfoResult struct {
+	RepositoryName string `json:"repositoryName"`
+}
+
 func NewContributionInfoFetcher(client *githubv4.Client, orgName string) *ContributionInfoFetcher {
 	return &ContributionInfoFetcher{client: client, orgName: orgName}
 }
 
 type contributionInfo struct {
 	Node struct {
 		NameWithOwner githubv4.String
+		Name          githubv4.String
 	}
 }
 
@@ -34,7 +38,7 @@ type contributionInfoQuery struct {
 	} `graphql:"organization(login:$organizationLogin)"`
 }
 
-func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (string, error) {
+func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (*map[string]ContributionInfoResult, error) {
 	variables := map[string]interface{}{
 		"organizationLogin": githubv4.String(c.orgName),
 		"reposCursor":       (*githubv4.String)(nil), // Null after argument to get first page.
@@ -47,7 +51,7 @@ func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (string, error) {
 	for {
 		err := c.client.Query(ctx, &q, variables)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 		allRepos = append(allRepos, q.Organization.Repositories.Edges...)
 		if !q.Organization.Repositories.PageInfo.HasNextPage {
@@ -56,23 +60,27 @@ func (c *ContributionInfoFetcher) Fetch(ctx context.Context) (string, error) {
 		variables["reposCursor"] = githubv4.NewString(q.Organization.Repositories.PageInfo.EndCursor)
 	}
 
-	csvString, err := c.formatCSV(allRepos)
-	if err != nil {
-		return "", err
-	}
-	return csvString, nil
+	result := c.buildResult(allRepos)
+	return result, nil
 }
 
-func (f *ContributionInfoFetcher) formatCSV(data []contributionInfo) (string, error) {
-	csvString := "repo_name,issues_opened\n"
+func (f *ContributionInfoFetcher) buildResult(data []contributionInfo) *map[string]ContributionInfoResult {
+	result := make(map[string]ContributionInfoResult)
 	for _, edge := range data {
-		csvString += fmt.Sprintf("%s\n",
-			edge.Node.NameWithOwner,
-			// edge.Node.OpenIssues.TotalCount,
-			// edge.Node.ClosedIssues.TotalCount,
-			// edge.Node.OpenPullRequests.TotalCount,
-			// edge.Node.MergedPullRequests.TotalCount,
-		)
+		result[string(edge.Node.NameWithOwner)] = ContributionInfoResult{
+			RepositoryName: string(edge.Node.Name),
+			// CollaboratorsCount: edge.Node.Collaborators.TotalCount,
+			// ProjectsCount:      edge.Node.Projects.TotalCount + edge.Node.ProjectsV2.TotalCount,
+			// DiscussionsCount:   edge.Node.Discussions.TotalCount,
+			// ForksCount:         edge.Node.Forks.TotalCount,
+			// IssuesCount:        edge.Node.Issues.TotalCount,
+			// OpenIssuesCount:    edge.Node.OpenIssues.TotalCount,
+			// ClosedIssuesCount:  edge.Node.ClosedIssues.TotalCount,
+			// OpenPRsCount:       edge.Node.OpenPullRequests.TotalCount,
+			// MergedPRsCount:     edge.Node.MergedPullRequests.TotalCount,
+			// LicenseName:        edge.Node.LicenseInfo.Name,
+			// WatchersCount:      edge.Node.Watchers.TotalCount,
+		}
 	}
-	return csvString, nil
+	return &result
 }

backend/business/core/collectors/github/fetchers/org_info.go

@@ -2,16 +2,27 @@ package fetchers
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/shurcooL/githubv4"
 )
 
-type OrgInfo struct {
+type OrgInfoFetcher struct {
 	client  *githubv4.Client
 	orgName string
 }
 
+type OrgInfoResult struct {
+	Login                string `json:"login"`
+	Name                 string `json:"name"`
+	Description          string `json:"description"`
+	CreatedAt            string `json:"createdAt"`
+	MembersWithRoleCount int    `json:"membersWithRoleCount"`
+	ProjectsCount        int    `json:"projectsCount"`
+	ProjectsV2Count      int    `json:"projectsV2Count"`
+	RepositoriesCount    int    `json:"repositoriesCount"`
+	TeamsCount           int    `json:"teamsCount"`
+}
+
 type orgInfo struct {
 	Login           githubv4.String
 	Name            githubv4.String
@@ -38,41 +49,40 @@ type orgInfoQuery struct {
 	Organization orgInfo `graphql:"organization(login:$organizationLogin)"`
 }
 
-func NewOrgInfo(client *githubv4.Client, orgName string) *OrgInfo {
-	return &OrgInfo{client: client, orgName: orgName}
+func NewOrgInfo(client *githubv4.Client, orgName string) *OrgInfoFetcher {
+	return &OrgInfoFetcher{client: client, orgName: orgName}
 }
 
-func (o *OrgInfo) Fetch(ctx context.Context) (string, error) {
+func (oif *OrgInfoFetcher) Fetch(ctx context.Context) (*OrgInfoResult, error) {
 	variables := map[string]interface{}{
-		"organizationLogin": githubv4.String(o.orgName),
+		"organizationLogin": githubv4.String(oif.orgName),
 	}
 
 	var q orgInfoQuery
 
-	err := o.client.Query(ctx, &q, variables)
+	err := oif.client.Query(ctx, &q, variables)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	csvString, err := o.formatCSV(q.Organization)
+	result := oif.buildResult(q.Organization)
 
 	if err != nil {
-		return "", err
+		return result, err
 	}
-	return csvString, nil
+	return result, nil
 }
 
-func (f *OrgInfo) formatCSV(data orgInfo) (string, error) {
-	csvString := "login,name,description,createdAt,totalMembers,totalProjects,totalRepositories,totalTeams\n"
-	csvString += fmt.Sprintf("%s,%s,%s,%s,%d,%d,%d,%d\n",
-		data.Login,
-		data.Name,
-		data.Description,
-		data.CreatedAt,
-		data.MembersWithRole.TotalCount,
-		data.Projects.TotalCount+data.ProjectsV2.TotalCount,
-		data.Repositories.TotalCount,
-		data.Teams.TotalCount,
-	)
-	return csvString, nil
+func (f *OrgInfoFetcher) buildResult(data orgInfo) *OrgInfoResult {
+	return &OrgInfoResult{
+		Login:                string(data.Login),
+		Name:                 string(data.Name),
+		Description:          string(data.Description),
+		CreatedAt:            data.CreatedAt.Format("2006-01-02T15:04:05-0700"),
+		MembersWithRoleCount: int(data.MembersWithRole.TotalCount),
+		ProjectsCount:        int(data.Projects.TotalCount),
+		ProjectsV2Count:      int(data.ProjectsV2.TotalCount),
+		RepositoriesCount:    int(data.Repositories.TotalCount),
+		TeamsCount:           int(data.Teams.TotalCount),
+	}
 }