Merge pull request #1 from DSACMS/dev

Prepare Metrics Backend for Becoming an External Module and Add Tests

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,15 @@
+## Issue Report
+
+module-name: One line summary of the issue (less than 72 characters)
+
+### Expected behavior
+
+As concisely as possible, describe the expected behavior.
+
+### Actual behavior
+
+As concisely as possible, describe the observed behavior.
+
+### Steps to reproduce the behavior
+
+List all relevant steps to reproduce the observed behavior.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,49 @@
+<!--
+Thank you for sending the PR! We appreciate you spending the time to work on
+these changes.
+
+Help us understand your motivation by explaining why you decided to make this change.
+
+Happy contributing!
+
+- Comments should be formatted to a width no greater than 80 columns.
+
+- Files should be exempt of trailing spaces.
+
+- We adhere to a specific format for commit messages. Please write your commit
+messages along these guidelines. Please keep the line width no greater than 80
+columns (You can use `fmt -n -p -w 80` to accomplish this).
+
+
+-->
+
+## module-name: One line description of your change (less than 72 characters)
+
+## Problem
+
+Explain the context and why you're making that change. What is the problem
+you're trying to solve? In some cases there is not a problem and this can be
+thought of being the motivation for your change.
+
+## Solution
+
+Describe the modifications you've done.
+
+## Result
+
+What will change as a result of your pull request? Note that sometimes this
+section is unnecessary because it is self-explanatory based on the solution.
+
+Some important notes regarding the summary line:
+
+* Describe what was done; not the result 
+* Use the active voice 
+* Use the present tense 
+* Capitalize properly 
+* Do not end in a period — this is a title/subject 
+* Prefix the subject with its scope
+
+## Test Plan
+
+(Write your test plan here. If you changed any code, please provide us with
+clear instructions on how you verified your changes work.)

.gitignore

@@ -0,0 +1,6 @@
+.venv
+.pytest_cache
+__pycache__/
+tests/__pycache__
+metrics_dash_backend_tools/__pycache__
+dist/

metrics_dash_backend_tools/__init__.py

@@ -0,0 +1,7 @@
+from .constants import *
+from .fetch_public_metrics import *
+from .graphs import *
+from .metrics_data_structures import *
+from .metrics_definitions import *
+from .oss_metric_entities import *
+from .reports import *

metrics_dash_backend_tools/constants.py

@@ -14,15 +14,6 @@
 GH_GQL_ENDPOINT = "https://api.github.com/graphql"
 AUGUR_HOST = os.getenv("AUGUR_HOST")
 
-PATH_TO_TEMPLATES = (Path(__file__).parent / "../../templates").resolve()
-
-template_path = os.path.join(PATH_TO_TEMPLATES, "repo_report_template.md")
-with open(template_path, "r", encoding="utf-8") as file:
-    REPO_REPORT_TEMPLATE = file.read()
-
-with open(os.path.join(PATH_TO_TEMPLATES, "org_report_template.md"), "r", encoding="utf-8") as file:
-    ORG_REPORT_TEMPLATE = file.read()
-
 class DesiredReportBehavior(Enum):
     """
     Enumeration class to define constants for report

metrics_dash_backend_tools/fetch_public_metrics.py

@@ -60,17 +60,18 @@ def parse_repos_and_orgs_into_objects(org_name_list, repo_name_list):
             repos.append(Repository(repo_url, org_id))
     return orgs, repos
 
-def get_all_data(all_orgs, all_repos):
+def get_all_data(data_path,graphs_path,all_orgs, all_repos):
     """
     Call relevant methods on orgs and repos
 
     Arguments:
+        data_path: Path to the directory to store json data
         all_orgs: List of all orgs to gather metrics for
         all_repos: List of all repos to gather metrics for
     """
-    fetch_all_new_metric_data(all_orgs, all_repos)
-    read_previous_metric_data(all_repos, all_orgs)
-    write_metric_data_json_to_file(all_orgs, all_repos)
+    fetch_all_new_metric_data(graphs_path,all_orgs, all_repos)
+    read_previous_metric_data(data_path,all_repos, all_orgs)
+    write_metric_data_json_to_file(data_path,all_orgs, all_repos)
 
 
 def add_info_to_org_from_list_of_repos(repo_list, org):
@@ -114,14 +115,15 @@ def add_info_to_org_from_list_of_repos(repo_list, org):
     org.store_metrics(org_counts)
 
 
-def fetch_all_new_metric_data(all_orgs, all_repos):
+def fetch_all_new_metric_data(graphs_data_path,all_orgs, all_repos):
     """
     This method applies all desired methods to all desired repos 
     and orgs. It applies and stores all the metrics
 
     This is mainly to avoid using more api calls than we have to.
 
     Arguments:
+        graphs_data_path: Path to where to store the graph data
         all_orgs: List of all orgs to gather metrics for
         all_repos: List of all repos to gather metrics for
     """
@@ -138,7 +140,7 @@ def fetch_all_new_metric_data(all_orgs, all_repos):
             repo.apply_metric_and_store_data(metric)
 
         for metric in RESOURCE_METRICS:
-            repo.apply_metric_and_store_data(metric, oss_entity=repo)
+            repo.apply_metric_and_store_data(metric,graphs_data_path, oss_entity=repo)
 
         for metric in ADVANCED_METRICS:
             repo.apply_metric_and_store_data(metric)
@@ -151,19 +153,20 @@ def fetch_all_new_metric_data(all_orgs, all_repos):
             print(metric.name)
         add_info_to_org_from_list_of_repos(all_repos, org)
 
-def read_current_metric_data(repos,orgs):
+def read_current_metric_data(data_path,repos,orgs):
     """
     Read current metrics and load previous metrics that 
     were saved in .old files.
 
     Arguments:
+        data_path: path to the json data
         orgs: orgs to read data for.
         repos: repos to read data for.
     """
 
     for org in orgs:
 
-        path = org.get_path_to_json_data()
+        path = org.get_path_to_json_data(data_path)
         #generate dict of previous and save it as {path}.old
         #previous_metric_org_json = json.dumps(org.previous_metric_data, indent=4)
 
@@ -183,7 +186,7 @@ def read_current_metric_data(repos,orgs):
 
     for repo in repos:
         #previous_metric_repo_json = json.dumps(repo.previous_metric_data, indent=4)
-        path = repo.get_path_to_json_data()
+        path = repo.get_path_to_json_data(data_path)
 
         with open(f"{path}.old","r",encoding="utf-8") as file:
             #file.write(previous_metric_repo_json)
@@ -199,20 +202,21 @@ def read_current_metric_data(repos,orgs):
         repo.metric_data.update(metric_repo_json)
 
 
-def read_previous_metric_data(repos, orgs):
+def read_previous_metric_data(data_path,repos, orgs):
     """
     This method reads the previously gathered metric data and 
     stores it in the OSSEntity objects passed in.
 
     This is for the reports that compare changes since last collection.
 
     Arguments:
+        data_path: path to read the previous json data
         repos: List of all orgs to read metrics for
         orgs: List of all repos to read metrics for
     """
     for org in orgs:
         try:
-            with open(org.get_path_to_json_data(), "r", encoding="utf-8") as file:
+            with open(org.get_path_to_json_data(data_path), "r", encoding="utf-8") as file:
                 prev_data = json.load(file)
                 org.previous_metric_data.update(prev_data)
         except FileNotFoundError:
@@ -222,28 +226,29 @@ def read_previous_metric_data(repos, orgs):
 
     for repo in repos:
         try:
-            with open(repo.get_path_to_json_data(), "r", encoding="utf-8") as file:
+            with open(repo.get_path_to_json_data(data_path), "r", encoding="utf-8") as file:
                 prev_data = json.load(file)
                 repo.previous_metric_data.update(prev_data)
         except FileNotFoundError:
             print("Could not find previous data for records for repo" +
                   repo.name)
 
 
-def write_metric_data_json_to_file(orgs, repos):
+def write_metric_data_json_to_file(data_path,orgs, repos):
     """
     Write all metric data to json files.
     
     Keep old metrics as a .old file.
 
     Arguments:
+        data_path: path to write json data to
         orgs: orgs to write to file
         repos: repos to write to file
     """
 
     for org in orgs:
 
-        path = org.get_path_to_json_data()
+        path = org.get_path_to_json_data(data_path)
         #generate dict of previous and save it as {path}.old
         previous_metric_org_json = json.dumps(org.previous_metric_data, indent=4)
 
@@ -261,7 +266,7 @@ def write_metric_data_json_to_file(orgs, repos):
             file.write(org_metric_data)
 
     for repo in repos:
-        path = repo.get_path_to_json_data()
+        path = repo.get_path_to_json_data(data_path)
 
         previous_metric_repo_json = json.dumps(repo.previous_metric_data, indent=4)
 

metrics_dash_backend_tools/metrics_data_structures.py

@@ -8,7 +8,7 @@
 from functools import reduce
 import operator
 import requests
-from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT, REQUEST_RETRIES
+from .constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT, REQUEST_RETRIES
 
 # Simple metric that can be represented by a count or value.
 
@@ -205,11 +205,11 @@ def hit_metric(self, params=None):
         # return response
         return response
 
-    def get_values(self,params=None, oss_entity=None):
+    def get_values(self,graphs_path,params=None, oss_entity=None):
 
         r = self.hit_metric(params=params)
 
-        path = oss_entity.get_path_to_resource_data(self.name, fmt=self.format)
+        path = oss_entity.get_path_to_resource_data(graphs_path,self.name, fmt=self.format)
 
         if r.status_code == 200:
             errtext = "There is no data for this repo, in the database you are accessing"

metrics_dash_backend_tools/metrics_definitions.py

@@ -1,11 +1,11 @@
 """
 Definitions of specific metrics for metricsLib
 """
-from metricsLib.metrics_data_structures import CustomMetric, parse_commits_by_month, RangeMetric
-from metricsLib.metrics_data_structures import GraphQLMetric, LengthMetric, ResourceMetric, BaseMetric
-from metricsLib.metrics_data_structures import ListMetric, parse_nadia_label_into_badge
-from metricsLib.metrics_data_structures import BaseMetric, LanguageMetric
-from metricsLib.constants import TOKEN, AUGUR_HOST
+from .metrics_data_structures import CustomMetric, parse_commits_by_month, RangeMetric
+from .metrics_data_structures import GraphQLMetric, LengthMetric, ResourceMetric, BaseMetric
+from .metrics_data_structures import ListMetric, parse_nadia_label_into_badge
+from .metrics_data_structures import BaseMetric, LanguageMetric
+from .constants import TOKEN, AUGUR_HOST
 
 # The general procedure is to execute all metrics against all repos and orgs
 

metrics_dash_backend_tools/oss_metric_entities.py

@@ -10,8 +10,8 @@
 import pathlib
 import requests
 from requests.exceptions import ReadTimeout
-from metricsLib.constants import AUGUR_HOST
-from metricsLib.constants import TIMEOUT_IN_SECONDS
+from .constants import AUGUR_HOST
+from .constants import TIMEOUT_IN_SECONDS
 
 
 def get_repo_owner_and_name(repo_http_url):
@@ -315,7 +315,7 @@ def get_path_to_resource_data(self,graphs_data_path, resource_name, fmt="png"):
         fname = f"{self.repo_owner}/{self.name}/{resource_name}_{self.name}_data.{fmt}"
         return os.path.join(graphs_data_path, fname)
 
-    def get_path_to_graph_data(self, graph_name):
+    def get_path_to_graph_data(self,graphs_data_path, graph_name):
         """
         Derive the path for graph data using svg
         parent path and extension
@@ -324,7 +324,7 @@ def get_path_to_graph_data(self, graph_name):
             String path to data.
         """
 
-        return self.get_path_to_resource_data(graph_name, fmt="svg")
+        return self.get_path_to_resource_data(graphs_data_path,graph_name, fmt="svg")
 
 
 class GithubOrg(OSSEntity):
@@ -442,7 +442,7 @@ def get_path_to_resource_data(self,graphs_data_path, resource_name, fmt="png"):
 
         return org_path
 
-    def get_path_to_graph_data(self, chart_name):
+    def get_path_to_graph_data(self,graphs_data_path, chart_name):
         """
         Derive the path for graph data using parent
         path and extension
@@ -451,4 +451,4 @@ def get_path_to_graph_data(self, chart_name):
             String path to data.
         """
 
-        return self.get_path_to_resource_data(chart_name, fmt="svg")
+        return self.get_path_to_resource_data(graphs_data_path,chart_name, fmt="svg")