Merge branch 'master' into pr/fixups

projects/GATech_Ramprasad.yaml

@@ -0,0 +1,8 @@
+Department: Material Science
+Description: 'Develop and apply computational and machine learning tools to accelerate
+  materials discovery. More information can be found here: https://ramprasad.mse.gatech.edu/'
+FieldOfScience: Materials Science
+FieldOfScienceID: 14.1801b
+InstitutionID: https://osg-htc.org/iid/uvf22j6xjbtv
+Organization: Georgia Institute of Technology
+PIName: Rampi Ramprasad

src/webapp/models.py

@@ -33,6 +33,7 @@ def time(self):
 log = logging.getLogger(__name__)
 
 topology_update_summary = Summary('topology_update_seconds', 'Time spent updating the topology repo data')
+topology_git_update_summary = Summary('topology_git_update_seconds', 'Time spent pulling/cloning the topology git repo')
 contact_update_summary = Summary('contact_update_seconds', 'Time spent updating the contact repo data')
 comanage_update_summary = Summary('comanage_update_seconds', 'Time spent updating the comanage LDAP data')
 ligo_update_summary = Summary('ligo_update_seconds', 'Time spent updating the LIGO LDAP data')
@@ -52,16 +53,16 @@ def should_update(self):
         """Return True if we should update, either because we're past the next update time
         or because force_update is True.
         """
-        return self.force_update or not self.data or time.time() > self.next_update
+        return self.force_update or not self.data or time.monotonic() > self.next_update
 
     def try_again(self):
         """Set the next update time to now + the retry delay."""
-        self.next_update = time.time() + self.retry_delay
+        self.next_update = time.monotonic() + self.retry_delay
 
     def update(self, data):
         """Cache new data and set the next update time to now + the cache lifetime."""
         self.data = data
-        self.timestamp = time.time()
+        self.timestamp = time.monotonic()
         self.next_update = self.timestamp + self.cache_lifetime
         self.force_update = False
 
@@ -90,6 +91,7 @@ def __init__(self, config=None, strict=False):
         self.topology = CachedData(cache_lifetime=topology_cache_lifetime)
         self.vos_data = CachedData(cache_lifetime=topology_cache_lifetime)
         self.mappings = CachedData(cache_lifetime=topology_cache_lifetime)
+        self.topology_repo_stamp = CachedData(cache_lifetime=topology_cache_lifetime)
         self.topology_data_dir = config["TOPOLOGY_DATA_DIR"]
         self.topology_data_repo = config.get("TOPOLOGY_DATA_REPO", "")
         self.topology_data_branch = config.get("TOPOLOGY_DATA_BRANCH", "")
@@ -153,6 +155,21 @@ def _update_topology_repo(self):
                 return False
         return True
 
+    def maybe_update_topology_repo(self) -> bool:
+        """Update the local git clone of the topology github repo if it hasn't
+        been updated recently (based on the cache time for self.topology_repo_stamp).
+        """
+        if self.topology_repo_stamp.should_update():
+            with topology_git_update_summary.time():
+                ok = self._update_topology_repo()
+            if ok:
+                self.topology_repo_stamp.update(time.monotonic())
+                return True
+            else:
+                self.topology_repo_stamp.try_again()
+                return False
+        return bool(self.topology_repo_stamp.data)
+
     def _update_contacts_repo(self):
         if not self.config["NO_GIT"]:
             parent = os.path.dirname(self.config["CONTACT_DATA_DIR"])
@@ -297,18 +314,20 @@ def get_topology(self) -> Optional[Topology]:
 
         return self.topology.data
 
-    def update_topology(self):
+    def update_topology(self) -> None:
         """
-        Update topology data
+        Update topology facility/site/ResourceGroup data
         """
-        ok = self._update_topology_repo()
+        ok = self.maybe_update_topology_repo()
         if ok:
             try:
+                log.debug("Updating topology RG data")
                 self.topology.update(rg_reader.get_topology(self.topology_dir, self.get_contacts_data(), strict=self.strict))
+                log.debug("Updated topology RG data successfully")
             except Exception as err:
                 if self.strict:
                     raise
-                log.exception("Failed to update topology (%s)", err)
+                log.exception("Failed to update topology RG data (%s)", err)
                 self.topology.try_again()
         else:
             self.topology.try_again()
@@ -320,10 +339,12 @@ def get_vos_data(self) -> Optional[VOsData]:
         """
         if self.vos_data.should_update():
             with topology_update_summary.time():
-                ok = self._update_topology_repo()
+                ok = self.maybe_update_topology_repo()
                 if ok:
                     try:
+                        log.debug("Updating VOs")
                         self.vos_data.update(vo_reader.get_vos_data(self.vos_dir, self.get_contacts_data(), strict=self.strict))
+                        log.debug("Updated VOs successfully")
                     except Exception as err:
                         if self.strict:
                             raise
@@ -341,10 +362,12 @@ def get_projects(self) -> Optional[Dict]:
         """
         if self.projects.should_update():
             with topology_update_summary.time():
-                ok = self._update_topology_repo()
+                ok = self.maybe_update_topology_repo()
                 if ok:
                     try:
+                        log.debug("Updating projects")
                         self.projects.update(project_reader.get_projects(self.projects_dir, strict=self.strict))
+                        log.debug("Updated projects successfully")
                     except Exception as err:
                         if self.strict:
                             raise
@@ -364,10 +387,12 @@ def get_mappings(self, strict=None) -> Optional[mappings.Mappings]:
             strict = self.strict
         if self.mappings.should_update():
             with topology_update_summary.time():
-                ok = self._update_topology_repo()
+                ok = self.maybe_update_topology_repo()
                 if ok:
                     try:
+                        log.debug("Updating mappings")
                         self.mappings.update(mappings.get_mappings(indir=self.mappings_dir, strict=strict))
+                        log.debug("Updated mappings successfully")
                     except Exception as err:
                         if self.strict:
                             raise

topology/DIII-D/National Fusion Facility/FDP-OSDF.yaml

@@ -0,0 +1,43 @@
+Production: true
+SupportCenter: Self Supported
+GroupDescription:  Fusion Data Platform
+Resources:
+  FDP_OSDF_CACHE:
+    Active: true
+    Description: FDP OSDF cache
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          Name: Fabio Andrijauskas 
+          ID: OSG1000162
+      Security Contact:
+        Primary:
+          Name: Fabio Andrijauskas 
+          ID: OSG1000162    
+    DN: /CN=fdp-d3d-cache.nationalresearchplatform.org
+    FQDN: fdp-d3d-cache.nationalresearchplatform.org
+    Services:
+      XRootD cache server:
+        Description: OSDF cache server for FDP project
+    AllowedVOs:
+      - ANY
+
+  FDP_OSDF_ORIGIN:
+    Active: false
+    Description: FDP OSDF origin
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          Name: Fabio Andrijauskas 
+          ID: OSG1000162
+      Security Contact:
+        Primary:
+          Name: Fabio Andrijauskas 
+          ID: OSG1000162    
+    DN: /CN=fdp-d3d-origin.nationalresearchplatform.org
+    FQDN: fdp-d3d-origin.nationalresearchplatform.org
+    Services:
+      XRootD cache server:
+        Description: OSDF origin server for FDP project
+    AllowedVOs:
+      - ANY

topology/DIII-D/National Fusion Facility/SITE.yaml

@@ -0,0 +1,9 @@
+LongName: DIII-D National Fusion Facility
+Description:  Fusion Data Platform
+AddressLine1: 3483 Dunhill St.
+City: San Diego
+Country: US
+Latitude: 32.90246879605678
+Longitude: -117.2283010938664
+State: CA
+Zipcode: 92121

topology/National Center for Atmospheric Research/NCAR-Wyoming Supercomputing Center/NCAR-OSDF.yaml

@@ -29,3 +29,29 @@ Resources:
           auth_endpoint_override: ncar.nationalresearchplatform.org:8443
     AllowedVOs:
       - ANY
+
+  NCAR_OSDF_S3_ORIGIN:
+    Active: true
+    Description: NCAR OSDF S3 Origin
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          Name: Fabio Andrijauskas
+          ID: OSG1000162
+        Secondary:
+          Name: Diego Davila
+          ID: bc36e7fe84fffcb5cf195fe09cc42336f5cd5d1f
+      Security Contact:
+        Primary:
+          Name: Fabio Andrijauskas
+          ID: OSG1000162
+    DN: /CN=ncar-s3-origin.nationalresearchplatform.org
+    FQDN: ncar-s3-origin.nationalresearchplatform.org
+    Services:
+      XRootD origin server:
+        Description: NCAR OSDF Origin
+        Details:
+          endpoint_override: ncar-s3-origin.nationalresearchplatform.org:8444
+          auth_endpoint_override: ncar-s3-origin.nationalresearchplatform.org:8444
+    AllowedVOs:
+      - ANY

topology/University of Missouri/Missouri ITRSS/Missouri-Rise.yaml

@@ -0,0 +1,27 @@
+Production: true
+SupportCenter: Community Support Center
+GroupDescription: Compute infrastructure
+
+Resources:
+  Missouri-Rise-Origin1-NRP:
+    Active: false
+    Description: MO NRP origin
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          Name: Fabio Andrijauskas
+          ID: OSG1000162
+        Secondary:
+          ID: bc36e7fe84fffcb5cf195fe09cc42336f5cd5d1f
+          Name: Diego Davila
+      Security Contact:
+        Primary:
+          Name: Fabio Andrijauskas
+          ID: OSG1000162
+    DN: /CN=mo-origin.nationalresearchplatform.org
+    FQDN: mo-origin.nationalresearchplatform.org
+    Services:
+      XRootD origin server:
+        Description: MO NRP origin
+    AllowedVOs:
+      - ANY

topology/University of Missouri/Missouri ITRSS/SITE.yaml

@@ -0,0 +1,7 @@
+City: Columbia
+Country: United States
+Description: Missouri ITRSS
+Latitude: 38.9410516354836
+Longitude: -92.32573369440686
+State: MO
+Zipcode: '65211'

topology/University of Texas Arlington/SWT2 ATLAS UTA/SWT2_CPB.yaml

@@ -34,7 +34,7 @@ Resources:
     VOOwnership:
       ATLAS: 100
     WLCGInformation:
-      APELNormalFactor: 10.43
+      APELNormalFactor: 12.52
       AccountingName: US-SWT2
       HEPSPEC: 102816
       InteropAccounting: true
@@ -73,7 +73,7 @@ Resources:
     VOOwnership:
       ATLAS: 100
     WLCGInformation:
-      APELNormalFactor: 10.43
+      APELNormalFactor: 12.52
       AccountingName: US-SWT2
       HEPSPEC: 102816
       InteropAccounting: false

topology/University of Wisconsin/CHTC/CHTC.yaml

@@ -45,7 +45,7 @@ Resources:
     VOOwnership:
       GLOW: 100
   CHTC-submit-1:
-    Active: true
+    Active: false
     ContactLists:
       Administrative Contact:
         Primary:
@@ -68,7 +68,7 @@ Resources:
     VOOwnership:
       GLOW: 100
   CHTC-submit2:
-    Active: true
+    Active: false
     ContactLists:
       Administrative Contact:
         Primary:
@@ -612,4 +612,52 @@ Resources:
     Tags:
       - OSPool
 
+  CHTC-ap2001:
+    Active: true
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          ID: OSG1000015
+          Name: Aaron Moate
+      Security Contact:
+        Primary:
+          ID: OSG1000015
+          Name: Aaron Moate
+    Description: CHTC Access Point
+    FQDN: ap2001.chtc.wisc.edu
+    ID: 10370
+    Services:
+      Submit Node:
+        Description: OS Pool access point
+        Details:
+          hidden: false
+    Tags:
+      - OSPool
+    VOOwnership:
+      GLOW: 100
+
+  CHTC-ap2002:
+    Active: true
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          ID: OSG1000015
+          Name: Aaron Moate
+      Security Contact:
+        Primary:
+          ID: OSG1000015
+          Name: Aaron Moate
+    Description: CHTC Access Point
+    FQDN: ap2002.chtc.wisc.edu
+    ID: 10371
+    Services:
+      Submit Node:
+        Description: OS Pool access point
+        Details:
+          hidden: false
+    Tags:
+      - OSPool
+    VOOwnership:
+      GLOW: 100
+
 SupportCenter: GLOW-TECH