Added CI to create PRs to bump to latest Ark timestamps

.github/workflows/update-timestamps.yml

@@ -0,0 +1,28 @@
+name: Check for new Release Train snapshots
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 7 * * *'  # Run at 7am on default branch
+
+jobs:
+  upstream_check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup environment
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+
+      - name: Check for updated Ark timestamps and replace in defaults.yml
+        run: ansible-playbook ansible/ci/update_timestamps.yml
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          add-paths: environments/common/inventory/group_vars/all/defaults.yml
+          commit-message: "[automated] bumped repo timestamps to latest"
+          branch: auto/bump-timestamps
+          title: "[Automation] Bumped repo timestamps to latest"
+          body: Automated changes by .github/workflows/update-timestamps.yml

ansible/ci/library/latest_timestamps.py

@@ -0,0 +1,88 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2018, Terry Jones <terry.jones@example.org>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: latest_timestamps
+
+short_description: Gets the latest set of snapshots from Pulp and overwrites
+
+version_added: "1.0.0"
+
+description: Gets the latest set of snapshots from given source URLs and returns dictionary in to replace 'appliances_repo_timestamps' with
+
+author:
+    - William Tripp
+'''
+
+EXAMPLES = r'''
+# Pass in a message
+- name: Get latest timestamps
+  latest_timestamps:
+    current_timestamps_dict: "{{ appliances_repo_timestamps }}"
+    sources_dict: "{{ appliances_repo_timestamp_sources }}"
+  register: result
+
+'''
+
+RETURN = r'''
+# These are examples of possible return values, and in general should use other names for return values.
+latest_dict:
+    description: Dictionary with updated timestamps
+    type: dict
+    returned: always
+changed_timestamps:
+    description: List of repos that've been updated
+    type: str[]
+    returned: always
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+import requests
+from bs4 import BeautifulSoup
+
+def run_module():
+    module_args = dict(
+        current_timestamps_dict=dict(type='dict', required=True),
+        sources_dict=dict(type='dict', required=True)
+    )
+
+    result = dict(
+        changed=False,
+        original_message='',
+        message=''
+    )
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    latest_timestamps = {}
+    changed_timestamps = []
+    for repo in module.params['sources_dict']:
+        latest_timestamps[repo] = module.params['sources_dict'][repo]
+        for version in module.params['sources_dict'][repo]:
+            html_txt = requests.get(url=module.params['sources_dict'][repo][version]).text
+            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all()
+            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # xml tags
+            latest_timestamps[repo][version] = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+            if module.params['sources_dict'][repo][version] != module.params['current_timestamps_dict'][repo][version]:
+                changed_timestamps.append(repo+' '+version)
+
+    result['latest_dict'] = latest_timestamps
+    result['changed_timestamps'] = changed_timestamps
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

ansible/ci/update_timestamps.yml

@@ -0,0 +1,28 @@
+- hosts: localhost
+  tasks:
+  - name: Check sources are defined for all repos
+    ansible.builtin.assert:
+      that: appliances_repo_timestamps[item].keys() == appliances_repo_timestamp_sources[item].keys()
+      fail_msg: "'appliances_repo_timestamps.{{ item }}' is missing a source definition in 'appliances_repo_timestamp_sources'"
+    loop: "{{ appliances_repo_timestamps.keys() }}"
+
+  - name: Get latest timestamps from sources
+    latest_timestamps:
+      sources_dict: "{{ appliances_repo_timestamp_sources }}"
+      current_timestamps_dict: "{{ appliances_repo_timestamps }}"
+    register: _result
+
+  - name: Print updated timestamps
+    ansible.builtin.debug:
+      var: _result.changed_timestamps
+
+  - name: Overwrite repo timestamps with latest
+    ansible.builtin.blockinfile:
+      path: "{{ playbook_dir }}/../../environments/common/inventory/group_vars/all/defaults.yml"
+      marker: "# {mark} Marker for ansible/ci/update_timestamps.yml (GH workflow managed)"
+      block: |
+          {{ yaml_template | to_nice_yaml(indent=2) }}
+    vars:
+      yaml_template:
+        appliances_repo_timestamps: "{{ _result.latest_dict }}"
+    when: (_result.changed_timestamps | count) > 0

environments/common/inventory/group_vars/all/defaults.yml

@@ -82,6 +82,7 @@ appliances_local_users: "{{ appliances_local_users_default + appliances_local_us
 
 ###########################################################################################
 
+# BEGIN Marker for ansible/ci/update_timestamps.yml (GH workflow managed)
 appliances_repo_timestamps:
   baseos:
     '9.4': 20240816T002610
@@ -93,3 +94,17 @@ appliances_repo_timestamps:
     '9.4': 20240816T002610
   epel:
     '9': 20240902T080424
+# END Marker for ansible/ci/update_timestamps.yml (GH workflow managed)
+
+# For CI
+appliances_repo_timestamp_sources:
+  baseos:
+    '9.4': https://ark.stackhpc.com/pulp/content/rocky/9.4/BaseOS/x86_64/os/
+  appstream:
+    '9.4': https://ark.stackhpc.com/pulp/content/rocky/9.4/AppStream/x86_64/os/
+  crb:
+    '9.4': https://ark.stackhpc.com/pulp/content/rocky/9.4/CRB/x86_64/os/
+  extras:
+    '9.4': https://ark.stackhpc.com/pulp/content/rocky/9.4/extras/x86_64/os/
+  epel:
+    '9': https://ark.stackhpc.com/pulp/content/epel/9/Everything/x86_64/

requirements.txt

@@ -10,3 +10,4 @@ selinux # this is a shim to avoid having to use --system-site-packages, you stil
 netaddr
 matplotlib
 pulp-cli==0.29.2
+beautifulsoup4==4.12.3