From d605511c41904af294cd85738a48d3a37dcd5dbf Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 23 Mar 2022 15:22:12 -0400 Subject: [PATCH] Support S3 backend for Stratum 0/1 --- .gitignore | 1 + defaults/main.yml | 17 +++++++++ tasks/stratum0.yml | 18 ++++++++-- tasks/stratum1.yml | 5 ++- tasks/stratumN.yml | 56 ++++++++++++++++++++++-------- templates/localproxy_squid.conf.j2 | 18 ++++++++++ templates/s3.conf.j2 | 6 ++++ vars/debian.yml | 6 ++-- vars/redhat.yml | 8 ++--- 9 files changed, 110 insertions(+), 25 deletions(-) create mode 100644 .gitignore create mode 100644 templates/localproxy_squid.conf.j2 create mode 100644 templates/s3.conf.j2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1377554 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/defaults/main.yml b/defaults/main.yml index 8f5e63c..7e10ba1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -21,6 +21,23 @@ cvmfs_localproxy_http_ports: cvmfs_stratum1_apache_port: 8008 cvmfs_stratum1_cache_mem: 128 #MB +# Storage backend for Stratum 0/1 servers +# Can be "disk" (/srv) or "s3" (see: https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#s3-compatible-storage-systems) +cvmfs_storage: disk +# The contents of the S3 config file passed as the -s option to cvmfs_server mkfs/add-replica. Dictionary keys are +# option names (e.g. CVMFS_S3_HOST) and values are the option values +cvmfs_s3_config: {} + +# Use POSIX ACLs to allow access to s3.conf by repository owners (access is required if any owners are non-root), if you +# don't want to use ACLs, the cvmfs_s3_config_mode and cvmfs_s3_config_group variables can be used (you are responsible +# for setting up and controlling membership of the group). +cvmfs_set_s3_config_acl: true + +# The -w option to cvmfs_server mkfs/add-replica +cvmfs_s3_url: null +# The actual options added to cvmfs_server mkfs/add-replica +cvmfs_storage_cli_options: "{% if cvmfs_storage == 's3' %}-s /etc/cvmfs/s3.conf -w {{ cvmfs_s3_url }}{% endif %}" + # Whether the client or server should be upgraded or just installed if missing cvmfs_upgrade_client: false cvmfs_upgrade_server: false diff --git a/tasks/stratum0.yml b/tasks/stratum0.yml index 2226033..cf2bb39 100644 --- a/tasks/stratum0.yml +++ b/tasks/stratum0.yml @@ -33,6 +33,7 @@ - name: Include Apache tasks include_tasks: apache.yml + when: "cvmfs_storage == 'disk'" - name: Include firewall tasks include_tasks: firewall.yml @@ -41,15 +42,26 @@ when: cvmfs_manage_firewall - name: Create repositories - command: /usr/bin/cvmfs_server mkfs {{ cvmfs_config_apache_flag }} -o {{ item.owner | default('root') }} -f {{ cvmfs_union_fs }} {{ item.repository }} + command: >- + /usr/bin/cvmfs_server mkfs + {{ cvmfs_storage_cli_options }} + {{ cvmfs_config_apache_flag }} + -o {{ item.owner | default('root') }} + -f {{ cvmfs_union_fs }} + {{ item.repository }} args: - creates: /srv/cvmfs/{{ item.repository }} + creates: "{{ '/srv/cvmfs/' ~ item.repository when (cvmfs_storage == 'disk') else '/etc/cvmfs/repositories.d/' ~ item.repository }}" with_items: "{{ cvmfs_repositories }}" notify: - restart apache - name: Ensure repositories are imported - command: /usr/bin/cvmfs_server import -r {{ cvmfs_config_apache_flag }} -o {{ item.owner | default('root') }} -f {{ cvmfs_union_fs }} {{ item.repository }} + command: >- + /usr/bin/cvmfs_server import -r + {{ cvmfs_config_apache_flag }} + -o {{ item.owner | default('root') }} + -f {{ cvmfs_union_fs }} + {{ item.repository }} args: creates: /etc/cvmfs/repositories.d/{{ item.repository }} with_items: "{{ cvmfs_repositories }}" diff --git a/tasks/stratum1.yml b/tasks/stratum1.yml index e119385..5e78edd 100644 --- a/tasks/stratum1.yml +++ b/tasks/stratum1.yml @@ -3,7 +3,7 @@ - name: Include initial OS-specific tasks include_tasks: "init_{{ ansible_os_family | lower }}.yml" vars: - _cvmfs_role: stratum1 + _cvmfs_role: "stratum1-{{ cvmfs_storage }}" _cvmfs_upgrade: "{{ cvmfs_upgrade_server }}" - name: Include key setup tasks @@ -24,11 +24,13 @@ - name: Include Apache tasks include_tasks: apache.yml + when: "cvmfs_storage == 'disk'" - name: Include squid tasks include_tasks: squid.yml vars: _cvmfs_squid_conf_src: "{{ cvmfs_squid_conf_src | default('stratum1_squid.conf.j2') }}" + when: "cvmfs_storage == 'disk'" - name: Include firewall tasks include_tasks: firewall.yml @@ -47,6 +49,7 @@ - name: Ensure replicas are configured command: >- /usr/bin/cvmfs_server add-replica -o {{ item.owner | default('root') }} + {{ cvmfs_storage_cli_options }} http://{{ item.stratum0 }}/cvmfs/{{ item.repository }} {{ item.key_dir | default('/etc/cvmfs/keys') }}/{{ item.repository }}.pub args: diff --git a/tasks/stratumN.yml b/tasks/stratumN.yml index 77041dd..4dac20b 100644 --- a/tasks/stratumN.yml +++ b/tasks/stratumN.yml @@ -1,16 +1,44 @@ --- -- name: Create /srv filesystem - filesystem: - dev: "{{ cvmfs_srv_device }}" - force: no - fstype: "{{ cvmfs_srv_fstype | default('ext4') }}" - when: cvmfs_srv_device is defined - -- name: Mount /srv - mount: - name: "{{ cvmfs_srv_mount }}" - src: "{{ cvmfs_srv_device }}" - fstype: "{{ cvmfs_srv_fstype | default('ext4') }}" - state: mounted - when: cvmfs_srv_device is defined +- name: Disk storage tasks + block: + + - name: Create /srv filesystem + filesystem: + dev: "{{ cvmfs_srv_device }}" + force: no + fstype: "{{ cvmfs_srv_fstype | default('ext4') }}" + + - name: Mount /srv + mount: + name: "{{ cvmfs_srv_mount }}" + src: "{{ cvmfs_srv_device }}" + fstype: "{{ cvmfs_srv_fstype | default('ext4') }}" + state: mounted + + when: cvmfs_storage == 'disk' and cvmfs_srv_device is defined + +- name: S3 storage tasks + block: + + - name: Create s3.conf + template: + src: s3.conf.j2 + dest: /etc/cvmfs/s3.conf + mode: "{{ cvmfs_s3_config_mode | default('0600') }}" + owner: root + group: "{{ cvmfs_s3_config_group | default('root') }}" + + - name: Set ACL for CVMFS repository owners to access s3.conf + acl: + path: /etc/cvmfs/s3.conf + entity: "{{ item.owner }}" + etype: user + permissions: r + state: present + loop: "{{ cvmfs_repositories }}" + when: "cvmfs_set_s3_config_acl and item.owner != 'root'" + loop_control: + label: "{{ item.repository }}: {{ item.owner }}" + + when: cvmfs_storage == 's3' diff --git a/templates/localproxy_squid.conf.j2 b/templates/localproxy_squid.conf.j2 new file mode 100644 index 0000000..2610de4 --- /dev/null +++ b/templates/localproxy_squid.conf.j2 @@ -0,0 +1,18 @@ +## +## This file is managed by Ansible. ALL CHANGES WILL BE OVERWRITTEN. +## + +http_port 3128 accel +http_access allow all + +#acl localnet src 10.0.0.0/8 +always_direct allow all + +cache_mem {{ cvmfs_localproxy_cache_mem }} MB + +minimum_expiry_time 0 +# This is for the disk cache +#maximum_object_size 1024 MB +maximum_object_size_in_memory {{ cvmfs_localproxy_maximum_object_size_in_memory }} MB + +#visible_hostname {{ inventory_hostname }} diff --git a/templates/s3.conf.j2 b/templates/s3.conf.j2 new file mode 100644 index 0000000..af518ef --- /dev/null +++ b/templates/s3.conf.j2 @@ -0,0 +1,6 @@ +# +# This file is managed by Ansible. ALL CHANGES WILL BE OVERWRITTEN. +# +{% for opt in (cvmfs_s3_config | dict2items) %} +{{ opt.key }}={{ opt.value }} +{% endfor %} diff --git a/vars/debian.yml b/vars/debian.yml index c26c7d6..cb5f273 100644 --- a/vars/debian.yml +++ b/vars/debian.yml @@ -12,11 +12,11 @@ cvmfs_packages: stratum0: - apache2 - cvmfs-server - - cvmfs-config-default - stratum1: + stratum1-disk: - apache2 - cvmfs-server - - cvmfs-config-default + stratum1-s3: + - cvmfs-server localproxy: - squid client: diff --git a/vars/redhat.yml b/vars/redhat.yml index fa30f12..3277daa 100644 --- a/vars/redhat.yml +++ b/vars/redhat.yml @@ -12,14 +12,14 @@ cvmfs_packages: stratum0: - httpd - cvmfs-server - - cvmfs-config-default - cvmfs - stratum1: + stratum1-disk: - httpd - - mod_wsgi + - python3-mod_wsgi - squid - cvmfs-server - - cvmfs-config-default + stratum1-s3: + - cvmfs-server localproxy: - squid client: