Merge branch 'master' into fix1

CHANGELOG/CHANGELOG-v1.2.0.md

@@ -0,0 +1,42 @@
+# Changelog over Greenland(v1.1.0) release
+
+## [v1.2.0-rc2](https://github.com/sodafoundation/installer/tree/v1.2.0-rc2) (2021-01-06)
+
+[Full Changelog](https://github.com/sodafoundation/installer/compare/v1.2.0-rc1...v1.2.0-rc2)
+
+**Closed issues:**
+
+- Dashboard installation fails when opensds\_auth\_strategy is set to noauth [\#419](https://github.com/sodafoundation/installer/issues/419)
+- Uninstalll of SODA fails because it cannot find srm-toolchain installation [\#417](https://github.com/sodafoundation/installer/issues/417)
+- Installation fails at Delfin installer with error "Could not get output from /usr/local/bin/virtualenv --help: AttributeError: module 'os' has no attribute 'PathLike'\n" [\#405](https://github.com/sodafoundation/installer/issues/405)
+
+**Merged pull requests:**
+
+- Added Hawaii RC2 release tag [\#421](https://github.com/sodafoundation/installer/pull/421) ([kumarashit](https://github.com/kumarashit))
+- Fixed dashboard installation when auth strategy is noauth [\#420](https://github.com/sodafoundation/installer/pull/420) ([anvithks](https://github.com/anvithks))
+- Fixed issue with uninstall SRM toolchain when not exists [\#418](https://github.com/sodafoundation/installer/pull/418) ([anvithks](https://github.com/anvithks))
+- Fix delfin OS version for 18.04 & 16.04 [\#372](https://github.com/sodafoundation/installer/pull/372) ([joseph-v](https://github.com/joseph-v))
+
+## [v1.2.0-rc1](https://github.com/sodafoundation/installer/tree/v1.2.0-rc1) (2020-12-24)
+
+[Full Changelog](https://github.com/sodafoundation/installer/compare/v1.1.0...v1.2.0-rc1)
+
+**Closed issues:**
+
+- \[Delfin Performance Monitoring\] Install Prometheus and Grafana and integrate with Delfin for metric collection and visualisation [\#406](https://github.com/sodafoundation/installer/issues/406)
+- Extend SODA installer to install Example Use cases  [\#404](https://github.com/sodafoundation/installer/issues/404)
+- Single Command Install for SODA with default configs. [\#403](https://github.com/sodafoundation/installer/issues/403)
+- SODA ansible script retry script fails [\#402](https://github.com/sodafoundation/installer/issues/402)
+- bug\(installer\): Ansible failures on Ubuntu 18.04 [\#393](https://github.com/sodafoundation/installer/issues/393)
+- Incorrect dashboard realease tag/version in dashboard.yml for v1.0.0 [\#371](https://github.com/sodafoundation/installer/issues/371)
+- Nodeplugin dns policy should be added [\#72](https://github.com/sodafoundation/installer/issues/72)
+
+**Merged pull requests:**
+
+- Adding the Hawaii RC1 release tags. [\#416](https://github.com/sodafoundation/installer/pull/416) ([kumarashit](https://github.com/kumarashit))
+- Support Delfin performance exporters and alert exporter configs [\#415](https://github.com/sodafoundation/installer/pull/415) ([joseph-v](https://github.com/joseph-v))
+- Added SRM Toolchain install and remove [\#413](https://github.com/sodafoundation/installer/pull/413) ([anvithks](https://github.com/anvithks))
+
+
+
+\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*

ansible/clean.yml

@@ -29,6 +29,7 @@
     - group_vars/dashboard.yml
     - group_vars/orchestration.yml
     - group_vars/delfin.yml
+    - group_vars/srm-toolchain.yml
   gather_facts: false
   become: True
   tasks:

ansible/group_vars/common.yml

@@ -34,7 +34,7 @@ deploy_project: all
 install_from: release
 # These fields below will specify the tag based on install_from type
 repo_branch: master
-release_version: v1.1.0
+release_version: v1.2.0
 
 # This field indicates which os family the system will be running, currently
 # support 'Debian' and 'RedHat'
@@ -49,3 +49,15 @@ database_purge: true
 
 # OpenSDS configuration file
 opensds_conf_file: /etc/opensds/opensds.conf
+
+###############
+# Respective projects release versions
+
+api_release_version: v1.0.0
+controller_release_version: v1.0.0
+dock_release_version: v1.2.0
+dashboard_release_version: v1.2.0
+gelato_release_version: v1.2.0
+sushi_release_version: v1.2.0
+orchestration_release_version: v0.12.0
+delfin_release_version: v1.1.0

ansible/group_vars/dashboard.yml

@@ -36,7 +36,7 @@ console_login_url: "{{ host_ip }}:8088"
 ##########
 
 
-dashboard_docker_image: sodafoundation/dashboard:{{ release_version }}
+dashboard_docker_image: sodafoundation/dashboard:{{ dashboard_release_version }}
 
 
 

ansible/group_vars/delfin.yml

@@ -26,7 +26,7 @@ dummy:
 enable_delfin: false
 
 # delfin installation types are: 'repository', 'release' and 'container''
-delfin_installation_type: "repository" # "{{ install_from }}"
+delfin_installation_type: "{{ install_from }}"
 
 # These fields are NOT suggested to be modifie
 delfin_work_dir: /opt/delfin-linux-amd64
@@ -36,6 +36,25 @@ venv: "{{ delfin_work_dir }}/venv"
 delfin_redis_ip: 127.0.0.1
 delfin_redis_port: 26379
 
+# Exporter configurations for Kafka, Prometheus & Alert Manager
+# Uncomment exporters to enable
+performance_exporters: #PerformanceExporterPrometheus, PerformanceExporterKafka
+alert_exporters: #AlertExporterPrometheus
+
+# Exporter configurations for Kafka
+delfin_exporter_kafka_ip: 'localhost'
+delfin_exporter_kafka_port: 9092
+delfin_exporter_kafka_topic: 'delfin-kafka'
+
+# Exporter configurations for Prometheus
+delfin_exporter_prometheus_ip: 0.0.0.0
+delfin_exporter_prometheus_port: 8195
+delfin_exporter_prometheus_cache_file: '/var/lib/delfin/delfin_exporter.txt'
+
+# Exporter configurations for Alert Manager
+delfin_exporter_alertmanager_host: 'localhost'
+delfin_exporter_alertmanager_port: 9093
+
 ##############
 # REPOSITORY #
 ##############
@@ -52,7 +71,7 @@ delfin_remote_url: https://github.com/sodafoundation/delfin.git
 ###########
 
 # If user specifies intalling from release,then he can choose the specific version
-delfin_release: "v1.0.0"
+delfin_release: "{{ delfin_release_version }}"
 
 # These fields are NOT suggested to be modified
 delfin_download_url: https://github.com/sodafoundation/delfin/archive/{{ delfin_release }}.tar.gz

ansible/group_vars/gelato.yml

@@ -27,7 +27,7 @@ gelato_work_dir: /opt/opensds-gelato-linux-amd64
 
 # The installation type of gelato is container, both REPOSITORY and RELEASE will
 # use gelato image to deploy.
-gelato_image_tag: "{{ release_version }}"
+gelato_image_tag: "{{ gelato_release_version }}"
 
 
 ##############
@@ -47,7 +47,7 @@ gelato_remote_url: https://github.com/sodafoundation/multi-cloud.git
 ###########
 
 # If user specifies intalling from release,then he can choose the specific version
-gelato_release: "{{ release_version }}"	
+gelato_release: "{{ gelato_release_version }}"	
 
 # These fields are NOT suggested to be modified
 gelato_download_url: https://github.com/sodafoundation/multi-cloud/archive/{{ gelato_release }}.tar.gz

ansible/group_vars/hotpot.yml

@@ -57,9 +57,9 @@ dock_remote_url: https://github.com/sodafoundation/dock.git
 ###########
 
 # If user specifies intalling from release,then he can choose the specific version
-api_release: "v1.0.0" # The version should be at least v0.12.0
-controller_release: "v1.0.0" # The version should be at least v0.12.0
-dock_release: "{{ release_version }}" # The version should be at least v0.12.0
+api_release: "{{ api_release_version }}" # The version should be at least v0.12.0
+controller_release: "{{ controller_release_version }}" # The version should be at least v0.12.0
+dock_release: "{{ dock_release_version }}" # The version should be at least v0.12.0
 
 # These fields are NOT suggested to be modified
 api_download_url: https://github.com/sodafoundation/api/releases/download/{{ api_release }}/soda-api-{{ api_release }}-linux-amd64.tar.gz
@@ -75,11 +75,7 @@ dock_tarball_dir: /tmp/soda-dock-{{ dock_release }}-linux-amd64
 # CONTAINER #
 #############
 
-api_image_tag: "v1.0.0"
-controller_image_tag: "v1.0.0"
-dock_image_tag: "{{ release_version }}"
-
 # These fields are NOT suggested to be modified
-apiserver_docker_image: sodafoundation/apiserver:{{ api_image_tag }}
-controller_docker_image: sodafoundation/controller:{{ controller_image_tag }}
-dock_docker_image: sodafoundation/dock:{{ dock_image_tag }}
+apiserver_docker_image: sodafoundation/apiserver:{{ api_release }}
+controller_docker_image: sodafoundation/controller:{{ controller_release }}
+dock_docker_image: sodafoundation/dock:{{ dock_release }}

ansible/group_vars/orchestration.yml

@@ -65,7 +65,7 @@ orchestration_st2_installer_remote_url: https://github.com/StackStorm/st2-docker
 ###########
 
 # If user specifies intalling from release,then he can choose the specific version
-orchestration_release:  v0.12.0 # "{{ release_version }}" The version should be greater than v0.5.4
+orchestration_release: "{{ orchestration_release_version }}" #The version should be greater than v0.5.4
 
 # These fields are NOT suggested to be modified
 orchestration_download_url: https://github.com/sodafoundation/orchestration/archive/{{ orchestration_release }}.tar.gz

ansible/group_vars/srm-toolchain.yml

@@ -0,0 +1,46 @@
+# Copyright 2020 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+#########################
+# INSTALL SRM TOOLCHAIN #
+#########################
+# The SRM Toolchain is required if you wish to use the SODA Dashboard to view the metrics and visualization. 
+# If you set this to true then the installer will install Prometheus:<version>, AlertManager<version> and Grafana<version>. 
+# Please note that this installer will install the above as docker containers. 
+# If you already have any of the above running then please make the appropriate 
+# changes to the docker container name and ports at {srm_toolchain_dir}/docker-compose.yml
+# If you set this to false then you will not be able to view the visualization using SODA Dashboard. 
+# You can however configure your own dashboard or receiver to connect with Delfin.
+
+install_srm_toolchain: false
+
+# The source directory where the SRM Toolchain installer files reside
+srm_toolchain_dir: srm-toolchain
+
+# The working directory where the SRM Toolchain installer files are copied.
+srm_toolchain_work_dir: /opt/srm-toolchain
+
+prometheus_image_tag: v2.23.0
+prometheus_port: 9090
+
+alertmanager_image_tag: v0.21.0
+alertmanager_port: 9093
+
+grafana_image_tag: 7.3.5
+grafana_port: 3000

ansible/group_vars/sushi.yml

@@ -50,7 +50,7 @@ sushi_remote_url: https://github.com/opensds/nbp.git
 ###########
 
 # If user specifies intalling from release,then he can choose the specific version
-sushi_release: "{{ release_version }}" # The version should be at least v0.2.1
+sushi_release: "{{ sushi_release_version }}" # The version should be at least v0.2.1
 
 # These fields are NOT suggested to be modified
 sushi_download_url: https://github.com/opensds/nbp/releases/download/{{ sushi_release }}/opensds-sushi-{{ sushi_release }}-linux-amd64.tar.gz

ansible/roles/cleaner/scenarios/srm-toolchain.yml

@@ -0,0 +1,51 @@
+# Copyright 2020 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: check for SRM Toolchain files existed
+  stat:
+    path: "{{ srm_toolchain_work_dir }}"
+  register: srmtoolchainexisted
+
+- name: Stop and remove Prometheus, Alertmanager, Grafana containers but don't delete the images
+  shell: docker-compose rm -fs
+  args:
+    chdir: "{{ srm_toolchain_work_dir }}/"
+  when:
+    - source_purge == false
+    - srmtoolchainexisted.stat.isdir is defined and srmtoolchainexisted.stat.isdir
+
+- name: Stop and remove Prometheus, Alertmanager, Grafana containers & delete the images
+  shell: docker-compose down --rmi all
+  args:
+    chdir: "{{ srm_toolchain_work_dir }}/"
+  when:
+    - source_purge is undefined or source_purge != false
+    - srmtoolchainexisted.stat.isdir is defined and srmtoolchainexisted.stat.isdir
+
+- name: clean up all SRM Toolchain directories
+  file:
+    path: "{{ item }}"
+    state: absent
+    force: yes
+  with_items:
+    - "{{ srm_toolchain_work_dir }}/"
+  ignore_errors: yes
+  tags:
+    - srm_toolchain
+    - clean
+  when:
+    - source_purge is undefined or source_purge != false
+    - srmtoolchainexisted.stat.isdir is defined and srmtoolchainexisted.stat.isdir
+

ansible/roles/cleaner/tasks/main.yml

@@ -125,6 +125,11 @@
   when: enable_delfin == true
   tags: delfin
 
+- name: include scenarios/srm-toolchain.yml for cleaning up SRM Toolchain
+  include_tasks: scenarios/srm-toolchain.yml
+  when: install_srm_toolchain == true
+  tags: srm_toolchain
+
 - name: clean all configuration and log files
   file:
     path: "{{ item }}"

ansible/roles/dashboard-installer/scenarios/container.yml

@@ -32,3 +32,8 @@
       OPENSDS_S3_URL: "http://{{ host_ip }}:8090"
       OPENSDS_S3_HOST: "{{ host_ip }}" 
       OPENSDS_S3_PORT: "8090"
+      SODA_PROMETHEUS_PORT: "{{ prometheus_port }}"
+      SODA_ALERTMANAGER_PORT: "{{ alertmanager_port }}"
+      SODA_GRAFANA_PORT: "{{ grafana_port }}"
+      SODA_ALERTMANAGER_URL: "http://{{ host_ip }}:{{ alertmanager_port }}"
+

ansible/roles/dashboard-installer/tasks/main.yml

@@ -15,11 +15,18 @@
 ---
 - name: use container to install dashboard
   include_tasks: scenarios/container.yml
-  when: dashboard_installation_type == "container"
+  when: 
+    - dashboard_installation_type == "container"
+    - opensds_auth_strategy == "keystone"
 
 - name: use source code to install dashboard
   include_tasks: scenarios/source-code.yml
-  when: dashboard_installation_type == "source_code"
+  when: 
+    - dashboard_installation_type == "source_code"
+    - opensds_auth_strategy == "keystone"
 
 - name: login console
   debug: msg="please use '{{ console_login_url }}' login console"
+  when: 
+    - dashboard_installation_type == "container" or dashboard_installation_type == "source_code"
+    - opensds_auth_strategy == "keystone"

ansible/roles/delfin-installer/scenarios/rabbitmq.yml

@@ -21,40 +21,40 @@
   apt:
     autoclean: yes
 
-- name: Add RabbitMQ official repo
-  apt_repository: 
-    repo: deb https://dl.bintray.com/rabbitmq/debian xenial main 
-    state: present
-    filename: rabbitmq
-  become: yes
-  when:
-    - rabbitmqservice.stat.exists is undefined or rabbitmqservice.stat.exists == false
-
 - name: Import RabbitMQ public key
-  apt_key: 
+  apt_key:
     url: https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc
     state: present
   become: yes
   when:
     - rabbitmqservice.stat.exists is undefined or rabbitmqservice.stat.exists == false
 
-- name: Add Erlang official repo
+- name: Add RabbitMQ official repo
   apt_repository: 
-    repo: deb https://binaries.erlang-solutions.com/debian xenial contrib 
+    repo: deb https://dl.bintray.com/rabbitmq/debian {{ ansible_distribution_release }} main
     state: present
-    filename: erlang
+    filename: rabbitmq
   become: yes
   when:
     - rabbitmqservice.stat.exists is undefined or rabbitmqservice.stat.exists == false
 
 - name: Import Erlang public key
-  apt_key: 
+  apt_key:
     url: https://packages.erlang-solutions.com/debian/erlang_solutions.asc
     state: present
   become: yes
   when:
     - rabbitmqservice.stat.exists is undefined or rabbitmqservice.stat.exists == false
 
+- name: Add Erlang official repo
+  apt_repository:
+    repo: deb https://binaries.erlang-solutions.com/debian {{ ansible_distribution_release }} contrib
+    state: present
+    filename: erlang
+  become: yes
+  when:
+    - rabbitmqservice.stat.exists is undefined or rabbitmqservice.stat.exists == false
+
 - name: Install RabbitMQ package
   apt: 
     name: "{{ item }}"