From 0e4d1137036d1ada95a48162e4e678da89aa30d2 Mon Sep 17 00:00:00 2001 From: ChenLong Ma Date: Fri, 2 Apr 2021 19:24:43 +0800 Subject: [PATCH] Fix bugs (#392) * fix helm chart of exchange fix CI task Signed-off-by: ChenLong Ma * fix pulsar port of docker-compose Signed-off-by: ChenLong Ma --- build/ci/ansible/group_vars/all | 2 + .../ci/ansible/roles/kubefate/tasks/main.yml | 128 ++++++++++-------- .../docker-compose-spark.yml | 2 +- helm-charts/FATE-Exchange/partyId-values.yaml | 18 ++- .../FATE-Exchange/templates/_helpers.tpl | 16 +++ .../templates/rollsite-module.yaml | 21 ++- .../values-template-example.yaml | 12 ++ .../FATE-Exchange/values-template.yaml | 19 ++- helm-charts/FATE-Exchange/values.yaml | 8 ++ 9 files changed, 162 insertions(+), 64 deletions(-) create mode 100644 helm-charts/FATE-Exchange/templates/_helpers.tpl diff --git a/build/ci/ansible/group_vars/all b/build/ci/ansible/group_vars/all index b8b171395..c6c72aab0 100644 --- a/build/ci/ansible/group_vars/all +++ b/build/ci/ansible/group_vars/all @@ -16,3 +16,5 @@ docker_registry: docker.io kube_create_timeout: 1800 check_interval: 60 fate_create_timeout: 1800 +retries: 3 +delay: 5 \ No newline at end of file diff --git a/build/ci/ansible/roles/kubefate/tasks/main.yml b/build/ci/ansible/roles/kubefate/tasks/main.yml index efb641f04..95f2a67f3 100644 --- a/build/ci/ansible/roles/kubefate/tasks/main.yml +++ b/build/ci/ansible/roles/kubefate/tasks/main.yml @@ -12,61 +12,61 @@ - name: Wait until lock file has been deleted or timeout block: - - name: Shell to Get Epoch - command: date +%s - register: ansible_date_time_epoch - - debug: - msg: "{{ ansible_date_time_epoch.stdout }}" - - - name: Get info on lock_dir - stat: - path: "{{ lock_dir }}" - register: lock_dir_cfg - - debug: - msg: "{{ lock_dir_cfg.stat.mtime }}" - - - name: Get Epochs Info - block: - - name: Module to Get Epoch - command: date +%s - register: ansible_date_time_epoch - - debug: - msg: "{{ ansible_date_time_epoch.stdout }}" - - - name: Get info on lock_dir - stat: - path: "{{ lock_dir }}" - register: lock_dir_cfg - - debug: - msg: "{{ lock_dir_cfg.stat.mtime }}" - - - name: Debug date comparison - debug: - msg: "wait_for timeout={{ (state_timeout - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime))|int }}" - - - name: wait for lock file to be absent - wait_for: - path: "{{ lock_dir }}" - state: absent - timeout: "{{ (state_timeout - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime))|int }}" - ignore_errors: true - register: waitfor - when: - - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime) > 0 - - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime) < state_timeout - - - name: Remove lock file lock_dir - block: - - name: Remove lock file lock_dir - file: - path: "{{ lock_dir }}" - state: absent - - - name: Sleep for {{ wait_processes_rm_lock_file }} seconds for all waiting processes to make the file absent, May need to increase this - wait_for: - delay: "{{ wait_processes_rm_lock_file }}" - timeout: 0 - when: (waitfor.skipped is defined and waitfor.skipped) or (waitfor.state is defined and waitfor.state!="absent") + - name: Shell to Get Epoch + command: date +%s + register: ansible_date_time_epoch + - debug: + msg: "{{ ansible_date_time_epoch.stdout }}" + + - name: Get info on lock_dir + stat: + path: "{{ lock_dir }}" + register: lock_dir_cfg + - debug: + msg: "{{ lock_dir_cfg.stat.mtime }}" + + - name: Get Epochs Info + block: + - name: Module to Get Epoch + command: date +%s + register: ansible_date_time_epoch + - debug: + msg: "{{ ansible_date_time_epoch.stdout }}" + + - name: Get info on lock_dir + stat: + path: "{{ lock_dir }}" + register: lock_dir_cfg + - debug: + msg: "{{ lock_dir_cfg.stat.mtime }}" + + - name: Debug date comparison + debug: + msg: "wait_for timeout={{ (state_timeout - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime))|int }}" + + - name: wait for lock file to be absent + wait_for: + path: "{{ lock_dir }}" + state: absent + timeout: "{{ (state_timeout - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime))|int }}" + ignore_errors: true + register: waitfor + when: + - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime) > 0 + - (ansible_date_time_epoch.stdout|float - lock_dir_cfg.stat.mtime) < state_timeout + + - name: Remove lock file lock_dir + block: + - name: Remove lock file lock_dir + file: + path: "{{ lock_dir }}" + state: absent + + - name: Sleep for {{ wait_processes_rm_lock_file }} seconds for all waiting processes to make the file absent, May need to increase this + wait_for: + delay: "{{ wait_processes_rm_lock_file }}" + timeout: 0 + when: (waitfor.skipped is defined and waitfor.skipped) or (waitfor.state is defined and waitfor.state!="absent") when: lockcode.rc|int==1 - name: Start kubefate test @@ -89,39 +89,55 @@ async: "{{ kube_create_timeout }}" poll: "{{ check_interval }}" shell: "bash {{ work_dir }}/utils/create_k8s.sh" + retries: "{{ retries }}" + delay: "{{ delay }}" - name: Pull images shell: "bash {{ work_dir }}/get_images.sh" + retries: "{{ retries }}" + delay: "{{ delay }}" - name: kubefate build async: "{{ kube_create_timeout }}" poll: "{{ check_interval }}" shell: "bash {{ work_dir }}/utils/kubefate_build.sh" + retries: "{{ retries }}" + delay: "{{ delay }}" - name: deploy kubefate async: "{{ kube_create_timeout }}" poll: "{{ check_interval }}" shell: "bash {{ work_dir }}/kubefate-e2e/kubefate_deploy_test.sh" + retries: "{{ retries }}" + delay: "{{ delay }}" - name: check fate curd async: "{{ kube_create_timeout }}" poll: "{{ check_interval }}" shell: "bash {{ work_dir }}/kubefate-e2e/fate_deploy_test.sh" - + retries: "{{ retries }}" + delay: "{{ delay }}" + - name: Running deployment script. async: "{{ fate_create_timeout }}" poll: "{{ check_interval }}" shell: "bash {{ work_dir }}/kubefate-e2e/deploy.sh" + retries: "{{ retries }}" + delay: "{{ delay }}" always: - name: Clean up k8s environment shell: "bash {{ work_dir }}/clean.sh" + retries: "{{ retries }}" + delay: "{{ delay }}" always: - name: Clean up the project files of the target machine file: path: "{{ base_dir }}" state: absent + retries: "{{ retries }}" + delay: "{{ delay }}" always: - name: Remove lock file lock_dir diff --git a/docker-deploy/training_template/docker-compose-spark.yml b/docker-deploy/training_template/docker-compose-spark.yml index ba9443f51..43bd410db 100644 --- a/docker-deploy/training_template/docker-compose-spark.yml +++ b/docker-deploy/training_template/docker-compose-spark.yml @@ -147,7 +147,7 @@ services: ports: - "6650:6650" - "6651:6651" - - "8000:8080" + - "8001:8080" deploy: restart_policy: condition: any diff --git a/helm-charts/FATE-Exchange/partyId-values.yaml b/helm-charts/FATE-Exchange/partyId-values.yaml index fdaebfbfd..bf7bf245d 100644 --- a/helm-charts/FATE-Exchange/partyId-values.yaml +++ b/helm-charts/FATE-Exchange/partyId-values.yaml @@ -9,14 +9,28 @@ image: imagePullSecrets: # - name: +exchange: + partyIp: 192.168.0.1 + partyPort: 30000 + +partyList: + - partyId: 10000 + partyIp: 192.168.0.1 + partyPort: 30101 + modules: rollsite: include: false + type: NodePort + nodePort: 30091 + partyList: + - partyId: 10000 + partyIp: 192.168.0.1 + partyPort: 30101 trafficServer: include: true type: ClusterIP - httpNodePort: 30097 - grpcNodePort: 30092 + nodePort: 30097 route_table: sni: - fqdn: 10000.fate.org diff --git a/helm-charts/FATE-Exchange/templates/_helpers.tpl b/helm-charts/FATE-Exchange/templates/_helpers.tpl new file mode 100644 index 000000000..11b7e2614 --- /dev/null +++ b/helm-charts/FATE-Exchange/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* Helm required labels */}} +{{- define "fate.labels" -}} +name: {{ .Values.partyName | quote }} +partyId: {{ .Values.partyId | quote }} +owner: kubefate +cluster: fate-exchange +heritage: {{ .Release.Service }} +release: {{ .Release.Name }} +chart: {{ .Chart.Name }} +{{- end -}} + +{{/* matchLabels */}} +{{- define "fate.matchLabels" -}} +name: {{ .Values.partyName | quote }} +partyId: {{ .Values.partyId | quote }} +{{- end -}} diff --git a/helm-charts/FATE-Exchange/templates/rollsite-module.yaml b/helm-charts/FATE-Exchange/templates/rollsite-module.yaml index 385be7aa4..154eb9f2a 100644 --- a/helm-charts/FATE-Exchange/templates/rollsite-module.yaml +++ b/helm-charts/FATE-Exchange/templates/rollsite-module.yaml @@ -18,6 +18,22 @@ metadata: fateMoudle: rollsite {{ include "fate.labels" . | indent 4 }} data: + eggroll.properties: | + # rollsite + eggroll.rollsite.coordinator={{ .Values.partyName }} + eggroll.rollsite.host=rollsite + eggroll.rollsite.port=9370 + eggroll.rollsite.party.id={{ .Values.partyId }} + eggroll.rollsite.route.table.path=conf/route_table/route_table.json + eggroll.rollsite.route.table.key= + eggroll.rollsite.route.table.whitelist=127.0.0.1 + eggroll.rollsite.jvm.options=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:logs/eggroll/rollsite.gc.log + + eggroll.rollsite.push.max.retry=3 + eggroll.rollsite.push.long.retry=2 + eggroll.rollsite.push.batches.per.stream=10 + eggroll.rollsite.adapter.sendbuf.size=100000 + route_table.json: | { "route_table": { @@ -115,7 +131,7 @@ spec: - mountPath: /data/projects/fate/eggroll/conf/route_table/ name: rollsite-confs - mountPath: /data/projects/fate/eggroll/conf/eggroll.properties - name: eggroll-confs + name: rollsite-confs subPath: eggroll.properties {{- with .Values.modules.rollsite.nodeSelector }} nodeSelector: @@ -130,9 +146,6 @@ spec: - name: rollsite-confs configMap: name: rollsite-config - - name: eggroll-confs - configMap: - name: eggroll-config --- apiVersion: v1 kind: Service diff --git a/helm-charts/FATE-Exchange/values-template-example.yaml b/helm-charts/FATE-Exchange/values-template-example.yaml index 2af3e4f8b..ab28796d2 100644 --- a/helm-charts/FATE-Exchange/values-template-example.yaml +++ b/helm-charts/FATE-Exchange/values-template-example.yaml @@ -12,6 +12,18 @@ istio: modules: - trafficServer +rollsite: + type: NodePort + nodePort: 30091 + exchange: + ip: 192.168.0.1 + port: 30000 + partyList: + - partyId: 10000 + partyIp: 192.168.0.1 + partyPort: 30101 + nodeSelector: + trafficServer: type: ClusterIP nodePort: 30097 diff --git a/helm-charts/FATE-Exchange/values-template.yaml b/helm-charts/FATE-Exchange/values-template.yaml index 95c1303cb..6849ee0af 100644 --- a/helm-charts/FATE-Exchange/values-template.yaml +++ b/helm-charts/FATE-Exchange/values-template.yaml @@ -9,10 +9,27 @@ image: imagePullSecrets: # - name: +exchange: +{{- with .rollsite }} +{{- with .exchange }} + partyIp: {{ .ip }} + partyPort: {{ .port }} +{{- end }} +{{- end }} + +partyList: +{{- with .rollsite }} +{{- range .partyList }} + - partyId: {{ .partyId }} + partyIp: {{ .partyIp }} + partyPort: {{ .partyPort }} +{{- end }} +{{- end }} + modules: rollsite: include: {{ has "rollsite" .modules }} - {{- with .rabbitmq }} + {{- with .rollsite }} {{- with .nodeSelector }} nodeSelector: {{ toYaml . | indent 6 }} diff --git a/helm-charts/FATE-Exchange/values.yaml b/helm-charts/FATE-Exchange/values.yaml index 39194b1d9..562a0504c 100644 --- a/helm-charts/FATE-Exchange/values.yaml +++ b/helm-charts/FATE-Exchange/values.yaml @@ -9,6 +9,14 @@ image: partyId: 9999 partyName: fate-9999 +partyList: +- partyId: 8888 + partyIp: 192.168.8.1 + partyPort: 30081 +- partyId: 10000 + partyIp: 192.168.10.1 + partyPort: 30101 + modules: rollsite: include: false