Update to FATE1.7 support (#495)

* docker-compose deploy support FATE v1.7.0

Signed-off-by: ChenLong Ma <owlet42@126.com>

* k8s deploy support FATE v1.7.0

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update doc of docker-compose:
 1. dsl v1-> dsl v2
 2. fate_flow_client.py -> flow

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update docker-compose deploy for support FATE-Serving.

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update docker-build of client

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update example of client

Signed-off-by: ChenLong Ma <owlet42@126.com>

* fix

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update docker-compose。yaml forclient

Signed-off-by: ChenLong Ma <owlet42@126.com>

* fix test for toy,
update fateflow log dir path

Signed-off-by: ChenLong Ma <owlet42@126.com>

* fix example for FATE v1.7.0

Signed-off-by: ChenLong Ma <owlet42@126.com>

* fix fateflow logs path

Signed-off-by: ChenLong Ma <owlet42@126.com>

* fix pipeline init

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update example for notebook

Signed-off-by: ChenLong Ma <owlet42@126.com>

* update docker-compose ci test

Signed-off-by: ChenLong Ma <owlet42@126.com>

* fix spark requests,and example

Signed-off-by: ChenLong Ma <owlet42@126.com>

build/ci/docker-deploy/docker_deploy.sh

@@ -2,9 +2,7 @@
 set -e
 dir=$(dirname $0)
 
-CONTAINER_NUM=10
-
-EXPECT_PYTHON_STATUS=' * Running on http://192.167.0.100:9380/ (Press CTRL+C to quit)'
+CONTAINER_NUM=13
 
 echo "# config prepare"
 target_dir=/data/projects/fate
@@ -27,7 +25,7 @@ docker-compose down
 docker volume rm -f confs-${target_party_id}_shared_dir_examples
 docker volume rm -f confs-${target_party_id}_shared_dir_federatedml
 # exclude client service to save time !
-docker-compose up -d python nodemanager clustermanager mysql rollsite fateboard
+docker-compose up -d
 
 cd ../
 rm -f confs-${target_party_id}.tar
@@ -44,16 +42,20 @@ rm -f serving-${target_party_id}.tar
 echo "# party ${target_party_id} serving cluster deploy is ok!"
 echo "# check containers"
 
-MAX_TRY=10
+MAX_TRY=12
 for ((i = 1; i <= MAX_TRY; i++)); do
     result=$(docker ps | wc -l)
     if [ "${result}" -eq ${CONTAINER_NUM} ]; then
         echo "# containers are ok"
-        python_status=$(docker logs confs-10000_python_1 --tail 1 2>&1)
-        echo "${python_status}"
-        if [ "${python_status}" = "${EXPECT_PYTHON_STATUS}" ]; then
-            exit 0
+        FATE_FLOW_STATUS=$(curl -s -X POST localhost:9380/v1/version/get)
+        success='"retmsg":"success"'
+        result=$(echo $FATE_FLOW_STATUS | grep "${success}")
+        if [[ "$result" != "" ]]
+        then
+          echo ${FATE_FLOW_STATUS}
+          exit 0
         fi
+        echo "FATEFLOW STATUS: ${FATE_FLOW_STATUS}, want ${success}"
     fi
     echo "# Currently have containers: ${result} want ${CONTAINER_NUM}"
     sleep 3

build/ci/docker-deploy/generate_config.sh

@@ -13,7 +13,7 @@ sed -i "s/serving_ip_list=(192.168.1.1 192.168.1.2)/serving_ip_list=(${host_ip})
 
 # Replace tag to latest
 # TODO should replace the serving as well
-sed -i "s/^TAG=.*/TAG=latest/g" .env
+# sed -i "s/^TAG=.*/TAG=latest/g" .env
 echo "# config prepare is ok"
 
 echo "# generate config"

build/ci/docker-deploy/test_cluster.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
 echo "# toy_example test"
-docker exec confs-10000_python_1 bash -c 'sed -i s/\"partition\":\ 48/\"partition\":\ 4/g /data/projects/fate/examples/toy_example/toy_example_conf.json'
-docker exec confs-10000_python_1 bash -c 'python /data/projects/fate/examples/toy_example/run_toy_example.py 10000 10000 1'
+docker exec confs-10000_client_1 bash -c 'flow test toy --guest-party-id 10000 --host-party-id 10000'
 echo "# test is ok!"

docker-build/client/Dockerfile

@@ -4,9 +4,16 @@ FROM ${SOURCE_PREFIX}/python:${SOURCE_TAG} as data
 
 FROM python:3.7
 
-COPY Examples /Examples
+COPY pipeline /data/projects/fate/pipeline
 RUN pip install notebook fate-client pandas sklearn
-RUN mkdir /Examples/Pipeline/logs
-copy --from=data /data/projects/fate/examples /Examples/examples
+RUN mkdir /data/projects/fate/logs
+COPY --from=data /data/projects/fate/examples /data/projects/fate/examples
+COPY --from=data /data/projects/fate/fateflow/examples /data/projects/fate/fateflow/examples
 
-CMD flow init -c /data/projects/fate/conf/service_conf.yaml && pipeline init -c /data/projects/fate/conf/pipeline_conf.yaml && jupyter notebook --ip=0.0.0.0 --port=20000 --allow-root --debug --NotebookApp.notebook_dir='/Examples' --no-browser --NotebookApp.token='' --NotebookApp.password=''
+
+WORKDIR /data/projects/fate/
+
+ENV FATE_FLOW_IP=fateflow
+ENV FATE_FLOW_PORT=9380
+
+CMD flow init --ip ${FATE_FLOW_IP} --port ${FATE_FLOW_PORT} && pipeline init --ip ${FATE_FLOW_IP} --port ${FATE_FLOW_PORT} && jupyter notebook --ip=0.0.0.0 --port=20000 --allow-root --debug --NotebookApp.notebook_dir='/data/projects/fate/' --no-browser --NotebookApp.token='' --NotebookApp.password=''