diff --git a/.github/workflows/deploy_gcp.yml b/.github/workflows/deploy_gcp.yml new file mode 100644 index 0000000..cc42385 --- /dev/null +++ b/.github/workflows/deploy_gcp.yml @@ -0,0 +1,87 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# The full GCP authentication setup is described in this nice blog post: https://cloud.google.com/blog/products/identity-security/enabling-keyless-authentication-from-github-actions +# Here you see how to push to google cloud artifact registry: https://roger-that-dev.medium.com/push-code-with-github-actions-to-google-clouds-artifact-registry-60d256f8072f +# Here you see how to deploy a cloudrun: https://github.com/google-github-actions/deploy-cloudrun +# Or more generically, setup gcloud: https://github.com/google-github-actions/setup-gcloud +name: GCP docker build + + +on: + push: + branches: [ deployment/dev, deployment/prd ] +# pull_request: +# branches: [ master ] + +#virtualenv venv +#source venv/bin/activate +#pip install -r ./requirements.txt +# +#gcloud auth configure-docker europe-west1-docker.pkg.dev +#docker build -t europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app . +#OR +#docker buildx build --platform linux/arm64/v8,linux/amd64 -t europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app . +#docker tag europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app:test1 +#docker push europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app:test1 +#docker run -it -e PORT=8080 -p 8080:8080 --rm europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app:test +## Cloudrun deployment +# gcloud run deploy tom-toolkit-instance-dev-b614bde8 --image europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app:test1 --update-labels ^,^managed-by=manual_deploy,commit-sha=XXXXXXXXXXXXXXX --format json --region europe-west1 --project tom-toolkit-dev-hxm +# gcloud run services proxy tom-toolkit-instance-dev-b614bde8 --port=8080 --project=tom-toolkit-dev-hxm --region=europe-west1 +# cloud-sql-proxy --auto-iam-authn tom-toolkit-dev-hxm:europe-west1:tom-toolkit-instance-dev-ae78f371 +#Then go to http://localhost:8080 + + +#env: +# IMAGE_NAME: '' +# PROJECT_ID: '' +# AR_REPO_LOCATION: '' +# AR_URL: '' +# SERVICE_ACCOUNT: '' +# WORKLOAD_IDENTITY_PROVIDER: '' + +jobs: + push_push_deploy: + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + runs-on: ubuntu-latest + environment: |- + ${{ github.ref_name == 'deployment/dev' && 'dev' + || github.ref_name == 'deployment/prd' && 'prd' }} + env: + GOOGLE_CLOUD_PROJECT: ${{ vars.PROJECT_ID }} + DEBUG: ${{ github.ref_name == 'deployment/dev' && 'True' }} + steps: +# - shell: bash +# run: | +# echo "YOUR WIP is ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }} " + - uses: 'actions/checkout@v4' + - id: 'auth' # The exact debug procedure is described here: https://cloud.google.com/iam/docs/audit-logging/examples-workload-identity#exchange-federated + uses: 'google-github-actions/auth@v2' # https://github.com/google-github-actions/auth + with: + token_format: access_token + project_id: ${{ vars.PROJECT_ID }} + workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.SERVICE_ACCOUNT }} + - id: 'docker-auth' + uses: 'docker/login-action@v3' + with: + username: 'oauth2accesstoken' + password: '${{ steps.auth.outputs.access_token }}' + registry: '${{ vars.AR_REPO_LOCATION }}-docker.pkg.dev' +# - id: 'Set up Cloud SDK' +# uses: 'google-github-actions/setup-gcloud@v2' +# with: +# version: '>= 363.0.0' + - id: 'get_gcp_client' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 416.0.0' + - id: 'build_and_push' + run: |- + mkdir -p ./astrometry_data/ && gsutil -m cp -n gs://astrometry_data/* ./astrometry_data/ + docker buildx build --platform linux/amd64 -t ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }} . + docker tag ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }} ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }}:${{ github.sha }} + docker push ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }}:${{ github.sha }} + docker tag ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }} ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }}:latest + docker push ${{ vars.AR_URL }}/${{ vars.IMAGE_NAME }}:latest diff --git a/Dockerfile b/Dockerfile index 07eeffd..6cf5e8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,10 @@ ENV PYINDI_VERSION v1.9.1 ENV INDI_3RD_PARTY_VERSION v2.0.8 ENV PYTHON_VERSION 3.12.3 +# Actual application code and configs (could be used in builds) +RUN mkdir -p /opt/remote_observatory/astrometry_data +COPY . /opt/remote_observatory/ + # Generic install / utilities / dev RUN apt-get update && apt-get --assume-yes --quiet install --no-install-recommends \ build-essential \ @@ -103,13 +107,13 @@ RUN apt-get --assume-yes --quiet install --no-install-recommends \ # && wget --recursive --no-parent --no-host-directories --cut-dirs=6 --accept "*.fits" --continue --directory-prefix=/usr/local/astrometry/data/ https://portal.nersc.gov/project/cosmo/temp/dstn/index-5200/LITE/ -# Downloading gcloud package -RUN curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=/opt/gcloud --disable-prompts -ENV PATH $PATH:/opt/gcloud/google-cloud-sdk/bin - -# Actual application code and configs (could be used in builds) -RUN mkdir -p /opt/remote_observatory -COPY . /opt/remote_observatory/ +# Now Download astrometry.net index files -- This needs to be moved when gsutil is updated +# RUN pyenv install 3.11 \ +# && pyenv global 3.11 \ +# && gsutil -m cp gs://astrometry_data/* /usr/local/astrometry/data/ \ +# && pyenv global $PYTHON_VERSION +#RUN mv /opt/remote_observatory/astrometry_data/* /usr/local/astrometry/data/ +RUN find /opt/remote_observatory/astrometry_data/ -maxdepth 1 -type f -exec mv '{}' /usr/local/astrometry/data/ \; ## Indi dependencies for pre-packages binaries #RUN apt-add-repository ppa:mutlaqja/ppa && apt-get --assume-yes --quiet install --no-install-recommends \ @@ -221,6 +225,10 @@ RUN apt-get --assume-yes --quiet install --no-install-recommends \ texlive-bibtex-extra \ texlive-science +# Downloading gcloud client +RUN curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=/opt/gcloud --disable-prompts +ENV PATH $PATH:/opt/gcloud/google-cloud-sdk/bin + # Using bash for lower level scripting from now-on SHELL ["/bin/bash", "-l", "-c"] RUN echo 'export PS1="\u@\h \w> "' | cat - /root/.profile > temp && mv temp /root/.profile @@ -233,12 +241,6 @@ RUN echo 'eval "$(pyenv init -)"' >> /root/.bashrc RUN pyenv install -v $PYTHON_VERSION RUN pyenv global $PYTHON_VERSION -# Now Download astrometry.net index files -- This needs to be moved when gsutil is updated -RUN pyenv install 3.11 \ - && pyenv global 3.11 \ - && gsutil -m cp gs://astrometry_data/* /usr/local/astrometry/data/ \ - && pyenv global $PYTHON_VERSION - # Python virtual environment ENV VIRTUAL_ENV=/opt/remote_observatory_venv RUN python -m venv $VIRTUAL_ENV @@ -291,3 +293,8 @@ RUN chmod 644 /etc/systemd/system/indiwebmanager_science_camera.service # docker buildx build --platform linux/arm64/v8 -t test_to_delete . # docker buildx build --platform linux/amd64 -t test_to_delete . # docker buildx build -t europe-west1-docker.pkg.dev/tom-toolkit-dev-hxm/remote-observatory-tom-repo/tom_app . + +# If you want to debug a layer: +# DOCKER_BUILDKIT=0 docker build --platform linux/arm64/v8 -t test_to_delete . +# docker run -it --rm 1941be9e1d8c /bin/bash +# docker buildx prune # To clean cache \ No newline at end of file