Merge branch 'master' of https://github.com/mao3267/flyte into add-da…

…taproxy-dataclass-variablemap

.github/PULL_REQUEST_TEMPLATE.md

@@ -33,6 +33,17 @@ If tests were added, say they were added here. Please make sure to add some test
 If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
 If tests were not added, please describe why they were not added and/or why it was difficult to add.
 -->
+### Labels
+
+Please add one or more of the following labels to categorize your PR:
+- **added**: For new features.
+- **changed**: For changes in existing functionality.
+- **deprecated**: For soon-to-be-removed features.
+- **removed**: For features being removed.
+- **fixed**: For any bug fixed.
+- **security**: In case of vulnerabilities
+
+This is important to improve the readability of release notes.
 
 ### Setup process
 

charts/flyte-binary/README.md

@@ -91,6 +91,7 @@ Chart for basic single Flyte executable deployment
 | deployment.extraVolumes | list | `[]` |  |
 | deployment.genAdminAuthSecret.args | list | `[]` |  |
 | deployment.genAdminAuthSecret.command | list | `[]` |  |
+| deployment.genAdminAuthSecret.securityContext | object | `{}` |  |
 | deployment.image.pullPolicy | string | `"IfNotPresent"` |  |
 | deployment.image.repository | string | `"cr.flyte.org/flyteorg/flyte-binary"` |  |
 | deployment.image.tag | string | `"latest"` |  |
@@ -105,13 +106,15 @@ Chart for basic single Flyte executable deployment
 | deployment.podSecurityContext.runAsGroup | int | `65534` |  |
 | deployment.podSecurityContext.runAsUser | int | `65534` |  |
 | deployment.readinessProbe | object | `{}` |  |
+| deployment.securityContext | object | `{}` |  |
 | deployment.sidecars | list | `[]` |  |
 | deployment.startupProbe | object | `{}` |  |
 | deployment.waitForDB.args | list | `[]` |  |
 | deployment.waitForDB.command | list | `[]` |  |
 | deployment.waitForDB.image.pullPolicy | string | `"IfNotPresent"` |  |
 | deployment.waitForDB.image.repository | string | `"postgres"` |  |
 | deployment.waitForDB.image.tag | string | `"15-alpine"` |  |
+| deployment.waitForDB.securityContext | object | `{}` |  |
 | enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service","echo"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |
 | enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service","echo"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) |
 | enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array","agent-service","echo"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins |

charts/flyte-binary/templates/deployment.yaml

@@ -94,6 +94,9 @@ spec:
           {{- if .Values.deployment.resources }}
           resources: {{- toYaml .Values.deployment.resources | nindent 12 }}
           {{- end }}
+          {{- if .Values.deployment.waitForDB.securityContext }}
+          securityContext: {{- toYaml .Values.deployment.waitForDB.securityContext | nindent 12 }}
+          {{- end }}
         {{- end }}
         {{- if .Values.configuration.auth.enabled }}
         - name: gen-admin-auth-secret
@@ -131,6 +134,9 @@ spec:
           {{- if .Values.deployment.resources }}
           resources: {{- toYaml .Values.deployment.resources | nindent 12 }}
           {{- end }}
+          {{- if .Values.deployment.genAdminAuthSecret.securityContext }}
+          securityContext: {{- toYaml .Values.deployment.genAdminAuthSecret.securityContext | nindent 12 }}
+          {{- end }}
         {{- end }}
         {{- if .Values.deployment.initContainers }}
         {{- tpl ( .Values.deployment.initContainers | toYaml ) . | nindent 8 }}
@@ -224,6 +230,9 @@ spec:
             {{- if .Values.deployment.extraVolumeMounts }}
             {{- tpl ( .Values.deployment.extraVolumeMounts | toYaml ) . | nindent 12 }}
             {{- end }}
+          {{- if .Values.deployment.securityContext }}
+          securityContext: {{- toYaml .Values.deployment.securityContext | nindent 12 }}
+          {{- end }}
         {{- if .Values.deployment.sidecars }}
         {{- tpl ( .Values.deployment.sidecars | toYaml ) . | nindent 8 }}
         {{- end }}

charts/flyte-binary/values.yaml

@@ -275,12 +275,16 @@ deployment:
     command: []
     # args Override default init container args
     args: []
+    # securityContext Specify security context for wait-for-db init container
+    securityContext: {}
   # genAdminAuthSecret Configure init container to generate secrets for internal use
   genAdminAuthSecret:
     # command Override default init container command
     command: []
     # args Override default init container args
     args: []
+    # securityContext Specify security context for gen-admin-auth-secret init container
+    securityContext: {}
   # labels Add labels to Flyte deployment
   labels: {}
   # annotations Add annotations to Flyte deployment
@@ -300,6 +304,8 @@ deployment:
   # extraPodSpec Specify additional configuration for Flyte pod
   # This can be used for adding affinity, tolerations, hostNetwork, etc.
   extraPodSpec: {}
+  # securityContext Specify security context for Flyte container
+  securityContext: {}
 
 # service Configure service for Flyte
 service:

datacatalog/go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flyteorg/stow v0.3.10 // indirect
+	github.com/flyteorg/stow v0.3.11 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect

datacatalog/go.sum

@@ -109,8 +109,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8=
-github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM=
+github.com/flyteorg/stow v0.3.11 h1:Uf4fzVbghCqMNvx50XvYzwdNeQDBSKQJ7zddWu7p3eI=
+github.com/flyteorg/stow v0.3.11/go.mod h1:nyaBf8ZWkpHWkKIl4rqKI2uXfPx+VbL0PmEtvq4Pxkc=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=

docker/sandbox-bundled/manifests/complete-agent.yaml

@@ -823,7 +823,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: ZVg2MDdXYnJyNXlYUWlLSA==
+  haSharedSecret: UnZJZHEzUExzbkJsOW1wYw==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1420,7 +1420,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 4d048d7850540d6862f070a28041c7c814d011e537872aecce2159789db2db79
+        checksum/secret: ce172103045f4215e361b4c109776a78fe06660a4ade01c7351ea07212e7cfb9
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/complete.yaml

@@ -805,7 +805,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: VXRnT3hJaE9CMEZldDJubg==
+  haSharedSecret: dDFiem04NjFzb29ZWHFtNA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1369,7 +1369,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 8a6224d331e9ccebb789367315d7f9677d9f4c7886640c8baeea547432cca5ad
+        checksum/secret: 529d34a9c4d3c82b9eec5028fcc30f26e923fa77a57eb29c4705d28c85355963
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/dev.yaml

@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: bnFsU3VrQTFsRmd0azdvcQ==
+  haSharedSecret: Y1V1RU03eGVhUDFFc1pSdQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 492f9045abb64dc1e4f8fa90bb957960ea3ccd8d03b424a3a2fa2a83bfb87531
+        checksum/secret: 66507f448be8010226a1ad2c741fb2866ef4372b68e61287c7500b47fae05572
       labels:
         app: docker-registry
         release: flyte-sandbox

docs/community/contribute/contribute_code.rst

@@ -9,6 +9,9 @@ Contributing code
 
 To understand how the below components interact with each other, refer to :ref:`Understand the lifecycle of a workflow <workflow-lifecycle>`.
 
+.. note::
+    With the exception of ``flytekit``, the below components are maintained in the `flyte <https://github.com/flyteorg/flyte>`__ monorepo.
+
 .. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/contribution_guide/dependency_graph.png
     :alt: Dependency graph between various flyteorg repos
     :align: center
@@ -25,39 +28,6 @@ To understand how the below components interact with each other, refer to :ref:`
     * - **Purpose**: Deployment, Documentation, and Issues
     * - **Languages**: RST
 
-In the ``flyteorg/flyte`` root directory you can run ``make dev-docs`` to build the documentation locally. The generated documentation will be in the ``docs/_build/html`` directory.
-
-**Setup process**
-
-1. First you need to make sure you can run linux/amd64 container
-2. Run the following commands to build the documentation and serve it locally
-
-.. prompt:: bash $
-
- make dev-docs
- python -m http.server --directory docs/_build/html
-
-3. Go to http://localhost:8000 to see the documentation.
-
-**Supported environment variables of** ``make dev-docs``
-
-* ``DEV_DOCS_WATCH``: If set, the docs will be built and served using `sphinx-autobuild <https://github.com/sphinx-doc/sphinx-autobuild>`__ for live updates.
-* ``FLYTEKIT_LOCAL_PATH``: If set, the local path to flytekit will be used instead of the source code from the ``flyteorg/flytekit repo``.
-* ``FLYTECTL_LOCAL_PATH``: If set, the local path to flytectl will be used instead of the source code from the ``flyteorg/flytectl repo``.
-* ``FLYTESNACKS_LOCAL_PATH``: If set, the local path to flytesnacks will be used instead of the source code from the ``flyteorg/flytesnacks`` repo.
-
-For example, to use the local flytekit source code instead of the source code from the ``flyteorg/flytekit`` repo, run ``export FLYTEKIT_LOCAL_PATH=/path/to/flytekit`` before running ``make dev-docs``.
-
-**Alternative conda setup steps**
-
-* Install ``conda``.
-    *  We recommend Miniconda installed with an `official installer <https://docs.conda.io/projects/miniconda/en/latest/index.html#latest-miniconda-installer-links>`__.
-* Install `conda-lock <https://github.com/conda/conda-lock>`__.
-* In the ``flyteorg/flyte`` root directory run:
-    * ``conda-lock install --name monodocs-env monodocs-environment.lock.yaml``
-    * ``conda activate monodocs-env``
-    * ``pip install ./flyteidl``
-
 ``flyteidl``
 ************
 

docs/community/contribute/contribute_docs.md

@@ -17,7 +17,40 @@ The Flyte documentation comprises the following types:
 * **{ref}`Deployment documentation <deployment>`:** Guidance on deploying and configuring the Flyte backend.
 * **{doc}`API documentation <../../api/index>`:** flytekit, flytectl, and flyteidl documentation.
 
-For minor edits that don't require a local setup, you can edit the page in GitHub page to propose improvements.
+For minor edits that don't require a local setup, you can edit the page in GitHub page to propose improvements.}
+
+In the ``flyteorg/flyte`` root directory you can run ``make dev-docs`` to build the documentation locally. The generated documentation will be in the ``docs/_build/html`` directory.
+
+**Setup process**
+
+1. First you need to make sure you can run linux/amd64 container
+2. From the root of your fork, run the following commands to build the documentation and serve it locally
+
+.. prompt:: bash $
+
+ make dev-docs
+ python -m http.server --directory docs/_build/html
+
+3. Go to http://localhost:8000 to see the documentation.
+
+**Supported environment variables of** ``make dev-docs``
+
+* ``DEV_DOCS_WATCH``: If set, the docs will be built and served using `sphinx-autobuild <https://github.com/sphinx-doc/sphinx-autobuild>`__ for live updates.
+* ``FLYTEKIT_LOCAL_PATH``: If set, the local path to flytekit will be used instead of the source code from the ``flyteorg/flytekit repo``.
+* ``FLYTECTL_LOCAL_PATH``: If set, the local path to flytectl will be used instead of the source code from the ``flyteorg/flytectl repo``.
+* ``FLYTESNACKS_LOCAL_PATH``: If set, the local path to flytesnacks will be used instead of the source code from the ``flyteorg/flytesnacks`` repo.
+
+For example, to use the local flytekit source code instead of the source code from the ``flyteorg/flytekit`` repo, run ``export FLYTEKIT_LOCAL_PATH=/path/to/flytekit`` before running ``make dev-docs``.
+
+**Alternative conda setup steps**
+
+* Install ``conda``.
+    *  We recommend Miniconda installed with an `official installer <https://docs.conda.io/projects/miniconda/en/latest/index.html#latest-miniconda-installer-links>`__.
+* Install `conda-lock <https://github.com/conda/conda-lock>`__.
+* In the ``flyteorg/flyte`` root directory run:
+    * ``conda-lock install --name monodocs-env monodocs-environment.lock.yaml``
+    * ``conda activate monodocs-env``
+    * ``pip install ./flyteidl``
 
 ## Contributing to user guide and deployment documentation
 

docs/community/contribute/index.rst

@@ -7,7 +7,6 @@ Contributing to Flyte
 .. tags:: Contribute, Basic
 
 Thank you for taking the time to contribute to Flyte!
-Please read our `Code of Conduct <https://lfprojects.org/policies/code-of-conduct/>`__ before contributing to Flyte.
 
 Here are some guidelines for you to follow, which will make your first and follow-up contributions easier.
 
@@ -18,24 +17,42 @@ TL;DR: Find the repo-specific contribution guidelines in the :ref:`Component Ref
 
 An issue tagged with `good first issue <https://github.com/flyteorg/flyte/labels/good%20first%20issue>`__ is the best place to start for first-time contributors.
 
-**Appetizer for every repo: Fork and clone the concerned repository. Create a new branch on your fork and make the required changes. Create a pull request once your work is ready for review.**
+**Fork and clone the concerned repository. Create a new branch on your fork and make the required changes. Create a pull request once your work is ready for review.**
 
 .. note::
     To open a pull request, refer to `GitHub's guide <https://guides.github.com/activities/forking/>`__ for detailed instructions.
 
 Example PR for your reference: `GitHub PR <https://github.com/flyteorg/flytepropeller/pull/242>`__.
-A couple of checks are introduced to help maintain the robustness of the project.
+Several checks are introduced to help maintain the robustness of the project:
 
 #. To get through DCO, sign off on every commit (`Reference <https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md>`__)
-#. To improve code coverage, write unit tests to test your code
-#. Make sure all the tests pass. If you face any issues, please let us know
+#. To improve code coverage, write unit tests to test your code.
+#. Make sure all the tests pass. If you face any issues, please let us know in the `#contribute <https://flyte-org.slack.com/archives/C04NJPLRWUX>`__ channel
+#. Format your Go code with ``golangci-lint`` followed by ``goimports`` (use ``make lint`` and ``make goimports``)
+#. Format your Python code with ``black`` and ``isort`` (use ``make fmt``).
+#. If make targets are not available, you can manually format the code.
 
-On a side note, format your Go code with ``golangci-lint`` followed by ``goimports`` (use ``make lint`` and ``make goimports``), and Python code with ``black`` and ``isort`` (use ``make fmt``).
-If make targets are not available, you can manually format the code.
-Refer to `Effective Go <https://golang.org/doc/effective_go>`__, `Black <https://github.com/psf/black>`__, and `Isort <https://github.com/PyCQA/isort>`__ for full coding standards.
+.. note::
+    Refer to `Effective Go <https://golang.org/doc/effective_go>`__, `Black <https://github.com/psf/black>`__, and `Isort <https://github.com/PyCQA/isort>`__ for full coding standards.
+
+As you become more involved with the project, you may be able to be added as a committer to the repos you're working on. Checkout the `Flyte Contributor Ladder <https://github.com/flyteorg/community/blob/main/GOVERNANCE.md#community-roles-and-path-to-maintainership>`__ to learn more. 
+
+Before submitting your PR
+**************************
+
+We strongly encourage you to add one of these labels to your Pull Request:
 
-As you become more involved with the project, you may be able to be added as a contributor to the repos you're working on,
-but there is a medium term effort to move all development to forks.
+- **added**: For new features.
+- **changed**: For changes in existing functionality.
+- **deprecated**: For soon-to-be-removed features.
+- **removed**: For features being removed.
+- **fixed**: For any bug fixes.
+- **security**: In case of vulnerabilities
+
+This is helpful to build human-readable release notes. `Learn more <https://keepachangelog.com/en/1.1.0/>`__
+
+.. note::
+    Learn how to apply a label to a PR in the `Github docs <https://docs.github.com/en/issues/using-labels-and-milestones-to-track-work/managing-labels#applying-a-label>`__.
 
 🐞 File an issue
 ================
@@ -53,7 +70,7 @@ We use `GitHub Issues <https://github.com/flyteorg/flyte/issues>`__ for issue tr
 If none of the above fit your requirements, file a `blank <https://github.com/flyteorg/flyte/issues/new>`__ issue.
 Also, add relevant labels to your issue. For example, if you are filing a Flytekit plugin request, add the ``flytekit`` label.
 
-For feedback at any point in the contribution process, feel free to reach out to us on `Slack <https://slack.flyte.org/>`__.
+For feedback at any point in the contribution process, feel free to reach out to us on `Slack <https://flyte-org.slack.com/archives/C04NJPLRWUX>`__.
 
 .. toctree::
     :maxdepth: 1