Merge pull request #325 from NVIDIA/branch-23.06

release 23.06

.github/workflows/auto-merge.yml

@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-23.04
+    - branch-23.06
     types: [closed]
 
 jobs:
@@ -29,14 +29,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: branch-23.04 # force to fetch from latest upstream instead of PR ref
+          ref: branch-23.06 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids-ml
-          HEAD: branch-23.04
-          BASE: branch-23.06
+          HEAD: branch-23.06
+          BASE: branch-23.08
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR
 

.github/workflows/blossom-ci.yml

@@ -46,6 +46,7 @@ jobs:
       NvTimLiu,\
       YanxuanLiu,\
       zhanga5,\
+      rishic3,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

ci/Dockerfile

@@ -35,7 +35,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linu
     && conda init
 
 # install cuML
-ARG CUML_VER=23.04
+ARG CUML_VER=23.06
 RUN conda install -c conda-forge mamba && \
-    mamba install -y -c rapidsai -c nvidia -c conda-forge cuml=$CUML_VER python=3.8 cuda-toolkit=11.5 \
+    mamba install -y -c rapidsai -c nvidia -c conda-forge cuml=$CUML_VER python=3.9 cuda-toolkit=11.5 \
     && mamba clean --all -f -y

ci/Jenkinsfile.premerge

@@ -132,7 +132,7 @@ pipeline {
 
                     stash(name: "source_tree", includes: "**")
 
-                    container('docker-build') {
+                    container('cpu') {
                         // check if pre-merge dockerfile modified
                         def dockerfileModified = sh(returnStdout: true,
                             script: 'BASE=$(git --no-pager log --oneline -1 | awk \'{ print $NF }\'); ' +
@@ -145,7 +145,7 @@ pipeline {
                         if (TEMP_IMAGE_BUILD) {
                             PREMERGE_TAG = "${BUILD_TAG}"
                             IMAGE_PREMERGE = "${ARTIFACTORY_NAME}/sw-spark-docker-local/spark-rapids-ml:${PREMERGE_TAG}"
-                            docker.build(IMAGE_PREMERGE, "-f ${PREMERGE_DOCKERFILE} -t $IMAGE_PREMERGE .")
+                            docker.build(IMAGE_PREMERGE, "--network=host -f ${PREMERGE_DOCKERFILE} -t $IMAGE_PREMERGE .")
                             uploadDocker(IMAGE_PREMERGE)
                         }
                     }

docker/Dockerfile.pip

@@ -18,7 +18,7 @@ ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
 
 ARG PYSPARK_VERSION=3.3.1
-ARG RAPIDS_VERSION=23.4.0
+ARG RAPIDS_VERSION=23.6.0
 
 # Install packages to build spark-rapids-ml
 RUN apt-get update -y \

docker/Dockerfile.python

@@ -17,7 +17,7 @@
 ARG CUDA_VERSION=11.5.2
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
 
-ARG CUML_VERSION=23.04
+ARG CUML_VERSION=23.06
 
 # Install packages to build spark-rapids-ml
 RUN apt update -y \
@@ -38,7 +38,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linu
 
 # install cuML
 
-RUN conda install -y -c rapidsai -c nvidia -c conda-forge python=3.8 cuda-toolkit=11.5 cuml=$CUML_VERSION \
+RUN conda install -y -c rapidsai -c nvidia -c conda-forge python=3.9 cuda-toolkit=11.5 cuml=$CUML_VERSION \
     && conda clean --all -f -y
 
 # install python dependencies

docs/Makefile

@@ -14,6 +14,10 @@ help:
 
 .PHONY: help Makefile
 
+clean: Makefile
+	rm -rf $(BUILDDIR)
+	rm -rf $(SOURCEDIR)/api
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile

docs/site/configuration.md

@@ -0,0 +1,12 @@
+---
+title: Configuration
+nav_order: 6
+---
+# Configuration
+
+The following configurations can be supplied as Spark properties.
+
+| Property name   | Default | Meaning  |
+| :-------------- | :------ | :------- |
+| spark.rapids.ml.uvm.enabled | false | if set to true, enables [CUDA unified virtual memory](https://developer.nvidia.com/blog/unified-memory-cuda-beginners/) (aka managed memory) during estimator.fit() operations to allow processing of larger datasets than would fit in GPU memory|
+

docs/site/get-started/dataproc.md

@@ -0,0 +1,8 @@
+---
+title: Dataproc
+parent: Getting Started
+---
+# Getting Started on Google Dataproc
+
+See [these instructions](https://github.com/NVIDIA/spark-rapids-ml/blob/main/notebooks/dataproc/README.md)
+

docs/site/get-started/emr.md

@@ -0,0 +1,8 @@
+---
+title: EMR
+parent: Getting Started
+---
+# Getting Started on AWS EMR
+
+See [these instructions](https://github.com/NVIDIA/spark-rapids-ml/blob/main/notebooks/aws-emr/README.md)
+

docs/source/_templates/autosummary/class.rst

@@ -0,0 +1,38 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. Workaround to avoid documenting __init__.
+
+{% extends "!autosummary/class.rst" %}
+
+{% if '__init__' in methods %}
+{% set caught_result = methods.remove('__init__') %}
+{% endif %}
+
+{% block methods %}
+{% if methods %}
+
+   .. rubric:: Methods
+
+   .. autosummary::
+      {% for item in methods %}
+      ~{{ name }}.{{ item }}
+      {%- endfor %}
+
+{% endif %}
+{% endblock %}
+

docs/source/_templates/autosummary/class_with_docs.rst

@@ -0,0 +1,84 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+{{ objname }}
+{{ underline }}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+    :no-private-members:
+
+    {% if '__init__' in methods %}
+      {% set caught_result = methods.remove('__init__') %}
+    {% endif %}
+
+    {% block methods_summary %}
+    {% if methods %}
+
+    .. rubric:: Methods
+
+    .. autosummary::
+    {% for item in methods %}
+       ~{{ name }}.{{ item }}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
+    {% block attributes_summary %}
+    {% if attributes %}
+
+    .. rubric:: Attributes
+
+    .. autosummary::
+    {% for item in attributes %}
+    {%- if item != "uid" %}
+       ~{{ name }}.{{ item }}
+    {%- endif %}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
+    {% block methods_documentation %}
+    {% if methods %}
+
+    .. rubric:: Methods Documentation
+
+    {% for item in methods %}
+    .. automethod:: {{ item }}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+
+    {% block attributes_documentation %}
+    {% if attributes %}
+
+    .. rubric:: Attributes Documentation
+
+    {% for item in attributes %}
+    {%- if item != "uid" %}
+    .. autoattribute:: {{ item }}
+    {%- endif %}
+    {%- endfor %}
+
+    {% endif %}
+    {% endblock %}
+

docs/source/conf.py

@@ -9,11 +9,12 @@
 project = 'spark-rapids-ml'
 copyright = '2023, NVIDIA'
 author = 'NVIDIA'
-release = '23.4.0'
+release = '23.6.0'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
+
 extensions = [
     'numpydoc',
     'sphinx.ext.autodoc',
@@ -23,6 +24,10 @@
     'sphinx.ext.intersphinx',
 ]
 
+numpydoc_show_class_members = False
+
+autodoc_inherit_docstrings = False
+
 templates_path = ['_templates']
 exclude_patterns = []
 
@@ -36,8 +41,29 @@
 
 html_theme = 'pydata_sphinx_theme'
 
-html_static_path = ['_static']
+import inspect
+from spark_rapids_ml.utils import _unsupported_methods_attributes
+
+_unsupported_by_class = {}
+def autodoc_skip_member(app, what, name, obj, skip, options):
+    # adapted from this https://github.com/sphinx-doc/sphinx/issues/9533#issuecomment-962007846
+    doc_class=None
+    for frame in inspect.stack():
+        if frame.function == "get_members":
+            doc_class = frame.frame.f_locals["obj"]
+            break
+
+    exclude = skip
+    if doc_class:
+        if doc_class not in _unsupported_by_class:
+            _unsupported_by_class[doc_class] = _unsupported_methods_attributes(doc_class)
+
+        exclude = name in _unsupported_by_class[doc_class]
+
+    # return True if (skip or exclude) else None  # Can interfere with subsequent skip functions.
+    return True if exclude or skip else None
 
 def setup(app):
     app.add_css_file("https://docs.rapids.ai/assets/css/custom.css")
     app.add_js_file("https://docs.rapids.ai/assets/js/custom.js", loading_method="defer")
+    app.connect('autodoc-skip-member', autodoc_skip_member)

docs/source/index.rst

@@ -4,18 +4,16 @@
    contain the root `toctree` directive.
 
 Welcome to spark-rapids-ml's documentation!
-=====================================
-This is the auto-generated documentation!!
+===========================================
 
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
 
-
+   spark_rapids_ml
 
 Indices and tables
 ==================
 
 * :ref:`genindex`
-* :ref:`modindex`
 * :ref:`search`