Skip to content

Commit

Permalink
Rename tensorflow-io-plugin-gs to tensorflow-io-gcs-filesystem (#1419)
Browse files Browse the repository at this point in the history
As was discussed in our SIG IO meeting, this PR renames
 tensorflow-io-plugin-gs to tensorflow-io-gcs-filesystem

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
  • Loading branch information
yongtang authored May 14, 2021
1 parent 87f4825 commit dcb3c1e
Show file tree
Hide file tree
Showing 24 changed files with 65 additions and 65 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/api.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
rm -rf tensorflow_io
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-gcs-filesystem-nightly ; fi
python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
python -m pip freeze
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
Expand Down Expand Up @@ -82,7 +82,7 @@ jobs:
rm -rf tensorflow_io
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-gcs-filesystem-nightly ; fi
python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
python -m pip freeze
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
Expand Down Expand Up @@ -123,7 +123,7 @@ jobs:
rm -rf tensorflow_io
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-gcs-filesystem-nightly ; fi
python -m pip install pytest-benchmark
python -m pip freeze
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ jobs:
python --version
df -h
rm -rf tensorflow_io
rm -rf tensorflow_io_plugin_gs
rm -rf tensorflow_io_gcs_filesystem
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-gcs-filesystem-nightly ; fi
python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
python -m pip freeze
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
python -c 'from tensorflow_io_plugin_gs.core.python.ops import plugin_gs; print(plugin_gs)'
python -c 'from tensorflow_io_gcs_filesystem.core.python.ops import plugin_gs; print(plugin_gs)'
python -m pytest --benchmark-only -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))
linux:
Expand Down Expand Up @@ -69,15 +69,15 @@ jobs:
python --version
df -h
rm -rf tensorflow_io
rm -rf tensorflow_io_plugin_gs
rm -rf tensorflow_io_gcs_filesystem
echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi
if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-gcs-filesystem-nightly ; fi
python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1
python -m pip freeze
python -c 'import tensorflow as tf; print(tf.version.VERSION)'
python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
python -c 'from tensorflow_io_plugin_gs.core.python.ops import plugin_gs; print(plugin_gs)'
python -c 'from tensorflow_io_gcs_filesystem.core.python.ops import plugin_gs; print(plugin_gs)'
python -m pytest --benchmark-only --benchmark-json benchmark.json -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))
- name: Store benchmark result
uses: rhysd/github-action-benchmark@v1
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build.bazel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ bazel build \
--noshow_loading_progress \
--verbose_failures \
--test_output=errors \
-- //tensorflow_io/... //tensorflow_io_plugin_gs/...
-- //tensorflow_io/... //tensorflow_io_gcs_filesystem/...

rm -rf build && mkdir -p build

cp -r bazel-bin/tensorflow_io build/tensorflow_io
cp -r bazel-bin/tensorflow_io_plugin_gs build/tensorflow_io_plugin_gs
cp -r bazel-bin/tensorflow_io_gcs_filesystem build/tensorflow_io_gcs_filesystem

exit 0
2 changes: 1 addition & 1 deletion .github/workflows/build.wheel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ export TF_USE_MODULAR_FILESYSTEM=1
run_test() {
entry=$1
CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))')
(cd wheelhouse && $entry -m pip install tensorflow_io_plugin_gs-*-cp${CPYTHON_VERSION}-*.whl)
(cd wheelhouse && $entry -m pip install tensorflow_io_gcs_filesystem-*-cp${CPYTHON_VERSION}-*.whl)
(cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl)
$entry -m pip install -q pytest pytest-benchmark pytest-xdist boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1 azure-storage-blob==12.8.1
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append --forked --numprocesses=auto --dist loadfile $(find . -type f \( -iname "test_*.py" \)))
Expand Down
60 changes: 30 additions & 30 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,16 +120,16 @@ jobs:
--noshow_loading_progress \
--verbose_failures \
--test_output=errors \
//tensorflow_io/... //tensorflow_io_plugin_gs/...
//tensorflow_io/... //tensorflow_io_gcs_filesystem/...
mkdir build
cp -r bazel-bin/tensorflow_io build
cp -r bazel-bin/tensorflow_io_plugin_gs build
cp -r bazel-bin/tensorflow_io_gcs_filesystem build
- uses: actions/upload-artifact@v2
with:
name: ${{ runner.os }}-bazel-bin
path: |
build/tensorflow_io
build/tensorflow_io_plugin_gs
build/tensorflow_io_gcs_filesystem
macos-wheel:
name: Wheel ${{ matrix.python }} macOS
Expand All @@ -153,7 +153,7 @@ jobs:
python -m pip install -U wheel setuptools
python --version
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64
python setup.py --project tensorflow-io-gcs-filesystem --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64
- name: Auditwheel ${{ matrix.python }} macOS
run: |
set -x -e
Expand Down Expand Up @@ -201,7 +201,7 @@ jobs:
set -x -e
python --version
df -h
(cd wheelhouse && python -m pip install tensorflow_io_plugin_gs-*.whl)
(cd wheelhouse && python -m pip install tensorflow_io_gcs_filesystem-*.whl)
(cd wheelhouse && python -m pip install tensorflow_io-*.whl)
- name: Test ${{ matrix.python }} macOS
run: |
Expand Down Expand Up @@ -232,14 +232,14 @@ jobs:
gcr.io/tensorflow-testing/nosla-ubuntu16.04-manylinux2010@sha256:3a9b4820021801b1fa7d0592c1738483ac7abc209fc6ee8c9ef06cf2eab2d170 /v/.github/workflows/build.bazel.sh
sudo chown -R $(id -nu):$(id -ng) .
sudo find build/tensorflow_io -name '*runfiles*' | sudo xargs rm -rf
sudo find build/tensorflow_io_plugin_gs -name '*runfiles*' | sudo xargs rm -rf
sudo find build/tensorflow_io_gcs_filesystem -name '*runfiles*' | sudo xargs rm -rf
sudo cp .bazelrc build/tensorflow_io/
- uses: actions/upload-artifact@v2
with:
name: ${{ runner.os }}-bazel-bin
path: |
build/tensorflow_io
build/tensorflow_io_plugin_gs
build/tensorflow_io_gcs_filesystem
linux-wheel:
name: Wheel ${{ matrix.python }} Linux
Expand All @@ -259,7 +259,7 @@ jobs:
set -x -e
mv bazel-bin/tensorflow_io/.bazelrc .
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io-gcs-filesystem --data bazel-bin -q bdist_wheel
- name: Auditwheel ${{ matrix.python }} Linux
run: |
set -x -e
Expand Down Expand Up @@ -348,16 +348,16 @@ jobs:
python3 setup.py --install-require | xargs python3 -m pip install
python3 tools/build/configure.py
cat .bazelrc
bazel build -s --verbose_failures %BAZEL_OPTIMIZATION% //tensorflow_io/core:python/ops/libtensorflow_io.so //tensorflow_io/core:python/ops/libtensorflow_io_plugins.so //tensorflow_io_plugin_gs/...
bazel build -s --verbose_failures %BAZEL_OPTIMIZATION% //tensorflow_io/core:python/ops/libtensorflow_io.so //tensorflow_io/core:python/ops/libtensorflow_io_plugins.so //tensorflow_io_gcs_filesystem/...
mkdir build
cp -r bazel-bin/tensorflow_io build
cp -r bazel-bin/tensorflow_io_plugin_gs build
cp -r bazel-bin/tensorflow_io_gcs_filesystem build
- uses: actions/upload-artifact@v2
with:
name: ${{ runner.os }}-bazel-bin
path: |
build/tensorflow_io
build/tensorflow_io_plugin_gs
build/tensorflow_io_gcs_filesystem
windows-wheel:
name: Wheel ${{ matrix.python }} Windows
Expand All @@ -382,7 +382,7 @@ jobs:
python --version
python -m pip install -U wheel setuptools
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel
python setup.py --project tensorflow-io-gcs-filesystem --data bazel-bin -q bdist_wheel
ls -la dist
- uses: actions/upload-artifact@v2
with:
Expand Down Expand Up @@ -418,15 +418,15 @@ jobs:
run: |
@echo on
python --version
(cd wheel && ls tensorflow_io_plugin_gs-*.whl | xargs python -m pip install && cd ..)
(cd wheel && ls tensorflow_io_gcs_filesystem-*.whl | xargs python -m pip install && cd ..)
(cd wheel && ls tensorflow_io-*.whl | xargs python -m pip install && cd ..)
- name: Test ${{ matrix.python }} Windows
shell: cmd
run: |
@echo on
python --version
python -m pip install -U pytest-benchmark
rm -rf tensorflow_io tensorflow_io_plugin_gs
rm -rf tensorflow_io tensorflow_io_gcs_filesystem
(cd tests && python -m pytest -s -v test_lmdb.py)
(python -m pytest -s -v test_image.py -k "webp or ppm or bmp or bounding or exif or hdr or openexr or tiff or avif")
(python -m pytest -s -v test_serialization.py)
Expand Down Expand Up @@ -576,7 +576,7 @@ jobs:
python -m pip install -U wheel setuptools
python --version
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64 --nightly $BUILD_NUMBER
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64 --nightly $BUILD_NUMBER
python setup.py --project tensorflow-io-gcs-filesystem --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64 --nightly $BUILD_NUMBER
- name: Auditwheel ${{ matrix.python }} macOS
run: |
set -x -e
Expand Down Expand Up @@ -616,7 +616,7 @@ jobs:
set -x -e
mv bazel-bin/tensorflow_io/.bazelrc .
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --nightly $BUILD_NUMBER
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --nightly $BUILD_NUMBER
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io-gcs-filesystem --data bazel-bin -q bdist_wheel --nightly $BUILD_NUMBER
- name: Auditwheel ${{ matrix.python }} Linux
run: |
set -x -e
Expand Down Expand Up @@ -660,7 +660,7 @@ jobs:
python --version
python -m pip install -U wheel setuptools
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --nightly %BUILD_NUMBER%
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --nightly %BUILD_NUMBER%
python setup.py --project tensorflow-io-gcs-filesystem --data bazel-bin -q bdist_wheel --nightly %BUILD_NUMBER%
ls -la dist
- uses: actions/upload-artifact@v2
with:
Expand Down Expand Up @@ -746,24 +746,24 @@ jobs:
set -e -x
rm -rf dist
mkdir -p dist
cp macOS-3.6-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp macOS-3.7-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp macOS-3.8-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp macOS-3.9-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Linux-3.6-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Linux-3.7-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Linux-3.8-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Linux-3.9-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Windows-3.6-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Windows-3.7-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Windows-3.8-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp Windows-3.9-nightly/tensorflow_io_plugin_gs_nightly*.whl dist/
cp macOS-3.6-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp macOS-3.7-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp macOS-3.8-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp macOS-3.9-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Linux-3.6-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Linux-3.7-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Linux-3.8-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Linux-3.9-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Windows-3.6-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Windows-3.7-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Windows-3.8-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
cp Windows-3.9-nightly/tensorflow_io_gcs_filesystem_nightly*.whl dist/
ls -la dist/
sha256sum dist/*.whl
- uses: pypa/gh-action-pypi-publish@master
with:
user: __token__
password: ${{ secrets.tensorflow_io_plugin_gs_nightly }}
password: ${{ secrets.tensorflow_io_gcs_filesystem_nightly }}

docker-nightly:
name: Docker Nightly
Expand Down
4 changes: 2 additions & 2 deletions docs/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ sudo ./configure.sh
# export BAZEL_OPTIMIZATION=

# Build shared libraries
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_plugin_gs/...
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_gcs_filesystem/...

# Once build is complete, shared libraries will be available in
# `bazel-bin/tensorflow_io/core/python/ops/` and it is possible
Expand Down Expand Up @@ -141,7 +141,7 @@ sudo ./configure.sh
# export BAZEL_OPTIMIZATION=

# Build shared libraries
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_plugin_gs/...
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_gcs_filesystem/...

# Once build is complete, shared libraries will be available in
# `bazel-bin/tensorflow_io/core/python/ops/` and it is possible
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
print(require)
sys.exit(0)

subpackages = ["tensorflow-io-plugin-gs"]
subpackages = ["tensorflow-io-gcs-filesystem"]

assert "--project" in sys.argv, "--project ({} or {}) must be provided".format(
"tensorflow-io", ", ".join(subpackages)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""tensorflow_io_plugin_gs"""
"""tensorflow_io_gcs_filesystem"""

from tensorflow_io_plugin_gs.core.python.ops import plugin_gs
from tensorflow_io_gcs_filesystem.core.python.ops import plugin_gs
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ cc_library(
)

cc_binary(
name = "python/ops/libtensorflow_io_plugin_gs.so",
name = "python/ops/libtensorflow_io_gcs_filesystem.so",
copts = tf_io_copts(),
linkshared = 1,
deps = [
"//tensorflow_io_plugin_gs/core:gs",
"//tensorflow_io_gcs_filesystem/core:gs",
],
)
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow_io_plugin_gs/core/file_system_plugin_gs.h"
#include "tensorflow_io_gcs_filesystem/core/file_system_plugin_gs.h"

#include "absl/strings/ascii.h"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ limitations under the License.
#include "google/cloud/storage/client.h"
#include "tensorflow/c/logging.h"
#include "tensorflow/c/tf_status.h"
#include "tensorflow_io_plugin_gs/core/expiring_lru_cache.h"
#include "tensorflow_io_plugin_gs/core/file_system_plugin_gs.h"
#include "tensorflow_io_plugin_gs/core/gcs_helper.h"
#include "tensorflow_io_plugin_gs/core/ram_file_block_cache.h"
#include "tensorflow_io_gcs_filesystem/core/expiring_lru_cache.h"
#include "tensorflow_io_gcs_filesystem/core/file_system_plugin_gs.h"
#include "tensorflow_io_gcs_filesystem/core/gcs_helper.h"
#include "tensorflow_io_gcs_filesystem/core/ram_file_block_cache.h"

namespace tensorflow {
namespace io {
Expand Down
Loading

0 comments on commit dcb3c1e

Please sign in to comment.