diff --git a/design_doc/planning_doc.md b/design_doc/planning_doc.md index 839ded3..c960848 100644 --- a/design_doc/planning_doc.md +++ b/design_doc/planning_doc.md @@ -4,10 +4,15 @@ 5. efficient image storage 6. compare with standard tfds on loading and storage 7. recover shema from save data +4. need to describe lazily loaded frames, more metadata fields + +### operations to be supported +1. merge +2. migration ### known bugs 3. sql part is completely broken -4. need to describe lazily loaded frames, more metadata fields +5. gcp - cloud seems to read as //, need to clean up the paths ### Big Steps 1. asynchonous writing, test with multiple processes & episodes diff --git a/examples/Fog_RTX_Cloud_Demo.ipynb b/examples/Fog_RTX_Cloud_Demo.ipynb deleted file mode 100644 index 9469984..0000000 --- a/examples/Fog_RTX_Cloud_Demo.ipynb +++ /dev/null @@ -1,425 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# Fog-RTX Work With Cloud\n", - "FogRTX supports a wide range of cloud service providers.\n", - "\n", - "### AWS" - ], - "metadata": { - "id": "upgVMgpfdSCk" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ex7qQ3klf2_y" - }, - "outputs": [], - "source": [ - "! git clone https://github.com/KeplerC/fog_x.git\n", - "! cd fog_x && git checkout cloud && pip install ." - ] - }, - { - "cell_type": "markdown", - "source": [ - "Install required AWS dependency and configure with your aws credential" - ], - "metadata": { - "id": "TbCXBT0rdgR7" - } - }, - { - "cell_type": "code", - "source": [ - "! curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\"\n", - "! unzip awscliv2.zip\n", - "! sudo ./aws/install\n", - "! aws configure" - ], - "metadata": { - "id": "k4Q0PXALS0Ol" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# create an AWS bucket named fog-rtx-test-east-1 (anything you want)\n", - "!aws s3api create-bucket --bucket fog-rtx-test-east-1 --region us-east-1" - ], - "metadata": { - "id": "r-VhhYFBdqit" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "Fog-RTX can load from the existing bucket, and add more to it!" - ], - "metadata": { - "id": "ZesCYo2udsei" - } - }, - { - "cell_type": "code", - "source": [ - "import fog_x\n", - "\n", - "dataset = fog_x.dataset.Dataset(\n", - " name=\"demo_ds\",\n", - " path='s3://fog-rtx-test-east-1',\n", - ")" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Il5tj-i1f8bG", - "outputId": "401bee3b-0807-4a01-e6e8-30a8f9756b48" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "INFO:fog_x.database.polars_connector:Prepare to load table demo_ds loaded from s3://fog-rtx-test-east-1//demo_ds.parquet.\n", - "INFO:fog_x.database.polars_connector:Table demo_ds loaded from s3://fog-rtx-test-east-1//demo_ds.parquet.\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "read from exisitng AWS stored dataset" - ], - "metadata": { - "id": "TArD9Frhd_U8" - } - }, - { - "cell_type": "code", - "source": [ - "dataset.get_episode_info()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 244 - }, - "id": "oLMT8ioUd6Km", - "outputId": "c821cf86-2b3e-4d6a-9b3b-71887e9a8b6f" - }, - "execution_count": 9, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "shape: (4, 44)\n", - "┌───────────┬──────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬───────────┐\n", - "│ episode_i ┆ Finished ┆ feature_g ┆ feature_g ┆ … ┆ robot_sta ┆ feature_r ┆ feature_r ┆ reward_co │\n", - "│ d ┆ --- ┆ ripper_cl ┆ ripper_cl ┆ ┆ te_count ┆ eward_typ ┆ eward_sha ┆ unt │\n", - "│ --- ┆ bool ┆ osedness_ ┆ osedness_ ┆ ┆ --- ┆ e ┆ pe ┆ --- │\n", - "│ i64 ┆ ┆ actio… ┆ actio… ┆ ┆ f64 ┆ --- ┆ --- ┆ f64 │\n", - "│ ┆ ┆ --- ┆ --- ┆ ┆ ┆ str ┆ str ┆ │\n", - "│ ┆ ┆ str ┆ str ┆ ┆ ┆ ┆ ┆ │\n", - "╞═══════════╪══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n", - "│ 0 ┆ true ┆ float32 ┆ () ┆ … ┆ 71.0 ┆ float32 ┆ () ┆ 71.0 │\n", - "│ 1 ┆ true ┆ float32 ┆ () ┆ … ┆ 76.0 ┆ float32 ┆ () ┆ 76.0 │\n", - "│ 2 ┆ true ┆ float32 ┆ () ┆ … ┆ 71.0 ┆ float32 ┆ () ┆ 71.0 │\n", - "│ 3 ┆ true ┆ float32 ┆ () ┆ … ┆ 76.0 ┆ float32 ┆ () ┆ 76.0 │\n", - "└───────────┴──────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴───────────┘" - ], - "text/html": [ - "
\n", - "shape: (4, 44)
episode_idFinishedfeature_gripper_closedness_action_typefeature_gripper_closedness_action_shapegripper_closedness_action_countfeature_rotation_delta_typefeature_rotation_delta_shaperotation_delta_countfeature_terminate_episode_typefeature_terminate_episode_shapeterminate_episode_countfeature_world_vector_typefeature_world_vector_shapeworld_vector_countfeature_is_first_typefeature_is_first_shapeis_first_countfeature_is_last_typefeature_is_last_shapeis_last_countfeature_is_terminal_typefeature_is_terminal_shapeis_terminal_countfeature_hand_image_typefeature_hand_image_shapehand_image_countfeature_image_typefeature_image_shapeimage_countfeature_image_with_depth_typefeature_image_with_depth_shapeimage_with_depth_countfeature_natural_language_embedding_typefeature_natural_language_embedding_shapenatural_language_embedding_countfeature_natural_language_instruction_typefeature_natural_language_instruction_shapenatural_language_instruction_countfeature_robot_state_typefeature_robot_state_shaperobot_state_countfeature_reward_typefeature_reward_shapereward_count
i64boolstrstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64
0true"float32""()"71.0"float32""(3,)"71.0"float32""()"71.0"float32""(3,)"71.0"bool""()"71.0"bool""()"71.0"bool""()"71.0"uint8""(480, 640, 3)"71.0"uint8""(480, 640, 3)"71.0"float32""(480, 640, 1)"71.0"float32""(512,)"71.0"string""()"71.0"float32""(15,)"71.0"float32""()"71.0
1true"float32""()"76.0"float32""(3,)"76.0"float32""()"76.0"float32""(3,)"76.0"bool""()"76.0"bool""()"76.0"bool""()"76.0"uint8""(480, 640, 3)"76.0"uint8""(480, 640, 3)"76.0"float32""(480, 640, 1)"76.0"float32""(512,)"76.0"string""()"76.0"float32""(15,)"76.0"float32""()"76.0
2true"float32""()"71.0"float32""(3,)"71.0"float32""()"71.0"float32""(3,)"71.0"bool""()"71.0"bool""()"71.0"bool""()"71.0"uint8""(480, 640, 3)"71.0"uint8""(480, 640, 3)"71.0"float32""(480, 640, 1)"71.0"float32""(512,)"71.0"string""()"71.0"float32""(15,)"71.0"float32""()"71.0
3true"float32""()"76.0"float32""(3,)"76.0"float32""()"76.0"float32""(3,)"76.0"bool""()"76.0"bool""()"76.0"bool""()"76.0"uint8""(480, 640, 3)"76.0"uint8""(480, 640, 3)"76.0"float32""(480, 640, 1)"76.0"float32""(512,)"76.0"string""()"76.0"float32""(15,)"76.0"float32""()"76.0
" - ] - }, - "metadata": {}, - "execution_count": 9 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "add more to the stored dataset" - ], - "metadata": { - "id": "mul7xRzFeDRt" - } - }, - { - "cell_type": "code", - "source": [ - "dataset.load_rtx_episodes(\n", - " name=\"berkeley_autolab_ur5\",\n", - " split=\"train[4:10]\",\n", - ")" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ivIyifc4f9ih", - "outputId": "daef546e-c378-45a1-87eb-562d0f92c8f0" - }, - "execution_count": 10, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "INFO:absl:Load dataset info from gs://gresearch/robotics/berkeley_autolab_ur5/0.1.0\n", - "INFO:absl:Creating a tf.data.Dataset reading 4 files located in folders: gs://gresearch/robotics/berkeley_autolab_ur5/0.1.0.\n", - "INFO:absl:Constructing tf.data.Dataset berkeley_autolab_ur5 for split train[4:10], from gs://gresearch/robotics/berkeley_autolab_ur5/0.1.0\n", - "INFO:fog_x.dataset:{'steps': <_VariantDataset element_spec={'action': {'gripper_closedness_action': TensorSpec(shape=(), dtype=tf.float32, name=None), 'rotation_delta': TensorSpec(shape=(3,), dtype=tf.float32, name=None), 'terminate_episode': TensorSpec(shape=(), dtype=tf.float32, name=None), 'world_vector': TensorSpec(shape=(3,), dtype=tf.float32, name=None)}, 'is_first': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_last': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_terminal': TensorSpec(shape=(), dtype=tf.bool, name=None), 'observation': {'hand_image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image_with_depth': TensorSpec(shape=(480, 640, 1), dtype=tf.float32, name=None), 'natural_language_embedding': TensorSpec(shape=(512,), dtype=tf.float32, name=None), 'natural_language_instruction': TensorSpec(shape=(), dtype=tf.string, name=None), 'robot_state': TensorSpec(shape=(15,), dtype=tf.float32, name=None)}, 'reward': TensorSpec(shape=(), dtype=tf.float32, name=None)}>}\n", - "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'gripper_closedness_action_count': 76, 'rotation_delta_count': 76, 'terminate_episode_count': 76, 'world_vector_count': 76, 'is_first_count': 76, 'is_last_count': 76, 'is_terminal_count': 76, 'hand_image_count': 76, 'image_count': 76, 'image_with_depth_count': 76, 'natural_language_embedding_count': 76, 'natural_language_instruction_count': 76, 'robot_state_count': 76, 'reward_count': 76}\n", - "INFO:fog_x.dataset:{'steps': <_VariantDataset element_spec={'action': {'gripper_closedness_action': TensorSpec(shape=(), dtype=tf.float32, name=None), 'rotation_delta': TensorSpec(shape=(3,), dtype=tf.float32, name=None), 'terminate_episode': TensorSpec(shape=(), dtype=tf.float32, name=None), 'world_vector': TensorSpec(shape=(3,), dtype=tf.float32, name=None)}, 'is_first': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_last': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_terminal': TensorSpec(shape=(), dtype=tf.bool, name=None), 'observation': {'hand_image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image_with_depth': TensorSpec(shape=(480, 640, 1), dtype=tf.float32, name=None), 'natural_language_embedding': TensorSpec(shape=(512,), dtype=tf.float32, name=None), 'natural_language_instruction': TensorSpec(shape=(), dtype=tf.string, name=None), 'robot_state': TensorSpec(shape=(15,), dtype=tf.float32, name=None)}, 'reward': TensorSpec(shape=(), dtype=tf.float32, name=None)}>}\n", - "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'gripper_closedness_action_count': 123, 'rotation_delta_count': 123, 'terminate_episode_count': 123, 'world_vector_count': 123, 'is_first_count': 123, 'is_last_count': 123, 'is_terminal_count': 123, 'hand_image_count': 123, 'image_count': 123, 'image_with_depth_count': 123, 'natural_language_embedding_count': 123, 'natural_language_instruction_count': 123, 'robot_state_count': 123, 'reward_count': 123}\n", - "INFO:fog_x.dataset:{'steps': <_VariantDataset element_spec={'action': {'gripper_closedness_action': TensorSpec(shape=(), dtype=tf.float32, name=None), 'rotation_delta': TensorSpec(shape=(3,), dtype=tf.float32, name=None), 'terminate_episode': TensorSpec(shape=(), dtype=tf.float32, name=None), 'world_vector': TensorSpec(shape=(3,), dtype=tf.float32, name=None)}, 'is_first': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_last': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_terminal': TensorSpec(shape=(), dtype=tf.bool, name=None), 'observation': {'hand_image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image_with_depth': TensorSpec(shape=(480, 640, 1), dtype=tf.float32, name=None), 'natural_language_embedding': TensorSpec(shape=(512,), dtype=tf.float32, name=None), 'natural_language_instruction': TensorSpec(shape=(), dtype=tf.string, name=None), 'robot_state': TensorSpec(shape=(15,), dtype=tf.float32, name=None)}, 'reward': TensorSpec(shape=(), dtype=tf.float32, name=None)}>}\n", - "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'gripper_closedness_action_count': 103, 'rotation_delta_count': 103, 'terminate_episode_count': 103, 'world_vector_count': 103, 'is_first_count': 103, 'is_last_count': 103, 'is_terminal_count': 103, 'hand_image_count': 103, 'image_count': 103, 'image_with_depth_count': 103, 'natural_language_embedding_count': 103, 'natural_language_instruction_count': 103, 'robot_state_count': 103, 'reward_count': 103}\n", - "INFO:fog_x.dataset:{'steps': <_VariantDataset element_spec={'action': {'gripper_closedness_action': TensorSpec(shape=(), dtype=tf.float32, name=None), 'rotation_delta': TensorSpec(shape=(3,), dtype=tf.float32, name=None), 'terminate_episode': TensorSpec(shape=(), dtype=tf.float32, name=None), 'world_vector': TensorSpec(shape=(3,), dtype=tf.float32, name=None)}, 'is_first': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_last': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_terminal': TensorSpec(shape=(), dtype=tf.bool, name=None), 'observation': {'hand_image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image_with_depth': TensorSpec(shape=(480, 640, 1), dtype=tf.float32, name=None), 'natural_language_embedding': TensorSpec(shape=(512,), dtype=tf.float32, name=None), 'natural_language_instruction': TensorSpec(shape=(), dtype=tf.string, name=None), 'robot_state': TensorSpec(shape=(15,), dtype=tf.float32, name=None)}, 'reward': TensorSpec(shape=(), dtype=tf.float32, name=None)}>}\n", - "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'gripper_closedness_action_count': 110, 'rotation_delta_count': 110, 'terminate_episode_count': 110, 'world_vector_count': 110, 'is_first_count': 110, 'is_last_count': 110, 'is_terminal_count': 110, 'hand_image_count': 110, 'image_count': 110, 'image_with_depth_count': 110, 'natural_language_embedding_count': 110, 'natural_language_instruction_count': 110, 'robot_state_count': 110, 'reward_count': 110}\n", - "INFO:fog_x.dataset:{'steps': <_VariantDataset element_spec={'action': {'gripper_closedness_action': TensorSpec(shape=(), dtype=tf.float32, name=None), 'rotation_delta': TensorSpec(shape=(3,), dtype=tf.float32, name=None), 'terminate_episode': TensorSpec(shape=(), dtype=tf.float32, name=None), 'world_vector': TensorSpec(shape=(3,), dtype=tf.float32, name=None)}, 'is_first': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_last': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_terminal': TensorSpec(shape=(), dtype=tf.bool, name=None), 'observation': {'hand_image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image_with_depth': TensorSpec(shape=(480, 640, 1), dtype=tf.float32, name=None), 'natural_language_embedding': TensorSpec(shape=(512,), dtype=tf.float32, name=None), 'natural_language_instruction': TensorSpec(shape=(), dtype=tf.string, name=None), 'robot_state': TensorSpec(shape=(15,), dtype=tf.float32, name=None)}, 'reward': TensorSpec(shape=(), dtype=tf.float32, name=None)}>}\n", - "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'gripper_closedness_action_count': 118, 'rotation_delta_count': 118, 'terminate_episode_count': 118, 'world_vector_count': 118, 'is_first_count': 118, 'is_last_count': 118, 'is_terminal_count': 118, 'hand_image_count': 118, 'image_count': 118, 'image_with_depth_count': 118, 'natural_language_embedding_count': 118, 'natural_language_instruction_count': 118, 'robot_state_count': 118, 'reward_count': 118}\n", - "INFO:fog_x.dataset:{'steps': <_VariantDataset element_spec={'action': {'gripper_closedness_action': TensorSpec(shape=(), dtype=tf.float32, name=None), 'rotation_delta': TensorSpec(shape=(3,), dtype=tf.float32, name=None), 'terminate_episode': TensorSpec(shape=(), dtype=tf.float32, name=None), 'world_vector': TensorSpec(shape=(3,), dtype=tf.float32, name=None)}, 'is_first': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_last': TensorSpec(shape=(), dtype=tf.bool, name=None), 'is_terminal': TensorSpec(shape=(), dtype=tf.bool, name=None), 'observation': {'hand_image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image': TensorSpec(shape=(480, 640, 3), dtype=tf.uint8, name=None), 'image_with_depth': TensorSpec(shape=(480, 640, 1), dtype=tf.float32, name=None), 'natural_language_embedding': TensorSpec(shape=(512,), dtype=tf.float32, name=None), 'natural_language_instruction': TensorSpec(shape=(), dtype=tf.string, name=None), 'robot_state': TensorSpec(shape=(15,), dtype=tf.float32, name=None)}, 'reward': TensorSpec(shape=(), dtype=tf.float32, name=None)}>}\n", - "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'gripper_closedness_action_count': 84, 'rotation_delta_count': 84, 'terminate_episode_count': 84, 'world_vector_count': 84, 'is_first_count': 84, 'is_last_count': 84, 'is_terminal_count': 84, 'hand_image_count': 84, 'image_count': 84, 'image_with_depth_count': 84, 'natural_language_embedding_count': 84, 'natural_language_instruction_count': 84, 'robot_state_count': 84, 'reward_count': 84}\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "The data is automatically uploaded to the cloud!\n", - "We can create a separate reader by" - ], - "metadata": { - "id": "mUneci9XeHsE" - } - }, - { - "cell_type": "code", - "source": [ - "dataset2 = fog_x.dataset.Dataset(\n", - " name=\"demo_ds\",\n", - " path='s3://fog-rtx-test-east-1',\n", - ")" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cQHIKeNAeSrY", - "outputId": "27379fd0-3a51-401c-e2b3-099bb7356ba6" - }, - "execution_count": 11, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "INFO:fog_x.database.polars_connector:Prepare to load table demo_ds loaded from s3://fog-rtx-test-east-1//demo_ds.parquet.\n", - "INFO:fog_x.database.polars_connector:Table demo_ds loaded from s3://fog-rtx-test-east-1//demo_ds.parquet.\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "# metadata\n", - "trajectory_metadata = dataset2.get_episode_info()\n", - "trajectory_metadata" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 432 - }, - "id": "E4slMiSzf-se", - "outputId": "9053cc4e-c8e2-4a16-c799-84a08712edd9" - }, - "execution_count": 12, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "shape: (10, 44)\n", - "┌───────────┬──────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬───────────┐\n", - "│ episode_i ┆ Finished ┆ feature_g ┆ feature_g ┆ … ┆ robot_sta ┆ feature_r ┆ feature_r ┆ reward_co │\n", - "│ d ┆ --- ┆ ripper_cl ┆ ripper_cl ┆ ┆ te_count ┆ eward_typ ┆ eward_sha ┆ unt │\n", - "│ --- ┆ bool ┆ osedness_ ┆ osedness_ ┆ ┆ --- ┆ e ┆ pe ┆ --- │\n", - "│ i64 ┆ ┆ actio… ┆ actio… ┆ ┆ f64 ┆ --- ┆ --- ┆ f64 │\n", - "│ ┆ ┆ --- ┆ --- ┆ ┆ ┆ str ┆ str ┆ │\n", - "│ ┆ ┆ str ┆ str ┆ ┆ ┆ ┆ ┆ │\n", - "╞═══════════╪══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n", - "│ 0 ┆ true ┆ float32 ┆ () ┆ … ┆ 71.0 ┆ float32 ┆ () ┆ 71.0 │\n", - "│ 1 ┆ true ┆ float32 ┆ () ┆ … ┆ 76.0 ┆ float32 ┆ () ┆ 76.0 │\n", - "│ 2 ┆ true ┆ float32 ┆ () ┆ … ┆ 71.0 ┆ float32 ┆ () ┆ 71.0 │\n", - "│ 3 ┆ true ┆ float32 ┆ () ┆ … ┆ 76.0 ┆ float32 ┆ () ┆ 76.0 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 6 ┆ true ┆ float32 ┆ () ┆ … ┆ 110.0 ┆ float32 ┆ () ┆ 110.0 │\n", - "│ 7 ┆ true ┆ float32 ┆ () ┆ … ┆ 118.0 ┆ float32 ┆ () ┆ 118.0 │\n", - "│ 8 ┆ true ┆ float32 ┆ () ┆ … ┆ 84.0 ┆ float32 ┆ () ┆ 84.0 │\n", - "│ 9 ┆ true ┆ float32 ┆ () ┆ … ┆ 97.0 ┆ float32 ┆ () ┆ 97.0 │\n", - "└───────────┴──────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴───────────┘" - ], - "text/html": [ - "
\n", - "shape: (10, 44)
episode_idFinishedfeature_gripper_closedness_action_typefeature_gripper_closedness_action_shapegripper_closedness_action_countfeature_rotation_delta_typefeature_rotation_delta_shaperotation_delta_countfeature_terminate_episode_typefeature_terminate_episode_shapeterminate_episode_countfeature_world_vector_typefeature_world_vector_shapeworld_vector_countfeature_is_first_typefeature_is_first_shapeis_first_countfeature_is_last_typefeature_is_last_shapeis_last_countfeature_is_terminal_typefeature_is_terminal_shapeis_terminal_countfeature_hand_image_typefeature_hand_image_shapehand_image_countfeature_image_typefeature_image_shapeimage_countfeature_image_with_depth_typefeature_image_with_depth_shapeimage_with_depth_countfeature_natural_language_embedding_typefeature_natural_language_embedding_shapenatural_language_embedding_countfeature_natural_language_instruction_typefeature_natural_language_instruction_shapenatural_language_instruction_countfeature_robot_state_typefeature_robot_state_shaperobot_state_countfeature_reward_typefeature_reward_shapereward_count
i64boolstrstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64strstrf64
0true"float32""()"71.0"float32""(3,)"71.0"float32""()"71.0"float32""(3,)"71.0"bool""()"71.0"bool""()"71.0"bool""()"71.0"uint8""(480, 640, 3)"71.0"uint8""(480, 640, 3)"71.0"float32""(480, 640, 1)"71.0"float32""(512,)"71.0"string""()"71.0"float32""(15,)"71.0"float32""()"71.0
1true"float32""()"76.0"float32""(3,)"76.0"float32""()"76.0"float32""(3,)"76.0"bool""()"76.0"bool""()"76.0"bool""()"76.0"uint8""(480, 640, 3)"76.0"uint8""(480, 640, 3)"76.0"float32""(480, 640, 1)"76.0"float32""(512,)"76.0"string""()"76.0"float32""(15,)"76.0"float32""()"76.0
2true"float32""()"71.0"float32""(3,)"71.0"float32""()"71.0"float32""(3,)"71.0"bool""()"71.0"bool""()"71.0"bool""()"71.0"uint8""(480, 640, 3)"71.0"uint8""(480, 640, 3)"71.0"float32""(480, 640, 1)"71.0"float32""(512,)"71.0"string""()"71.0"float32""(15,)"71.0"float32""()"71.0
3true"float32""()"76.0"float32""(3,)"76.0"float32""()"76.0"float32""(3,)"76.0"bool""()"76.0"bool""()"76.0"bool""()"76.0"uint8""(480, 640, 3)"76.0"uint8""(480, 640, 3)"76.0"float32""(480, 640, 1)"76.0"float32""(512,)"76.0"string""()"76.0"float32""(15,)"76.0"float32""()"76.0
4true"float32""()"123.0"float32""(3,)"123.0"float32""()"123.0"float32""(3,)"123.0"bool""()"123.0"bool""()"123.0"bool""()"123.0"uint8""(480, 640, 3)"123.0"uint8""(480, 640, 3)"123.0"float32""(480, 640, 1)"123.0"float32""(512,)"123.0"string""()"123.0"float32""(15,)"123.0"float32""()"123.0
5true"float32""()"103.0"float32""(3,)"103.0"float32""()"103.0"float32""(3,)"103.0"bool""()"103.0"bool""()"103.0"bool""()"103.0"uint8""(480, 640, 3)"103.0"uint8""(480, 640, 3)"103.0"float32""(480, 640, 1)"103.0"float32""(512,)"103.0"string""()"103.0"float32""(15,)"103.0"float32""()"103.0
6true"float32""()"110.0"float32""(3,)"110.0"float32""()"110.0"float32""(3,)"110.0"bool""()"110.0"bool""()"110.0"bool""()"110.0"uint8""(480, 640, 3)"110.0"uint8""(480, 640, 3)"110.0"float32""(480, 640, 1)"110.0"float32""(512,)"110.0"string""()"110.0"float32""(15,)"110.0"float32""()"110.0
7true"float32""()"118.0"float32""(3,)"118.0"float32""()"118.0"float32""(3,)"118.0"bool""()"118.0"bool""()"118.0"bool""()"118.0"uint8""(480, 640, 3)"118.0"uint8""(480, 640, 3)"118.0"float32""(480, 640, 1)"118.0"float32""(512,)"118.0"string""()"118.0"float32""(15,)"118.0"float32""()"118.0
8true"float32""()"84.0"float32""(3,)"84.0"float32""()"84.0"float32""(3,)"84.0"bool""()"84.0"bool""()"84.0"bool""()"84.0"uint8""(480, 640, 3)"84.0"uint8""(480, 640, 3)"84.0"float32""(480, 640, 1)"84.0"float32""(512,)"84.0"string""()"84.0"float32""(15,)"84.0"float32""()"84.0
9true"float32""()"97.0"float32""(3,)"97.0"float32""()"97.0"float32""(3,)"97.0"bool""()"97.0"bool""()"97.0"bool""()"97.0"uint8""(480, 640, 3)"97.0"uint8""(480, 640, 3)"97.0"float32""(480, 640, 1)"97.0"float32""(512,)"97.0"string""()"97.0"float32""(15,)"97.0"float32""()"97.0
" - ] - }, - "metadata": {}, - "execution_count": 12 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Google Cloud Platform" - ], - "metadata": { - "id": "cB7QVbp6i-Mx" - } - }, - { - "cell_type": "markdown", - "source": [ - "Register google cloud credentials\n", - "\n", - "Alternative in non-colab environment, run following command instead:\n", - "```\n", - "gcloud auth application-default login --quiet --no-launch-browser\n", - "```\n" - ], - "metadata": { - "id": "8MIV3MZUjNta" - } - }, - { - "cell_type": "code", - "source": [ - "from google.colab import auth\n", - "PROJECT_ID = \"canvas-rampart-342500\"\n", - "auth.authenticate_user(project_id=PROJECT_ID)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ryd_To6LL3nX", - "outputId": "d571dfd9-6ffc-4847-e5d2-a455359eefd6" - }, - "execution_count": 13, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "INFO:google.colab.auth:Failure refreshing credentials: (\"Failed to retrieve http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true from the Google Compute Engine metadata service. Status: 404 Response:\\nb''\", )\n", - "INFO:google.colab.auth:Failure refreshing credentials: (\"Failed to retrieve http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true from the Google Compute Engine metadata service. Status: 404 Response:\\nb''\", )\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "! gcloud storage buckets create gs://fog_rtx_test --location=us-east1" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "fYM3ExvGL3z7", - "outputId": "721e97ca-0a90-4e85-af14-3503eb62dc06" - }, - "execution_count": 14, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Creating gs://fog_rtx_test/...\n", - "\u001b[1;31mERROR:\u001b[0m (gcloud.storage.buckets.create) HTTPError 409: Your previous request to create the named bucket succeeded and you already own it.\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "dataset = fog_x.dataset.Dataset(\n", - " name=\"demo_ds\",\n", - " path='gs://fog_rtx_test/',\n", - ")" - ], - "metadata": { - "id": "pd94S4VlL32u" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### Known issues\n", - "\n", - "1. `export` as rlds format to the cloud directly does not work yet for S3 (known issue for tensorflow Gfile)\n", - "2. (will fix) automatically check the existence" - ], - "metadata": { - "id": "P2RCUMs6knNc" - } - }, - { - "cell_type": "code", - "source": [], - "metadata": { - "id": "QKS5jK-Qk9fN" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/examples/Fog_X Demo.ipynb b/examples/Fog_X_Analytics_Demo.ipynb similarity index 100% rename from examples/Fog_X Demo.ipynb rename to examples/Fog_X_Analytics_Demo.ipynb diff --git a/examples/Fog_X_Cloud_Demo.ipynb b/examples/Fog_X_Cloud_Demo.ipynb new file mode 100644 index 0000000..b6a9b89 --- /dev/null +++ b/examples/Fog_X_Cloud_Demo.ipynb @@ -0,0 +1,609 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Fog-RTX Cloud Data Collection Demo\n", + "\n", + "FogRTX supports a wide range of cloud service providers. In this workbook, we show the support of AWS and google cloud.\n", + "\n", + "### AWS" + ], + "metadata": { + "id": "upgVMgpfdSCk" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ex7qQ3klf2_y" + }, + "outputs": [], + "source": [ + "! git clone https://github.com/KeplerC/fog_x.git\n", + "! cd fog_x && git checkout cloud-demo-dev && pip install ." + ] + }, + { + "cell_type": "markdown", + "source": [ + "Install required AWS dependency and configure with your aws credential" + ], + "metadata": { + "id": "TbCXBT0rdgR7" + } + }, + { + "cell_type": "code", + "source": [ + "! curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\"\n", + "! unzip -q awscliv2.zip\n", + "! sudo ./aws/install\n", + "! aws configure" + ], + "metadata": { + "id": "k4Q0PXALS0Ol" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# create an AWS bucket named fog-rtx-test-east-1 (anything you want)\n", + "!aws s3api create-bucket --bucket fog-rtx-test-east-2 --region us-east-1" + ], + "metadata": { + "id": "r-VhhYFBdqit", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "dc9c4b4e-baa9-4401-c4d6-eabed618d30d" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{\n", + " \"Location\": \"/fog-rtx-test-east-2\"\n", + "}\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!pip3 install boto3" + ], + "metadata": { + "id": "TlSvbrw8deO1" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Creating or Loading from existing datasets\n", + "\n", + "\n", + "Fog-RTX can load from the existing bucket, and add more to it!" + ], + "metadata": { + "id": "ZesCYo2udsei" + } + }, + { + "cell_type": "code", + "source": [ + "import fog_x\n", + "\n", + "dataset = fog_x.dataset.Dataset(\n", + " name=\"demo_ds\",\n", + " path='s3://fog-rtx-test-east-2',\n", + ")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Il5tj-i1f8bG", + "outputId": "73e922e2-a44d-4d2d-bda9-97c54b15c445" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:fog_x.database.polars_connector:Prepare to load table demo_ds loaded from s3://fog-rtx-test-east-2/demo_ds.parquet.\n", + "INFO:botocore.credentials:Found credentials in shared credentials file: ~/.aws/credentials\n", + "INFO:fog_x.database.polars_connector:Table demo_ds loaded from s3://fog-rtx-test-east-2/demo_ds.parquet.\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "read from exisitng AWS stored dataset" + ], + "metadata": { + "id": "TArD9Frhd_U8" + } + }, + { + "cell_type": "code", + "source": [ + "dataset.get_episode_info()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 129 + }, + "id": "oLMT8ioUd6Km", + "outputId": "c7510513-96b8-4942-abc6-84c0541c6759" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "shape: (1, 8)\n", + "┌────────────┬──────────┬────────────┬────────────┬────────────┬───────────┬───────────┬───────────┐\n", + "│ episode_id ┆ Finished ┆ feature_ar ┆ feature_ar ┆ arm_camera ┆ feature_g ┆ feature_g ┆ gripper_a │\n", + "│ --- ┆ --- ┆ m_camera_v ┆ m_camera_v ┆ _view_coun ┆ ripper_ac ┆ ripper_ac ┆ cton_coun │\n", + "│ i64 ┆ bool ┆ iew_type ┆ iew_shape ┆ t ┆ ton_type ┆ ton_shape ┆ t │\n", + "│ ┆ ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ ┆ ┆ str ┆ str ┆ f64 ┆ str ┆ str ┆ f64 │\n", + "╞════════════╪══════════╪════════════╪════════════╪════════════╪═══════════╪═══════════╪═══════════╡\n", + "│ 0 ┆ true ┆ float64 ┆ (480, 640, ┆ 0.0 ┆ float64 ┆ (7,) ┆ 0.0 │\n", + "│ ┆ ┆ ┆ 3) ┆ ┆ ┆ ┆ │\n", + "└────────────┴──────────┴────────────┴────────────┴────────────┴───────────┴───────────┴───────────┘" + ], + "text/html": [ + "
\n", + "shape: (1, 8)
episode_idFinishedfeature_arm_camera_view_typefeature_arm_camera_view_shapearm_camera_view_countfeature_gripper_acton_typefeature_gripper_acton_shapegripper_acton_count
i64boolstrstrf64strstrf64
0true"float64""(480, 640, 3)"0.0"float64""(7,)"0.0
" + ] + }, + "metadata": {}, + "execution_count": 6 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Adding new data to the dataset" + ], + "metadata": { + "id": "lcij8xiWui0P" + } + }, + { + "cell_type": "code", + "source": [ + "import numpy as np\n", + "\n", + "# create a new trajectory\n", + "episode = dataset.new_episode()\n", + "# collect step data for the episode\n", + "episode.add(feature = \"arm_camera_view\", value = np.random.rand(480, 640, 3))\n", + "episode.add(feature = \"gripper_acton\", value = np.random.rand(7))\n", + "# Automatically time-aligns and saves the trajectory\n", + "episode.close()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "akiVQqstdnWR", + "outputId": "a71f273a-025e-4102-cab5-6ecc398140ff" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:fog_x.database.db_manager:Closing the episode with metadata {}\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "dataset.get_episode_info()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 161 + }, + "id": "uHZZnvAmeqqx", + "outputId": "a827585e-d5d0-4fd7-ce9c-51350e50de71" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "shape: (2, 8)\n", + "┌────────────┬──────────┬────────────┬────────────┬────────────┬───────────┬───────────┬───────────┐\n", + "│ episode_id ┆ Finished ┆ feature_ar ┆ feature_ar ┆ arm_camera ┆ feature_g ┆ feature_g ┆ gripper_a │\n", + "│ --- ┆ --- ┆ m_camera_v ┆ m_camera_v ┆ _view_coun ┆ ripper_ac ┆ ripper_ac ┆ cton_coun │\n", + "│ i64 ┆ bool ┆ iew_type ┆ iew_shape ┆ t ┆ ton_type ┆ ton_shape ┆ t │\n", + "│ ┆ ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ ┆ ┆ str ┆ str ┆ f64 ┆ str ┆ str ┆ f64 │\n", + "╞════════════╪══════════╪════════════╪════════════╪════════════╪═══════════╪═══════════╪═══════════╡\n", + "│ 0 ┆ true ┆ float64 ┆ (480, 640, ┆ 0.0 ┆ float64 ┆ (7,) ┆ 0.0 │\n", + "│ ┆ ┆ ┆ 3) ┆ ┆ ┆ ┆ │\n", + "│ 1 ┆ true ┆ float64 ┆ (480, 640, ┆ 0.0 ┆ float64 ┆ (7,) ┆ 0.0 │\n", + "│ ┆ ┆ ┆ 3) ┆ ┆ ┆ ┆ │\n", + "└────────────┴──────────┴────────────┴────────────┴────────────┴───────────┴───────────┴───────────┘" + ], + "text/html": [ + "
\n", + "shape: (2, 8)
episode_idFinishedfeature_arm_camera_view_typefeature_arm_camera_view_shapearm_camera_view_countfeature_gripper_acton_typefeature_gripper_acton_shapegripper_acton_count
i64boolstrstrf64strstrf64
0true"float64""(480, 640, 3)"0.0"float64""(7,)"0.0
1true"float64""(480, 640, 3)"0.0"float64""(7,)"0.0
" + ] + }, + "metadata": {}, + "execution_count": 8 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Load Cloud Dataset at different place!\n", + "The data is automatically uploaded to the cloud!\n", + "We can create a different reader (you can run this on a different machine).\n", + "The data is automatically loaded and read!" + ], + "metadata": { + "id": "mUneci9XeHsE" + } + }, + { + "cell_type": "code", + "source": [ + "dataset2 = fog_x.dataset.Dataset(\n", + " name=\"demo_ds\",\n", + " path='s3://fog-rtx-test-east-2',\n", + ")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cQHIKeNAeSrY", + "outputId": "421fb7d5-9839-4ab7-c935-26025ba783d3" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:fog_x.database.polars_connector:Prepare to load table demo_ds loaded from s3://fog-rtx-test-east-2/demo_ds.parquet.\n", + "INFO:fog_x.database.polars_connector:Table demo_ds loaded from s3://fog-rtx-test-east-2/demo_ds.parquet.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# metadata\n", + "trajectory_metadata = dataset2.get_episode_info()\n", + "trajectory_metadata" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 161 + }, + "id": "E4slMiSzf-se", + "outputId": "79b9813c-beac-4ad2-8c06-625e3d388754" + }, + "execution_count": 10, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "shape: (2, 8)\n", + "┌────────────┬──────────┬────────────┬────────────┬────────────┬───────────┬───────────┬───────────┐\n", + "│ episode_id ┆ Finished ┆ feature_ar ┆ feature_ar ┆ arm_camera ┆ feature_g ┆ feature_g ┆ gripper_a │\n", + "│ --- ┆ --- ┆ m_camera_v ┆ m_camera_v ┆ _view_coun ┆ ripper_ac ┆ ripper_ac ┆ cton_coun │\n", + "│ i64 ┆ bool ┆ iew_type ┆ iew_shape ┆ t ┆ ton_type ┆ ton_shape ┆ t │\n", + "│ ┆ ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ ┆ ┆ str ┆ str ┆ f64 ┆ str ┆ str ┆ f64 │\n", + "╞════════════╪══════════╪════════════╪════════════╪════════════╪═══════════╪═══════════╪═══════════╡\n", + "│ 0 ┆ true ┆ float64 ┆ (480, 640, ┆ 0.0 ┆ float64 ┆ (7,) ┆ 0.0 │\n", + "│ ┆ ┆ ┆ 3) ┆ ┆ ┆ ┆ │\n", + "│ 1 ┆ true ┆ float64 ┆ (480, 640, ┆ 0.0 ┆ float64 ┆ (7,) ┆ 0.0 │\n", + "│ ┆ ┆ ┆ 3) ┆ ┆ ┆ ┆ │\n", + "└────────────┴──────────┴────────────┴────────────┴────────────┴───────────┴───────────┴───────────┘" + ], + "text/html": [ + "
\n", + "shape: (2, 8)
episode_idFinishedfeature_arm_camera_view_typefeature_arm_camera_view_shapearm_camera_view_countfeature_gripper_acton_typefeature_gripper_acton_shapegripper_acton_count
i64boolstrstrf64strstrf64
0true"float64""(480, 640, 3)"0.0"float64""(7,)"0.0
1true"float64""(480, 640, 3)"0.0"float64""(7,)"0.0
" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Google Cloud Platform" + ], + "metadata": { + "id": "cB7QVbp6i-Mx" + } + }, + { + "cell_type": "markdown", + "source": [ + "This can also be done on GCP!\n", + "\n", + "Register google cloud credentials\n", + "\n", + "Alternative in non-colab environment, run following command instead:\n", + "```\n", + "gcloud auth application-default login --quiet --no-launch-browser\n", + "```\n" + ], + "metadata": { + "id": "8MIV3MZUjNta" + } + }, + { + "cell_type": "code", + "source": [ + "from google.colab import auth\n", + "PROJECT_ID = \"canvas-rampart-342500\"\n", + "auth.authenticate_user(project_id=PROJECT_ID)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ryd_To6LL3nX", + "outputId": "714ea38c-11d9-44fd-b8c4-5cb4ebd8b242" + }, + "execution_count": 11, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:google.colab.auth:Failure refreshing credentials: (\"Failed to retrieve http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true from the Google Compute Engine metadata service. Status: 404 Response:\\nb''\", )\n", + "INFO:google.colab.auth:Failure refreshing credentials: (\"Failed to retrieve http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true from the Google Compute Engine metadata service. Status: 404 Response:\\nb''\", )\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "! gcloud storage buckets create gs://fog_rtx_test --location=us-east1" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fYM3ExvGL3z7", + "outputId": "31c6bc57-4c3a-4b6f-b7ef-4132af7a926c" + }, + "execution_count": 12, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Creating gs://fog_rtx_test/...\n", + "\u001b[1;31mERROR:\u001b[0m (gcloud.storage.buckets.create) HTTPError 409: Your previous request to create the named bucket succeeded and you already own it.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "dataset = fog_x.dataset.Dataset(\n", + " name=\"demo_ds\",\n", + " path='gs://fog_rtx_test/',\n", + ")" + ], + "metadata": { + "id": "pd94S4VlL32u", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "840c1668-983d-4320-f052-34ab77bb5930" + }, + "execution_count": 13, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:fog_x.database.polars_connector:Prepare to load table demo_ds loaded from gs://fog_rtx_test/demo_ds.parquet.\n", + "WARNING:fog_x.database.polars_connector:Failed to load table demo_ds from gs://fog_rtx_test/demo_ds.parquet.\n", + "ERROR:fog_x.database.polars_connector:Table demo_ds does not exist, available tables are dict_keys([]).\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import numpy as np\n", + "\n", + "# create a new trajectory\n", + "episode = dataset.new_episode()\n", + "# collect step data for the episode\n", + "episode.add(feature = \"arm_camera_view\", value = np.random.rand(480, 640, 3))\n", + "episode.add(feature = \"gripper_acton\", value = np.random.rand(7))\n", + "# Automatically time-aligns and saves the trajectory\n", + "episode.close()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Boc13CkhmQEs", + "outputId": "7aa83acf-ce3e-437b-975c-00df0cb999b0" + }, + "execution_count": 14, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:fog_x.database.db_manager:Closing the episode with metadata {'Finished': True, 'arm_camera_view_count': 0, 'gripper_acton_count': 0}\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "dataset2 = fog_x.dataset.Dataset(\n", + " name=\"demo_ds\",\n", + " path='gs://fog_rtx_test/',\n", + ")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "LtzsrO_BtvHB", + "outputId": "5c5c2bec-f769-4bc2-e185-638a42127af6" + }, + "execution_count": 17, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "INFO:fog_x.database.polars_connector:Prepare to load table demo_ds loaded from gs://fog_rtx_test/demo_ds.parquet.\n", + "INFO:fog_x.database.polars_connector:Table demo_ds loaded from gs://fog_rtx_test/demo_ds.parquet.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "dataset2.get_episode_info()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 129 + }, + "id": "95utD8pRtxws", + "outputId": "0871ad47-d812-41fe-8cc6-67bbb77fe10e" + }, + "execution_count": 18, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "shape: (1, 8)\n", + "┌────────────┬──────────┬────────────┬────────────┬────────────┬───────────┬───────────┬───────────┐\n", + "│ episode_id ┆ Finished ┆ feature_ar ┆ feature_ar ┆ arm_camera ┆ feature_g ┆ feature_g ┆ gripper_a │\n", + "│ --- ┆ --- ┆ m_camera_v ┆ m_camera_v ┆ _view_coun ┆ ripper_ac ┆ ripper_ac ┆ cton_coun │\n", + "│ i64 ┆ bool ┆ iew_type ┆ iew_shape ┆ t ┆ ton_type ┆ ton_shape ┆ t │\n", + "│ ┆ ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ ┆ ┆ str ┆ str ┆ f64 ┆ str ┆ str ┆ f64 │\n", + "╞════════════╪══════════╪════════════╪════════════╪════════════╪═══════════╪═══════════╪═══════════╡\n", + "│ 0 ┆ true ┆ float64 ┆ (480, 640, ┆ 0.0 ┆ float64 ┆ (7,) ┆ 0.0 │\n", + "│ ┆ ┆ ┆ 3) ┆ ┆ ┆ ┆ │\n", + "└────────────┴──────────┴────────────┴────────────┴────────────┴───────────┴───────────┴───────────┘" + ], + "text/html": [ + "
\n", + "shape: (1, 8)
episode_idFinishedfeature_arm_camera_view_typefeature_arm_camera_view_shapearm_camera_view_countfeature_gripper_acton_typefeature_gripper_acton_shapegripper_acton_count
i64boolstrstrf64strstrf64
0true"float64""(480, 640, 3)"0.0"float64""(7,)"0.0
" + ] + }, + "metadata": {}, + "execution_count": 18 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Known issues\n", + "\n", + "1. `export` as rlds format to the cloud directly does not work yet for S3 (known issue for tensorflow Gfile)\n", + "2. (will fix) automatically check the existence" + ], + "metadata": { + "id": "P2RCUMs6knNc" + } + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "QKS5jK-Qk9fN" + }, + "execution_count": 14, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/fog_x/dataset.py b/fog_x/dataset.py index 07f25fa..eb99de0 100644 --- a/fog_x/dataset.py +++ b/fog_x/dataset.py @@ -33,6 +33,38 @@ def convert_to_h264(input_file, output_file): ] subprocess.run(command) +def create_cloud_bucket_if_not_exist(provider, bucket_name, dir_name): + logger.info(f"Creating bucket '{bucket_name}' in cloud provider '{provider}' with folder '{dir_name}'...") + if provider == "s3": + import boto3 + s3_client = boto3.client('s3') + # s3_client.create_bucket(Bucket=bucket_name) + s3_client.put_object(Bucket=bucket_name, Key=f"{dir_name}/") + logger.info(f"Bucket '{bucket_name}' created in AWS S3.") + elif provider == "gs": + from google.cloud import storage + """Create a folder in a Google Cloud Storage bucket if it does not exist.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Ensure the folder name ends with a '/' + if not dir_name.endswith('/'): + dir_name += '/' + + # Check if folder exists by trying to list objects with the folder prefix + blobs = storage_client.list_blobs(bucket_name, prefix=dir_name, delimiter='/') + exists = any(blob.name == dir_name for blob in blobs) + + if not exists: + # Create an empty blob to simulate a folder + blob = bucket.blob(dir_name) + blob.upload_from_string('') + print(f"Folder '{dir_name}' created.") + else: + print(f"Folder '{dir_name}' already exists.") + else: + raise ValueError(f"Unsupported cloud provider '{provider}'.") + class Dataset: """ Create or load from a new dataset. @@ -68,7 +100,7 @@ def __init__( * is replace_existing actually used anywhere? """ self.name = name - path = os.path.expanduser(path) + path = os.path.expanduser(path).strip("/") self.path = path if path is None: raise ValueError("Path is required") @@ -90,21 +122,9 @@ def __init__( except: logger.info(f"Path does not exist. ({path}/{name})") cloud_provider = path[:2] - if cloud_provider == "s3": - logger.info(f"Creating {cloud_provider} bucket...") - import boto3 - s3_client = boto3.client('s3') - bucket_name = path[5:] - # s3_client.create_bucket(Bucket=bucket_name) - s3_client.put_object(Bucket=bucket_name, Key=f"{name}/") - logger.info(f"Bucket '{bucket_name}' created in AWS S3.") - # Reinitialize step_data_connector - step_data_connector = LazyFrameConnector(f"{path}/{name}") - elif cloud_provider == "gs": - logger.info(f"Creating {cloud_provider} bucket...") - pass - else: - logger.info(f"Unsupported cloud_provider {cloud_provider}.") + bucket_name = path[5:] + create_cloud_bucket_if_not_exist(cloud_provider, bucket_name, f"{name}/") + step_data_connector = LazyFrameConnector(f"{path}/{name}") self.db_manager = DatabaseManager(episode_info_connector, step_data_connector) self.db_manager.initialize_dataset(self.name, features)