diff --git a/benchmarking/arrow.py b/benchmarking/arrow.py new file mode 100644 index 0000000..6bec4c3 --- /dev/null +++ b/benchmarking/arrow.py @@ -0,0 +1,88 @@ +import os +import sys +import time +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pyarrow.feather as ft +from pyarrow import RecordBatchFileReader as Reader +from pyarrow import RecordBatchFileWriter as Writer + +PATH = os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") + +PATH += "/datasets/" +NAME = "test_convert" +out_dir = PATH + "arrow_convert" + + +def pq_to_arrow(in_dir): + table = pq.read_table(in_dir) + file = in_dir.split("/")[-1].split("-")[0] + + with pa.OSFile(f"{out_dir}/REG/{file}.arrow", "wb") as sink: + with Writer(sink, table.schema) as w: + w.write_table(table) + + with pa.ipc.new_file(f"{out_dir}/IPC/{file}.arrow", table.schema) as w: + w.write_table(table) + + ft.write_feather(table, f"{out_dir}/FTH/{file}.feather", "uncompressed") + + +N = 51 +MB = 1024 * 1024 + +if False: + for i in range(N): + pq_to_arrow(f"{PATH}/{NAME}/{NAME}_{i}-0.parquet") + +def measure_traj(read_func, write_func, name): + read_time, write_time, data_size = 0, 0, 0 + + for i in range(N): + print(f"Measuring trajectory {i}") + + extn = "arrow" if (name[0] == "A") else "feather" + path = f"{out_dir}/{name[-3:]}/{NAME}_{i}.{extn}" + + stop = time.time() + traj = read_func(path) + read_time += time.time() - stop + + data_size += os.path.getsize(path) + path = f"{PATH}/temp.{extn}" + + stop = time.time() + write_func(path, traj) + write_time += time.time() - stop + + os.remove(path) + return read_time, write_time, data_size / MB + + +if __name__ == "__main__": + + reg_dict = {"name": "Arrow_REG", + "read_func": lambda path: Reader(pa.OSFile(path, "rb")).read_all(), + "write_func": lambda path, data: Writer(pa.OSFile(path, "wb"), data.schema).write_table(data) + } + ipc_dict = {"name": "Arrow_IPC", + "read_func": lambda path: pa.ipc.open_file(path).read_all(), + "write_func": lambda path, data: pa.ipc.new_file(path, data.schema).write_table(data) + } + fth_dict = {"name": "Feather_FTH", + "read_func": lambda path: ft.read_table(path), + "write_func": lambda path, data: ft.write_feather(data, path) + } + pd_dict = {"name": "Pandas_FTH", + "read_func": lambda path: pd.read_feather(path), + "write_func": lambda path, data: data.to_feather(path) + } + + for lib in [reg_dict, ipc_dict, fth_dict, pd_dict]: + rt, wt, mb = measure_traj(lib["read_func"], lib["write_func"], lib["name"]) + + print(f"\n{lib['name']}: \nDisk size = {mb:.4f} MB; Num. traj = {N}") + print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") + print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") \ No newline at end of file diff --git a/benchmarking/davidh/PQ_Arrow_Loaders.ipynb b/benchmarking/davidh/PQ_Arrow_Loaders.ipynb new file mode 100644 index 0000000..22ce85b --- /dev/null +++ b/benchmarking/davidh/PQ_Arrow_Loaders.ipynb @@ -0,0 +1,845 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 7, + "id": "608ae1bd-0b68-4e0e-b748-e7f89807724e", + "metadata": {}, + "outputs": [], + "source": [ + "import fog_x \n", + "import os\n", + "from logging import getLogger\n", + "import numpy as np\n", + "import os\n", + "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # or any {'0', '1', '2'}\n", + "import tensorflow as tf\n", + "\n", + "class BaseLoader():\n", + " def __init__(self, data_path):\n", + " super(BaseLoader, self).__init__()\n", + " self.data_dir = data_path\n", + " self.logger = getLogger(__name__)\n", + " self.index = 0\n", + "\n", + "\n", + " def __len__(self):\n", + " raise NotImplementedError\n", + "\n", + " def __iter___(self):\n", + " raise NotImplementedError\n", + "\n", + "class BaseExporter():\n", + " def __init__(self):\n", + " super(BaseExporter, self).__init__()\n", + " self.logger = getLogger(__name__)\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " raise NotImplementedError\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fe6f0124-e66c-4529-8354-12fbc0769de5", + "metadata": {}, + "outputs": [], + "source": [ + "class RTXLoader(BaseLoader):\n", + " def __init__(self, data_path, split):\n", + " super(RTXLoader, self).__init__(data_path)\n", + " import tensorflow_datasets as tfds\n", + "\n", + " builder = tfds.builder_from_directory(data_path)\n", + "\n", + " self.ds = builder.as_dataset(split=split)\n", + " # https://www.determined.ai/blog/tf-dataset-the-bad-parts\n", + "\n", + " def __len__(self):\n", + " return len(self.ds)\n", + " \n", + " def __next__(self):\n", + " if self.index < len(self.ds):\n", + " self.index += 1\n", + " nested_dataset = self.ds.__iter__()\n", + " trajectory = list(nested_dataset)[0][\"steps\"]\n", + " ret = []\n", + " # Iterate through the outer dataset\n", + " for step_data in trajectory:\n", + " step = {}\n", + " for dataset_key, element in step_data.items():\n", + " # print(np.array(element))\n", + " if dataset_key == \"observation\":\n", + " step[\"observation\"] = {}\n", + " for obs_key, obs_element in element.items():\n", + " step[\"observation\"][obs_key] = np.array(obs_element)\n", + " elif dataset_key == \"action\":\n", + " step[\"action\"] = {}\n", + " for action_key, action_element in element.items():\n", + " step[\"action\"][action_key] = np.array(action_element)\n", + " else:\n", + " step[dataset_key] = np.array(element)\n", + " ret.append(step)\n", + " return ret\n", + " else:\n", + " raise StopIteration\n", + " \n", + "\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + "\n", + "class RTXExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(RTXExporter, self).__init__()\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " raise NotImplementedError" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9920acb4", + "metadata": {}, + "outputs": [], + "source": [ + "import av\n", + "import pickle \n", + "\n", + "class MKVLoader(BaseLoader):\n", + " def __init__(self, data_path):\n", + " super(MKVLoader, self).__init__(data_path)\n", + " self.files = [data_path + f for f in os.listdir(data_path) if f.endswith('.mkv')]\n", + " self.index = 0\n", + "\n", + " def __len__(self):\n", + " return len(self.files)\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + " def __next__(self):\n", + " if self.index < len(self):\n", + " result = self.files[self.index]\n", + " self.index += 1\n", + " return self._parse_mkv_file(result)\n", + " else:\n", + " raise StopIteration\n", + " \n", + " def _parse_mkv_file(self, filename):\n", + " print(filename)\n", + " input_container = av.open(filename)\n", + " video_stream1 = input_container.streams.video[0] \n", + " video_stream1.thread_type = 'AUTO'\n", + " video_stream2 = input_container.streams.video[1] \n", + " video_stream2.thread_type = 'AUTO'\n", + " depth_stream = input_container.streams.video[2] \n", + " depth_stream.thread_type = 'AUTO'\n", + " data_stream = input_container.streams[3] \n", + "\n", + " decoded_stream_1 = []\n", + " decoded_stream_2 = []\n", + " decoded_stream_depth = []\n", + " decoded_stream_data = []\n", + "\n", + " pkt_counter = 0\n", + " for packet in input_container.demux(video_stream1, video_stream2, depth_stream, data_stream):\n", + " pkt_counter += 1\n", + " if packet.stream.index == video_stream1.index: \n", + " frame = packet.decode()\n", + " if frame:\n", + " for f in frame:\n", + " image = f.to_ndarray(format='rgb24')\n", + " decoded_stream_1.append(image)\n", + " elif packet.stream.index == video_stream2.index:\n", + " frame = packet.decode()\n", + " if frame:\n", + " for f in frame:\n", + " image = f.to_ndarray(format='rgb24')\n", + " decoded_stream_2.append(image)\n", + " elif packet.stream.index == depth_stream.index:\n", + " frame = packet.decode()\n", + " if frame:\n", + " for f in frame:\n", + " image = f.to_ndarray(format='gray')\n", + " decoded_stream_depth.append(image)\n", + " elif packet.stream.index == data_stream.index:\n", + " packet_in_bytes = bytes(packet)\n", + " if packet_in_bytes:\n", + " non_dict = pickle.loads(packet_in_bytes)\n", + " decoded_stream_data.append(non_dict)\n", + " else:\n", + " print(\"Unknown stream\")\n", + " print(pkt_counter, len(decoded_stream_1), len(decoded_stream_2), len(decoded_stream_depth), len(decoded_stream_data))\n", + " input_container.close()\n", + "\n", + " \n", + "\n", + "class MKVExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(MKVExporter, self).__init__()\n", + "\n", + " # Function to create a frame from numpy array\n", + " def create_frame(self, image_array, stream):\n", + " frame = av.VideoFrame.from_ndarray(np.array(image_array), format='rgb24')\n", + " frame.pict_type = 'NONE'\n", + " frame.time_base = stream.time_base\n", + " return frame\n", + " \n", + " # Function to create a frame from numpy array\n", + " def create_frame_depth(self, image_array, stream):\n", + " image_array = np.array(image_array)\n", + " # if float, convert to uint8\n", + " if image_array.dtype == np.float32:\n", + " image_array = (image_array * 255).astype(np.uint8)\n", + " # if 3 dim, convert to 2 dim\n", + " if len(image_array.shape) == 3:\n", + " image_array = image_array[:,:,0]\n", + " frame = av.VideoFrame.from_ndarray(image_array, format='gray')\n", + " frame.pict_type = 'NONE'\n", + " frame.time_base = stream.time_base\n", + " return frame\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " # Create an output container\n", + " i = -1\n", + " for trajectory in loader:\n", + " i += 1\n", + " output = av.open(f'{output_path}/output_{i}.mkv', mode='w')\n", + " print(f'{output_path}/output_{i}.mkv')\n", + " # Define video streams (assuming images are 640x480 RGB)\n", + " video_stream_1 = output.add_stream('libx264', rate=1)\n", + " video_stream_1.width = 640\n", + " video_stream_1.height = 480\n", + " video_stream_1.pix_fmt = 'yuv420p'\n", + "\n", + " video_stream_2 = output.add_stream('libx264', rate=1)\n", + " video_stream_2.width = 640\n", + " video_stream_2.height = 480\n", + " video_stream_2.pix_fmt = 'yuv420p'\n", + "\n", + " # Define custom data stream for vectors\n", + " depth_stream = output.add_stream('libx264', rate=1)\n", + "\n", + " data_stream = output.add_stream('rawvideo', rate=1)\n", + "\n", + " ts = 0\n", + " # convert step data to stream\n", + " for step in trajectory:\n", + " obesrvation = step[\"observation\"].copy()\n", + " obesrvation.pop(\"image\")\n", + " obesrvation.pop(\"hand_image\")\n", + " obesrvation.pop(\"image_with_depth\")\n", + " non_image_data_step = step.copy()\n", + " non_image_data_step[\"observation\"] = obesrvation\n", + "\n", + " non_image_data_bytes = pickle.dumps(non_image_data_step)\n", + " packet = av.Packet(non_image_data_bytes)\n", + " packet.stream = data_stream\n", + " packet.pts = ts\n", + " output.mux(packet)\n", + "\n", + " image =np.array(step[\"observation\"][\"image\"])\n", + " # Create a frame from the numpy array\n", + " frame = self.create_frame(image, video_stream_1)\n", + " frame.pts = ts\n", + " packet = video_stream_1.encode(frame)\n", + " \n", + " output.mux(packet)\n", + "\n", + " hand_image =np.array(step[\"observation\"][\"hand_image\"])\n", + " # Create a frame from the numpy array\n", + " frame = self.create_frame(hand_image, video_stream_2)\n", + " frame.pts = ts\n", + " packet = video_stream_2.encode(frame)\n", + " output.mux(packet)\n", + "\n", + " # # Create a frame from the numpy array\n", + " frame = self.create_frame_depth(step[\"observation\"][\"image_with_depth\"], depth_stream)\n", + " # frame.pts = ts\n", + " # Encode the frame\n", + " packet = depth_stream.encode(frame)\n", + " # Write the packet to the output file\n", + " output.mux(packet)\n", + "\n", + " ts += 1\n", + "\n", + " \n", + " # Flush the remaining frames\n", + " for packet in video_stream_1.encode():\n", + " output.mux(packet)\n", + "\n", + " for packet in video_stream_2.encode():\n", + " output.mux(packet)\n", + "\n", + " for packet in depth_stream.encode():\n", + " output.mux(packet)\n", + " output.close()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "89164adb", + "metadata": {}, + "outputs": [], + "source": [ + "number_of_samples = 1\n", + "def iterate_dataset(loader: BaseLoader, number_of_samples):\n", + " for i, data in enumerate(loader): \n", + " list(dict(data)[\"steps\"])\n", + " if i == number_of_samples:\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "1f55f705", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I 2024-06-23 06:05:40,410 dataset_info.py:595] Load dataset info from /home/davidh/datasets/berkeley_autolab_ur5/0.1.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I 2024-06-23 06:05:40,525 logging_logger.py:49] Constructing tf.data.Dataset berkeley_autolab_ur5 for split train[:1], from /home/davidh/datasets/berkeley_autolab_ur5/0.1.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exported /home/davidh/fog_x_fork/examples/dataloader/pq_output/output_0.parquet\n" + ] + } + ], + "source": [ + "import os\n", + "import pickle\n", + "import pandas as pd\n", + "import fastparquet as fp\n", + "import numpy as np\n", + "from logging import getLogger\n", + "import fog_x\n", + "\n", + "\n", + "class ParquetExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(ParquetExporter, self).__init__()\n", + " self.logger = getLogger(__name__)\n", + "\n", + " def _serialize(self, data):\n", + " return pickle.dumps(np.array(data))\n", + "\n", + " def _step_data_to_df(self, step_data):\n", + " step = {}\n", + " for dataset_key, element in step_data.items():\n", + " if dataset_key == \"observation\":\n", + " for obs_key, obs_element in element.items():\n", + " step[obs_key] = self._serialize(obs_element)\n", + " elif dataset_key == \"action\":\n", + " for action_key, action_element in element.items():\n", + " step[action_key] = self._serialize(action_element)\n", + " else:\n", + " step[dataset_key] = self._serialize(element)\n", + " return step\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " if not os.path.exists(output_path):\n", + " os.makedirs(output_path)\n", + " i = 0\n", + " for steps_data in loader:\n", + " step_data_as_df = [self._step_data_to_df(step_data) for step_data in steps_data]\n", + " combined_df = pd.DataFrame(step_data_as_df)\n", + " output_file = os.path.join(output_path, f'output_{i}.parquet')\n", + " fp.write(output_file, combined_df)\n", + " print(f'Exported {output_file}')\n", + " i += 1\n", + "\n", + "\n", + "class ParquetLoader(BaseLoader):\n", + " def __init__(self, data_path):\n", + " super(ParquetLoader, self).__init__(data_path)\n", + " self.files = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.parquet')]\n", + " self.index = 0\n", + "\n", + " def __len__(self):\n", + " return len(self.files)\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + " def __next__(self):\n", + " if self.index < len(self.files):\n", + " result = self._load_file(self.files[self.index])\n", + " self.index += 1\n", + " return result\n", + " else:\n", + " raise StopIteration\n", + "\n", + " def _parse_parquet_file(self, filename):\n", + " df = fp.ParquetFile(filename).to_pandas()\n", + " steps = [dict(row) for _, row in df.iterrows()]\n", + " return steps\n", + "\n", + "number_of_samples = 1\n", + "\n", + "rtx_loader = RTXLoader(os.path.expanduser(\"~/datasets/berkeley_autolab_ur5/0.1.0\"), split=f'train[:{number_of_samples}]')\n", + "\n", + "output_path = \"/home/davidh/fog_x_fork/examples/dataloader/pq_output\"\n", + "os.makedirs(output_path, exist_ok=True)\n", + "\n", + "exporter = ParquetExporter()\n", + "exporter.export(rtx_loader, output_path)\n", + "\n", + "parquet_loader = ParquetLoader(output_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6b68d1eb", + "metadata": {}, + "outputs": [ + { + "ename": "ImportError", + "evalue": "cannot import name 'BaseExporter' from 'fog_x' (/home/davidh/miniconda3/envs/fog_x_env/lib/python3.8/site-packages/fog_x/__init__.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[16], line 7\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mlogging\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m getLogger\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mfog_x\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BaseExporter, BaseLoader\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mFeatherExporter\u001b[39;00m(BaseExporter):\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'BaseExporter' from 'fog_x' (/home/davidh/miniconda3/envs/fog_x_env/lib/python3.8/site-packages/fog_x/__init__.py)" + ] + } + ], + "source": [ + "import os\n", + "import pickle\n", + "import pandas as pd\n", + "import pyarrow.feather as feather\n", + "import numpy as np\n", + "from logging import getLogger\n", + "\n", + "class FeatherExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(FeatherExporter, self).__init__()\n", + " self.logger = getLogger(__name__)\n", + "\n", + " def _serialize(self, data):\n", + " return pickle.dumps(np.array(data))\n", + "\n", + " def _step_data_to_df(self, step_data):\n", + " step = {}\n", + " for dataset_key, element in step_data.items():\n", + " if dataset_key == \"observation\":\n", + " for obs_key, obs_element in element.items():\n", + " step[obs_key] = self._serialize(obs_element)\n", + " elif dataset_key == \"action\":\n", + " for action_key, action_element in element.items():\n", + " step[action_key] = self._serialize(action_element)\n", + " else:\n", + " step[dataset_key] = self._serialize(element)\n", + " return step\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " if not os.path.exists(output_path):\n", + " os.makedirs(output_path)\n", + " i = 0\n", + " for steps_data in loader:\n", + " step_data_as_df = [self._step_data_to_df(step_data) for step_data in steps_data]\n", + " combined_df = pd.DataFrame(step_data_as_df)\n", + " output_file = os.path.join(output_path, f'output_{i}.feather')\n", + " feather.write_feather(combined_df, output_file)\n", + " print(f'Exported {output_file}')\n", + " i += 1\n", + "import os\n", + "import pickle\n", + "import pandas as pd\n", + "import pyarrow.feather as feather\n", + "import numpy as np\n", + "from logging import getLogger\n", + "\n", + "class FeatherLoader(BaseLoader):\n", + " def __init__(self, data_path):\n", + " super(FeatherLoader, self).__init__(data_path)\n", + " self.files = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.feather')]\n", + " self.index = 0\n", + "\n", + " def __len__(self):\n", + " return len(self.files)\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + " def __next__(self):\n", + " if self.index < len(self.files):\n", + " result = self._load_file(self.files[self.index])\n", + " self.index += 1\n", + " return result\n", + " else:\n", + " raise StopIteration\n", + "\n", + " def _load_file(self, filename):\n", + " df = feather.read_feather(filename)\n", + " steps = [dict(row) for _, row in df.iterrows()]\n", + " return steps\n", + "import os\n", + "from fog_x import RTXLoader\n", + "\n", + "number_of_samples = 1\n", + "\n", + "rtx_loader = RTXLoader(os.path.expanduser(\"~/datasets/berkeley_autolab_ur5/0.1.0\"), split=f'train[:{number_of_samples}]')\n", + "\n", + "output_path = os.path.expanduser(\"~\") + \"/fog_x/examples/dataloader/feather_output/\"\n", + "os.makedirs(output_path, exist_ok=True)\n", + "exporter = FeatherExporter()\n", + "exporter.export(rtx_loader, output_path)\n", + "\n", + "feather_loader = FeatherLoader(output_path)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "7364020c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "218424901" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.path.getsize(\"/home/davidh/fog_x_fork/examples/dataloader/pq_output/output_0.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "d4566c75", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I 2024-06-23 06:09:07,749 dataset_info.py:595] Load dataset info from /home/davidh/datasets/berkeley_autolab_ur5/0.1.0\n", + "I 2024-06-23 06:09:07,996 logging_logger.py:49] Constructing tf.data.Dataset berkeley_autolab_ur5 for split train[:1], from /home/davidh/datasets/berkeley_autolab_ur5/0.1.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exported /home/davidh/fog_x/examples/dataloader/feather_output/output_0.feather\n" + ] + } + ], + "source": [ + "import os\n", + "import pickle\n", + "import pandas as pd\n", + "import pyarrow.feather as feather\n", + "import numpy as np\n", + "from logging import getLogger\n", + "\n", + "\n", + "class FeatherExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(FeatherExporter, self).__init__()\n", + " self.logger = getLogger(__name__)\n", + "\n", + " def _serialize(self, data):\n", + " return pickle.dumps(np.array(data))\n", + "\n", + " def _step_data_to_df(self, step_data):\n", + " step = {}\n", + " for dataset_key, element in step_data.items():\n", + " if dataset_key == \"observation\":\n", + " for obs_key, obs_element in element.items():\n", + " step[obs_key] = self._serialize(obs_element)\n", + " elif dataset_key == \"action\":\n", + " for action_key, action_element in element.items():\n", + " step[action_key] = self._serialize(action_element)\n", + " else:\n", + " step[dataset_key] = self._serialize(element)\n", + " return step\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " if not os.path.exists(output_path):\n", + " os.makedirs(output_path)\n", + " i = 0\n", + " for steps_data in loader:\n", + " step_data_as_df = [self._step_data_to_df(step_data) for step_data in steps_data]\n", + " combined_df = pd.DataFrame(step_data_as_df)\n", + " output_file = os.path.join(output_path, f'output_{i}.feather')\n", + " feather.write_feather(combined_df, output_file)\n", + " print(f'Exported {output_file}')\n", + " i += 1\n", + "\n", + "class FeatherLoader(BaseLoader):\n", + " def __init__(self, data_path):\n", + " super(FeatherLoader, self).__init__(data_path)\n", + " self.files = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.feather')]\n", + " self.index = 0\n", + "\n", + " def __len__(self):\n", + " return len(self.files)\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + " def __next__(self):\n", + " if self.index < len(self.files):\n", + " result = self._load_file(self.files[self.index])\n", + " self.index += 1\n", + " return result\n", + " else:\n", + " raise StopIteration\n", + "\n", + " def _load_file(self, filename):\n", + " df = feather.read_feather(filename)\n", + " steps = [dict(row) for _, row in df.iterrows()]\n", + " return steps\n", + "\n", + "number_of_samples = 1\n", + "\n", + "rtx_loader = RTXLoader(os.path.expanduser(\"~/datasets/berkeley_autolab_ur5/0.1.0\"), split=f'train[:{number_of_samples}]')\n", + "\n", + "output_path = os.path.expanduser(\"~\") + \"/fog_x/examples/dataloader/feather_output/\"\n", + "os.makedirs(output_path, exist_ok=True)\n", + "\n", + "exporter = FeatherExporter()\n", + "exporter.export(rtx_loader, output_path)\n", + "\n", + "feather_loader = FeatherLoader(output_path)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "53c1529a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "91502858" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.path.getsize(\"/home/davidh/fog_x/examples/dataloader/feather_output/output_0.feather\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d56f44d", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "class ParquetLoader():\n", + " def __init__(self, data_path):\n", + " super(ParquetLoader, self).__init__()\n", + " self.data_dir = data_path\n", + " self.logger = getLogger(__name__)\n", + "\n", + " def __len__(self):\n", + " raise NotImplementedError\n", + "\n", + " def __iter___(self):\n", + " raise NotImplementedError\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "80867561", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "exporter = MKVExporter()\n", + "output_path = \"/home/davidh/fog_x_fork/examples/dataloader/mkv_output/\"\n", + "exporter.export(rtx_loader, output_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fcfc1bdb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/davidh/fog_x_fork/examples/dataloader/mkv_output/output_11.mkv\n", + "408 101 101 101 101\n", + "/home/davidh/fog_x_fork/examples/dataloader/mkv_output/output_16.mkv\n", + "436 108 108 108 108\n" + ] + } + ], + "source": [ + "mkv_loader = MKVLoader(output_path)\n", + "def iterate_dataset(loader: BaseLoader, number_of_samples = 50):\n", + " for i, data in enumerate(loader): \n", + " if i == number_of_samples:\n", + " break\n", + "iterate_dataset(mkv_loader, number_of_samples)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0822a45a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "30ba398b", + "metadata": {}, + "outputs": [ + { + "ename": "ImportError", + "evalue": "cannot import name 'RTXLoader' from 'fog_x' (/home/davidh/miniconda3/envs/fog_x_env/lib/python3.8/site-packages/fog_x/__init__.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[9], line 7\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mparquet\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpq\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mfog_x\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RTXLoader, BaseLoader, BaseExporter\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mParquetExporter\u001b[39;00m(BaseExporter):\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'RTXLoader' from 'fog_x' (/home/davidh/miniconda3/envs/fog_x_env/lib/python3.8/site-packages/fog_x/__init__.py)" + ] + } + ], + "source": [ + "import os\n", + "import pickle\n", + "import pandas as pd\n", + "import pyarrow as pa\n", + "import pyarrow.parquet as pq\n", + "import numpy as np\n", + "from fog_x import RTXLoader, BaseLoader, BaseExporter\n", + "\n", + "class ParquetExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(ParquetExporter, self).__init__()\n", + "\n", + " def _serialize(self, data):\n", + " return pickle.dumps(np.array(data))\n", + "\n", + " def _step_data_to_df(self, step_data):\n", + " step = {}\n", + " for dataset_key, element in step_data.items():\n", + " if dataset_key == \"observation\":\n", + " for obs_key, obs_element in element.items():\n", + " step[obs_key] = self._serialize(obs_element)\n", + " elif dataset_key == \"action\":\n", + " for action_key, action_element in element.items():\n", + " step[action_key] = self._serialize(action_element)\n", + " else:\n", + " step[dataset_key] = self._serialize(element)\n", + " return step\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " for steps_data in loader:\n", + " step_data_as_df = [self._step_data_to_df(step_data) for step_data in steps_data]\n", + " combined_df = pd.DataFrame(step_data_as_df)\n", + " table = pa.Table.from_pandas(combined_df)\n", + " pq.write_table(table, output_path)\n", + "\n", + "class ParquetLoader(BaseLoader):\n", + " def __init__(self, data_path):\n", + " super(ParquetLoader, self).__init__(data_path)\n", + " self.files = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.parquet')]\n", + " self.index = 0\n", + "\n", + " def __len__(self):\n", + " return len(self.files)\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + " def __next__(self):\n", + " if self.index < len(self.files):\n", + " result = self._load_file(self.files[self.index])\n", + " self.index += 1\n", + " return result\n", + " else:\n", + " raise StopIteration\n", + "\n", + " def _load_file(self, filename):\n", + " table = pq.read_table(filename)\n", + " df = table.to_pandas()\n", + " steps = [dict(row) for _, row in df.iterrows()]\n", + " return steps\n", + "\n", + "# Example usage\n", + "rtx_loader = RTXLoader(os.path.expanduser(\"~/datasets/berkeley_autolab_ur5/0.1.0\"), split='train[:51]')\n", + "exporter = ParquetExporter()\n", + "exporter.export(rtx_loader, \"output.parquet\")\n", + "\n", + "parquet_loader = ParquetLoader(\"/home/davidh/fog_x_fork/benchmarking/\")\n", + "print(parquet_loader._load_file(\"/home/davidh/fog_x_fork/benchmarking/output.parquet\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "359bb163", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/benchmarking/davidh/pq_arrow_loaders.py b/benchmarking/davidh/pq_arrow_loaders.py new file mode 100644 index 0000000..9f3b3ea --- /dev/null +++ b/benchmarking/davidh/pq_arrow_loaders.py @@ -0,0 +1,259 @@ +import os +import time +from logging import getLogger +import numpy as np +import pickle +import av +import pandas as pd +import pyarrow.feather as feather +import pyarrow as pa +import pyarrow.parquet as pq +import tensorflow_datasets as tfds +import tensorflow as tf +import fastparquet as fp + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # or any {'0', '1', '2'} + +class BaseLoader(): + def __init__(self, data_path): + super(BaseLoader, self).__init__() + self.data_dir = data_path + self.logger = getLogger(__name__) + self.index = 0 + + def __len__(self): + raise NotImplementedError + + def __iter___(self): + raise NotImplementedError + + +class BaseExporter(): + def __init__(self): + super(BaseExporter, self).__init__() + self.logger = getLogger(__name__) + + def export(self, loader: BaseLoader, output_path: str): + raise NotImplementedError + +class RTXLoader(BaseLoader): + def __init__(self, data_path, split): + super(RTXLoader, self).__init__(data_path) + builder = tfds.builder_from_directory(data_path) + self.ds = builder.as_dataset(split=split) + + def __len__(self): + return len(self.ds) + + def __next__(self): + if self.index < len(self.ds): + self.index += 1 + nested_dataset = self.ds.__iter__() + trajectory = list(nested_dataset)[0]["steps"] + ret = [] + for step_data in trajectory: + step = {} + for dataset_key, element in step_data.items(): + if dataset_key == "observation": + step["observation"] = {} + for obs_key, obs_element in element.items(): + step["observation"][obs_key] = np.array(obs_element) + elif dataset_key == "action": + step["action"] = {} + for action_key, action_element in element.items(): + step["action"][action_key] = np.array(action_element) + else: + step[dataset_key] = np.array(element) + ret.append(step) + return ret + else: + raise StopIteration + + def __iter__(self): + return self + +class RTXExporter(BaseExporter): + def __init__(self): + super(RTXExporter, self).__init__() + + def export(self, loader: BaseLoader, output_path: str): + raise NotImplementedError + + +class ParquetExporter(BaseExporter): + def __init__(self): + super(ParquetExporter, self).__init__() + self.logger = getLogger(__name__) + + def _serialize(self, data): + return pickle.dumps(np.array(data)) + + def _step_data_to_df(self, step_data): + step = {} + for dataset_key, element in step_data.items(): + if dataset_key == "observation": + for obs_key, obs_element in element.items(): + step[obs_key] = self._serialize(obs_element) + elif dataset_key == "action": + for action_key, action_element in element.items(): + step[action_key] = self._serialize(action_element) + else: + step[dataset_key] = self._serialize(element) + return step + + def export(self, loader: BaseLoader, output_path: str): + if not os.path.exists(output_path): + os.makedirs(output_path) + i = 0 + for steps_data in loader: + step_data_as_df = [self._step_data_to_df(step_data) for step_data in steps_data] + combined_df = pd.DataFrame(step_data_as_df) + output_file = os.path.join(output_path, f'output_{i}.parquet') + fp.write(output_file, combined_df) + print(f'Exported {output_file}') + i += 1 + +class ParquetLoader(BaseLoader): + def __init__(self, data_path): + super(ParquetLoader, self).__init__(data_path) + self.files = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.parquet')] + self.index = 0 + + def __len__(self): + return len(self.files) + + def __iter__(self): + return self + + def __next__(self): + if self.index < len(self.files): + result = self._load_file(self.files[self.index]) + self.index += 1 + return result + else: + raise StopIteration + + def _load_file(self, filename): + table = pq.read_table(filename) + df = table.to_pandas() + steps = [dict(row) for _, row in df.iterrows()] + return steps + +class FeatherExporter(BaseExporter): + def __init__(self): + super(FeatherExporter, self).__init__() + self.logger = getLogger(__name__) + + def _serialize(self, data): + return pickle.dumps(np.array(data)) + + def _step_data_to_df(self, step_data): + step = {} + for dataset_key, element in step_data.items(): + if dataset_key == "observation": + for obs_key, obs_element in element.items(): + step[obs_key] = self._serialize(obs_element) + elif dataset_key == "action": + for action_key, action_element in element.items(): + step[action_key] = self._serialize(action_element) + else: + step[dataset_key] = self._serialize(element) + return step + + def export(self, loader: BaseLoader, output_path: str): + if not os.path.exists(output_path): + os.makedirs(output_path) + i = 0 + for steps_data in loader: + step_data_as_df = [self._step_data_to_df(step_data) for step_data in steps_data] + combined_df = pd.DataFrame(step_data_as_df) + output_file = os.path.join(output_path, f'output_{i}.feather') + feather.write_feather(combined_df, output_file) + print(f'Exported {output_file}') + i += 1 + +class FeatherLoader(BaseLoader): + def __init__(self, data_path): + super(FeatherLoader, self).__init__(data_path) + self.files = [os.path.join(data_path, f) for f in os.listdir(data_path) if f.endswith('.feather')] + self.index = 0 + + def __len__(self): + return len(self.files) + + def __iter__(self): + return self + + def __next__(self): + if self.index < len(self.files): + result = self._load_file(self.files[self.index]) + self.index += 1 + return result + else: + raise StopIteration + + def _load_file(self, filename): + df = feather.read_feather(filename) + steps = [dict(row) for _, row in df.iterrows()] + return steps + +def iterate_parquet(rtx_loader, parquet_exporter, n): + read_time, write_time, data_size = 0, 0, 0 + path = os.path.expanduser("~/fog_x_fork/examples/dataloader/pq_output/") + os.makedirs(path, exist_ok=True) + + stop = time.time() + parquet_exporter.export(rtx_loader, path) + write_time += time.time() - stop + + parquet_loader = ParquetLoader(path) + + stop = time.time() + for i, data in enumerate(parquet_loader): + print(f"Reading trajectory {i}") + if i == n: break + read_time += time.time() - stop + + for i in range(n): + data_size += os.path.getsize(f"{path}/output_{i}.parquet") + + return read_time, write_time, data_size / 1024**2 + +def iterate_feather(rtx_loader, feather_exporter, n): + read_time, write_time, data_size = 0, 0, 0 + path = os.path.expanduser("~/fog_x/examples/dataloader/feather_output/") + os.makedirs(path, exist_ok=True) + + stop = time.time() + feather_exporter.export(rtx_loader, path) + write_time += time.time() - stop + + feather_loader = FeatherLoader(path) + + stop = time.time() + for i, data in enumerate(feather_loader): + print(f"Reading trajectory {i}") + if i == n: break + read_time += time.time() - stop + + for i in range(n): + data_size += os.path.getsize(f"{path}/output_{i}.feather") + + return read_time, write_time, data_size / 1024**2 + +# parquet use case +rtx_loader = RTXLoader(os.path.expanduser("~/datasets/berkeley_autolab_ur5/0.1.0"), split='train[:51]') +parquet_exporter = ParquetExporter() +rt, wt, mb = iterate_parquet(rtx_loader, parquet_exporter, 51) + +print(f"\nParquet Data: \nMem. size = {mb:.4f} MB; Num. traj = 51") +print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {51 / rt :.4f} traj/s") +print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {51 / wt :.4f} traj/s") + +# feather pyarrow version use case +feather_exporter = FeatherExporter() +rt, wt, mb = iterate_feather(rtx_loader, feather_exporter, 51) + +print(f"\nFeather Data: \nMem. size = {mb:.4f} MB; Num. traj = 51") +print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {51 / rt :.4f} traj/s") +print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {51 / wt :.4f} traj/s") diff --git a/benchmarking/davidh/tf_record.ipynb b/benchmarking/davidh/tf_record.ipynb new file mode 100644 index 0000000..58e3845 --- /dev/null +++ b/benchmarking/davidh/tf_record.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Measuring read and write speed of TFrecord" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import time\n", + "import os\n", + "\n", + "read_files = [\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00000-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00001-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00002-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00003-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00004-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00005-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00006-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00007-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00008-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00009-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00010-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00011-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00012-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00013-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00014-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00015-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00016-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00017-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00018-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00019-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00020-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00021-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00022-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00023-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00024-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00025-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00026-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00027-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00028-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00029-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00030-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00031-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00032-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00033-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00034-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00035-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00036-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00037-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00038-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00039-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00040-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00041-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00042-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00043-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00044-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00045-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00046-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00047-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00048-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00049-of-00050'\n", + "]\n", + "\n", + "write_file = '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test_write_test.tfrecord'\n", + "\n", + "def measure_read_speed(files):\n", + " start_time = time.time()\n", + " total_bytes = 0\n", + "\n", + " for file_path in files:\n", + " for record in tf.data.TFRecordDataset(file_path):\n", + " # Measure the speed of converting the record to numpy + for loop time\n", + " record.numpy()\n", + " total_bytes += len(record.numpy())\n", + "\n", + " end_time = time.time()\n", + " elapsed_time = end_time - start_time\n", + " read_speed = total_bytes / elapsed_time / (1024 * 1024)\n", + "\n", + " print(f\"Read {total_bytes} bytes in {elapsed_time:.2f} seconds\")\n", + " print(f\"Read Speed: {read_speed:.2f} MB/s\")\n", + " print(f\"Read Latency: {elapsed_time:.2f} seconds\")\n", + "\n", + "\n", + "def measure_write_speed(files, output_file):\n", + " start_time = time.time()\n", + " total_bytes = 0\n", + "\n", + " with tf.io.TFRecordWriter(output_file) as writer:\n", + " for file_path in files:\n", + " for record in tf.data.TFRecordDataset(file_path):\n", + " writer.write(record.numpy())\n", + " total_bytes += len(record.numpy())\n", + "\n", + " end_time = time.time()\n", + " elapsed_time = end_time - start_time\n", + " write_speed = total_bytes / elapsed_time / (1024 * 1024) \n", + "\n", + " print(f\"Wrote {total_bytes} bytes in {elapsed_time:.2f} seconds\")\n", + " print(f\"Write Speed: {write_speed:.2f} MB/s\")\n", + " print(f\"Write Latency: {elapsed_time:.2f} seconds\")\n", + "\n", + "\n", + "print(\"Measuring Read Speed...\")\n", + "measure_read_speed(read_files)\n", + "print(\"Measuring Write Speed...\")\n", + "measure_write_speed(read_files, write_file)\n", + "os.remove(write_file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "from PIL import Image\n", + "from IPython.display import display as ip_display\n", + "import time\n", + "import os\n", + "\n", + "exit()\n", + "\n", + "read_files = [\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00000-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00001-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00002-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00003-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00004-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00005-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00006-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00007-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00008-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00009-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00010-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00011-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00012-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00013-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00014-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00015-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00016-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00017-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00018-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00019-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00020-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00021-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00022-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00023-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00024-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00025-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00026-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00027-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00028-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00029-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00030-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00031-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00032-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00033-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00034-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00035-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00036-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00037-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00038-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00039-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00040-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00041-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00042-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00043-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00044-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00045-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00046-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00047-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00048-of-00050',\n", + " '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test.tfrecord-00049-of-00050'\n", + "]\n", + "\n", + "write_file = '/home/davidh/datasets/berkeley_autolab_ur5/berkeley_autolab_ur5-test_write_test.tfrecord'\n", + "\n", + "def dataset2path(dataset_name):\n", + " if dataset_name == 'robo_net':\n", + " version = '1.0.0'\n", + " elif dataset_name == 'language_table':\n", + " version = '0.0.1'\n", + " else:\n", + " version = '0.1.0'\n", + " return f'gs://gresearch/robotics/{dataset_name}/{version}'\n", + "\n", + "def measure_specifying_path_latency(files):\n", + " start_time = time.time()\n", + " for file_path in files:\n", + " _ = file_path\n", + " end_time = time.time()\n", + " latency = end_time - start_time\n", + " return latency\n", + "\n", + "def measure_getting_image_latency_and_display(dataset_name):\n", + " dataset_path = dataset2path(dataset_name)\n", + " b = tfds.builder_from_directory(builder_dir=dataset_path)\n", + " if 'image' not in b.info.features['steps']['observation']:\n", + " raise ValueError(\n", + " f\"The key 'image' was not found in this dataset.\\n\"\n", + " \"Please choose a different image key to display for this dataset.\\n\"\n", + " \"Here is the observation spec:\\n\"\n", + " + str(b.info.features['steps']['observation']))\n", + "\n", + " start_time = time.time()\n", + " ds = b.as_dataset(split='train[:1]').shuffle(1) # take only the first episode\n", + " episode = next(iter(ds))\n", + " images = [step['observation']['image'] for step in episode['steps']]\n", + " images = [Image.fromarray(image.numpy()) for image in images]\n", + " end_time = time.time()\n", + " latency = end_time - start_time\n", + "\n", + " # Display some first image\n", + " ip_display(images[0])\n", + "\n", + " return latency\n", + "\n", + "def measure_read_speed(files):\n", + " start_time = time.time()\n", + " total_bytes = 0\n", + " total_trajectories = 0\n", + "\n", + " for file_path in files:\n", + " for record in tf.data.TFRecordDataset(file_path):\n", + " record.numpy()\n", + " total_bytes += len(record.numpy())\n", + " total_trajectories += 1\n", + "\n", + " end_time = time.time()\n", + " elapsed_time = end_time - start_time\n", + " read_speed = total_bytes / elapsed_time / (1024 * 1024) # conversion to mb/s\n", + " read_throughput = total_trajectories / elapsed_time # trajectories/second\n", + "\n", + " print(f\"Read {total_bytes} bytes in {elapsed_time:.2f} seconds\")\n", + " print(f\"Read Speed: {read_speed:.2f} MB/s\")\n", + " print(f\"Read Latency: {elapsed_time:.2f} seconds\")\n", + " print(f\"Read Throughput: {read_throughput:.2f} trajectories/second\")\n", + "\n", + "def measure_write_speed(files, output_file):\n", + " start_time = time.time()\n", + " total_bytes = 0\n", + " total_trajectories = 0\n", + "\n", + " with tf.io.TFRecordWriter(output_file) as writer:\n", + " for file_path in files:\n", + " for record in tf.data.TFRecordDataset(file_path):\n", + " writer.write(record.numpy())\n", + " total_bytes += len(record.numpy())\n", + " total_trajectories += 1\n", + "\n", + " end_time = time.time()\n", + " elapsed_time = end_time - start_time\n", + " write_speed = total_bytes / elapsed_time / (1024 * 1024) # conversion to MB/S\n", + " write_throughput = total_trajectories / elapsed_time # conversion to trajectories/second\n", + "\n", + " print(f\"Wrote {total_bytes} bytes in {elapsed_time:.2f} seconds\")\n", + " print(f\"Write Speed: {write_speed:.2f} MB/s\")\n", + " print(f\"Write Latency: {elapsed_time:.2f} seconds\")\n", + " print(f\"Write Throughput: {write_throughput:.2f} trajectories/second\")\n", + "\n", + "print(\"Measuring Specifying Path Latency...\")\n", + "path_latency = measure_specifying_path_latency(read_files)\n", + "print(f\"Specifying Path Latency: {path_latency:.4f} seconds\")\n", + "\n", + "print(\"Measuring Getting Image Latency and Displaying Image...\")\n", + "image_latency = measure_getting_image_latency_and_display('berkeley_autolab_ur5')\n", + "print(f\"Getting Image Latency: {image_latency:.4f} seconds\")\n", + "\n", + "print(f\"Actual Image Fetch Latency: {image_latency - path_latency:.4f} seconds\")\n", + "\n", + "print(\"Measuring Read Speed...\")\n", + "measure_read_speed(read_files)\n", + "print(\"Measuring Write Speed...\")\n", + "measure_write_speed(read_files, write_file)\n", + "\n", + "os.remove(write_file)\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarking/ffmpeg_loaders.py b/benchmarking/ffmpeg_loaders.py new file mode 100644 index 0000000..534e01e --- /dev/null +++ b/benchmarking/ffmpeg_loaders.py @@ -0,0 +1,348 @@ +import os +import av +import sys +import time +import pickle +import numpy as np +from logging import getLogger + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = "2" +import tensorflow_datasets as tfds +import tensorflow as tf + +PATH = os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") +PATH += "/datasets/" + + +class BaseLoader(): + def __init__(self, path): + super(BaseLoader, self).__init__() + self.path = path + self.logger = getLogger(__name__) + + def __len__(self): + raise NotImplementedError + + def __iter___(self): + raise NotImplementedError + + +class BaseWriter(): + def __init__(self): + super(BaseWriter, self).__init__() + self.logger = getLogger(__name__) + + def export(self, loader, path): + raise NotImplementedError + + +class RTXLoader(BaseLoader): + + def __init__(self, path, split): + super(RTXLoader, self).__init__(path) + builder = tfds.builder_from_directory(path) + self.ds = builder.as_dataset(split) + self.index = 0 + + def __len__(self): + return len(self.ds) + + def __next__(self): + if self.index < len(self.ds): + self.index += 1 + nest_ds = self.ds.__iter__() + traj = list(nest_ds)[0]["steps"] + data = [] + + for step_data in traj: + step = {} + for key, val in step_data.items(): + + if key == "observation": + step["observation"] = {} + for obs_key, obs_val in val.items(): + step["observation"][obs_key] = np.array(obs_val) + + elif key == "action": + step["action"] = {} + for act_key, act_val in val.items(): + step["action"][act_key] = np.array(act_val) + else: + step[key] = np.array(val) + + data.append(step) + return data + else: + self.index = 0 + raise StopIteration + + def __iter__(self): + return self + + +class RTXWriter(BaseWriter): + + def __init__(self): + super(RTXWriter, self).__init__() + self.wr = tf.io.TFRecordWriter + + def export(self, loader, path): + for i, data in enumerate(loader): + # data_list = list(data["steps"].as_numpy_iterator()) + # data_flat = flatten(data_list) + + with self.wr(f"{path}/output_{i}.tfrecord") as writer: + for dl in data: + writer.write(dl) + + def export_temp(self, data, path): + with self.wr(path + "/temp.tfrecord") as writer: + for dl in data: + writer.write(dl) + + +N = 51 +MB = 1024 * 1024 + +def flatten(data): + if isinstance(data, dict): + data = list(data.values()) + if isinstance(data, list): + return [x for lst in data for x in flatten(lst)] + if isinstance(data, bytes): + return [data] + return [data.tobytes()] + + +def iterate_rtx(num_iters): + read_time, write_time, data_size = 0, 0, 0 + + # Read time + stop = time.time() + rtx_loader = RTXLoader(PATH + "berkeley_autolab_ur5", f"train[:{N}]") + + for i, data in enumerate(rtx_loader): + print(f"Reading trajectory {i}") + if i == num_iters: + break + read_time = time.time() - stop + + # Write time + rtx_writer = RTXWriter() + for i, data in enumerate(rtx_loader): + print(f"Writing trajectory {i}") + flat = flatten(data) + + stop = time.time() + rtx_writer.export_temp(flat, PATH) + write_time += time.time() - stop + if i == num_iters: + break + + # Data size + for i in range(num_iters): + name = "berkeley_autolab_ur5" + num = f"{str(i).zfill(5)}-of-00412" + data_size += os.path.getsize(f"{PATH}/{name}/{name}-train.tfrecord-{num}") + + return read_time, write_time, data_size / MB + + +rt, wt, mb = iterate_rtx(N) +print(f"\nRTX Data: \nMem. size = {mb:.4f} MB; Num. traj = {N}") +print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") +print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") + + +class MKVLoader(BaseLoader): + + def __init__(self, path): + super(MKVLoader, self).__init__(path) + self.files = [path + f for f in os.listdir(path) if f.endswith(".mkv")] + self.index = 0 + + def __len__(self): + return len(self.files) + + def __iter__(self): + return self + + def __next__(self): + if self.index < len(self): + result = self.files[self.index] + self.index += 1 + return self._parse_mkv_file(result) + else: + raise StopIteration + + def _parse_mkv_file(self, file): + inp_cont = av.open(file) + print(file) + + vid_str1 = inp_cont.streams.video[0] + vid_str1.thread_type = "AUTO" + vid_str2 = inp_cont.streams.video[1] + vid_str2.thread_type = "AUTO" + dpth_str = inp_cont.streams.video[2] + dpth_str.thread_type = "AUTO" + data_str = inp_cont.streams[3] + + dec_str1, dec_str2, dec_dpth, dec_data = [], [], [], [] + + pkt_counter = 0 + for packet in inp_cont.demux(vid_str1, vid_str2, dpth_str, data_str): + pkt_counter += 1 + if packet.stream.index == vid_str1.index: + frame = packet.decode() + if frame: + for f in frame: + image = f.to_ndarray(format="rgb24") + dec_str1.append(image) + elif packet.stream.index == vid_str2.index: + frame = packet.decode() + if frame: + for f in frame: + image = f.to_ndarray(format="rgb24") + dec_str2.append(image) + elif packet.stream.index == dpth_str.index: + frame = packet.decode() + if frame: + for f in frame: + image = f.to_ndarray(format="gray") + dec_dpth.append(image) + elif packet.stream.index == data_str.index: + pib = bytes(packet) + if pib: + non_dict = pickle.loads(pib) + dec_data.append(non_dict) + else: + print("Unknown stream") + + print(pkt_counter, len(dec_str1), len(dec_str2), len(dec_dpth), len(dec_data)) + inp_cont.close() + + +class MKVWriter(BaseWriter): + + def __init__(self): + super(MKVWriter, self).__init__() + + def create_frame(self, image, stream): + frame = av.VideoFrame.from_ndarray(np.array(image), format="rgb24") + frame.pict_type = "NONE" + frame.time_base = stream.time_base + return frame + + def create_frame_depth(self, image, stream): + image = np.array(image) + + if image.dtype == np.float32: + image = (image * 255).astype(np.uint8) + if len(image.shape) == 3: + image = image[:,:,0] + + frame = av.VideoFrame.from_ndarray(image, format="gray") + frame.pict_type = "NONE" + frame.time_base = stream.time_base + return frame + + def export(self, loader, path): + i = -1 + for traj in loader: + i += 1 + outdir = f"{path}/output_{i}.mkv" + output = av.open(outdir, mode="w") + print(outdir) + + vid_str1 = output.add_stream('libx264', rate=1) + vid_str1.width = 640 + vid_str1.height = 480 + vid_str1.pix_fmt = 'yuv420p' + + vid_str2 = output.add_stream('libx264', rate=1) + vid_str2.width = 640 + vid_str2.height = 480 + vid_str2.pix_fmt = 'yuv420p' + + dpth_str = output.add_stream('libx264', rate=1) + data_str = output.add_stream('rawvideo', rate=1) + + ts = 0 + for step in traj: + obs = step["observation"].copy() + obs.pop("image") + obs.pop("hand_image") + obs.pop("image_with_depth") + misc_step = step.copy() + misc_step["observation"] = obs + + misc_byte = pickle.dumps(misc_step) + packet = av.Packet(misc_byte) + packet.stream = data_str + packet.pts = ts + output.mux(packet) + + image = np.array(step["observation"]["image"]) + frame = self.create_frame(image, vid_str1) + frame.pts = ts + packet = vid_str1.encode(frame) + output.mux(packet) + + hand_image = np.array(step["observation"]["hand_image"]) + frame = self.create_frame(hand_image, vid_str2) + frame.pts = ts + packet = vid_str2.encode(frame) + output.mux(packet) + + frame = self.create_frame_depth(step["observation"]["image_with_depth"], dpth_str) + # frame.pts = ts + packet = dpth_str.encode(frame) + output.mux(packet) + ts += 1 + + for packet in vid_str1.encode(): + output.mux(packet) + + for packet in vid_str2.encode(): + output.mux(packet) + + for packet in dpth_str.encode(): + output.mux(packet) + output.close() + + +def iterate_mkv(loader, num_iters): + read_time, write_time, data_size = 0, 0, 0 + path = PATH + "mkv_convert/" + + if len(os.listdir(path)) != 0: + raise SystemError(f"{path} must be empty.") + + # Write time + stop = time.time() + mkv_writer = MKVWriter() + mkv_writer.export(loader, path) + write_time = time.time() - stop + + # Read time + mkv_loader = MKVLoader(path) + stop = time.time() + for i, data in enumerate(mkv_loader): + if i == num_iters: + break + read_time = time.time() - stop + + # Data size + for i in range(num_iters): + data_size += os.path.getsize(f"{path}/output_{i}.mkv") + + return read_time, write_time, data_size / MB + + +rtx_loader = RTXLoader(PATH + "berkeley_autolab_ur5", f"train[:{N}]") +rt, wt, mb = iterate_mkv(rtx_loader, N) + +print(f"\nMKV Data: \nMem. size = {mb:.4f} MB; Num. traj = {N}") +print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") +print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") \ No newline at end of file diff --git a/benchmarking/hdf5.py b/benchmarking/hdf5.py new file mode 100644 index 0000000..1052937 --- /dev/null +++ b/benchmarking/hdf5.py @@ -0,0 +1,80 @@ +import os +import sys +import time +import h5py +import numpy as np +import pyarrow.parquet as pq + +PATH = os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") + +PATH += "/datasets/" +NAME = "test_convert" +out_dir = PATH + "hdf5_convert" + + +def pq_to_hdf5(in_dir): + table = pq.read_table(in_dir) + file = in_dir.split("/")[-1].split("-")[0] + + with h5py.File(f"{out_dir}/{file}.hdf5", "w") as hdf5: + for name in table.column_names: + data = table[name].to_numpy() + dtype = data.dtype + try: + hdf5.create_dataset(name, data=data, dtype=dtype, compression=None) + except Exception: + dtype = h5py.string_dtype("utf-8") + + data = np.array([str(x) for x in data], dtype=dtype) + hdf5.create_dataset(name, data=data, dtype=dtype, compression=None) + +N = 51 +MB = 1024 * 1024 + +if False: + for i in range(N): + pq_to_hdf5(f"{PATH}{NAME}/{NAME}_{i}-0.parquet") + +def measure_traj(read_func, write_func): + read_time, write_time, data_size = 0, 0, 0 + + for i in range(N): + print(f"Measuring trajectory {i}") + path = f"{out_dir}/{NAME}_{i}.hdf5" + + stop = time.time() + traj = read_func(path) + read_time += time.time() - stop + + data_size += os.path.getsize(path) + path = PATH + "/temp.hdf5" + + stop = time.time() + write_func(path, traj) + write_time += time.time() - stop + + os.remove(path) + return read_time, write_time, data_size / MB + + +if __name__ == "__main__": + + def read_h5(path): + data = {} + with h5py.File(path, "r") as file: + for key in file.keys(): + if isinstance(file[key], h5py.Dataset): + data[key] = file[key][:] + return data + + def write_h5(path, data): + with h5py.File(path, "w") as file: + for name, dset in data.items(): + file.create_dataset(name, data=dset) + + rt, wt, mb = measure_traj(read_h5, write_h5) + + print(f"\nHDF5: \nDisk size = {mb:.4f} MB; Num. traj = {N}") + print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") + print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") \ No newline at end of file diff --git a/benchmarking/misc/fogx_notes.py b/benchmarking/misc/fogx_notes.py new file mode 100644 index 0000000..01c9876 --- /dev/null +++ b/benchmarking/misc/fogx_notes.py @@ -0,0 +1,69 @@ +import os +import sys +import polars as pl +import pyarrow.dataset as ds + +PATH = os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") +from fog_x.dataset import Dataset + +PATH += "/datasets/" +NAME = "test_convert" +dataset = Dataset(name=NAME, path=PATH) + +if False: + dataset.load_rtx_episodes(name="berkeley_autolab_ur5", split="train[:51]") + +# Debugging: path = "/home/joshzhang/datasets" + # Starts with "/" but does not end with it + # Should not initialize any cloud buckets! + +"""###""" + +# Retrieve episode-level metadata as a pandas DataFrame +episode_info = dataset.get_episode_info() + +# Codebase map: +db_manager = dataset.db_manager # DatabaseManager +traj_info_c = db_manager.episode_info_connector # DataFrameC, child of (PolarsC) +step_data_c = db_manager.step_data_connector # LazyFrameC, child of (PolarsC) + +# Call tracing: + # dataset.get_episode_info() + # db_manager.get_episode_info_table() + # traj_info_c.select_table(dataset_name) + # -> traj_info_c.tables[dataset_name] + +episode_info = traj_info_c.tables[NAME] # Same object! +print(episode_info) + +"""###""" + +# Obtain the actual episodes containing their step data +episode_0 = dataset.read_by(episode_info)[0] + +# Call tracing: + # dataset.read_by(episode_info) + # :: i = list(episode_info["episode_id"])[0] + # db_manager.get_step_table(i) + # step_data_c.select_table(dataset_name_i) + # -> step_data_c.tables[dataset_name_i] + +episode_0 = step_data_c.tables[NAME + "_0"] # Same object! +print(episode_0.collect()) + +"""###""" + +# Retrieve Fog-X dataset step data as a Polars LazyFrame +step_data = dataset.get_step_data() + +# Call tracing: + # dataset.get_step_data() + # db_manager.get_step_table_all() + # step_data_c.get_dataset_table() + # :: d = ds.dataset(dataset_path, format="parquet") + # -> pl.scan_pyarrow_dataset(d) + +d = ds.dataset(PATH + NAME, format="parquet") +step_data = pl.scan_pyarrow_dataset(d) # Same object! +print(step_data.collect()) \ No newline at end of file diff --git a/benchmarking/misc/hdf5_loader.ipynb b/benchmarking/misc/hdf5_loader.ipynb new file mode 100644 index 0000000..6036f99 --- /dev/null +++ b/benchmarking/misc/hdf5_loader.ipynb @@ -0,0 +1,59220 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: fog_x in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (0.1.0b4)\n", + "Requirement already satisfied: pandas in /home/lifelong/.local/lib/python3.10/site-packages (from fog_x) (2.1.4)\n", + "Requirement already satisfied: numpy in /home/lifelong/.local/lib/python3.10/site-packages (from fog_x) (1.26.2)\n", + "Requirement already satisfied: polars in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from fog_x) (0.20.31)\n", + "Requirement already satisfied: pillow in /home/lifelong/.local/lib/python3.10/site-packages (from fog_x) (9.5.0)\n", + "Requirement already satisfied: pyarrow in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from fog_x) (16.1.0)\n", + "Requirement already satisfied: opencv-python in /home/lifelong/.local/lib/python3.10/site-packages (from fog_x) (4.6.0.66)\n", + "Requirement already satisfied: sqlalchemy==1.4.51 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from fog_x) (1.4.51)\n", + "Requirement already satisfied: smart-open in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from fog_x) (7.0.4)\n", + "Requirement already satisfied: greenlet!=0.4.17 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from sqlalchemy==1.4.51->fog_x) (3.0.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/lifelong/.local/lib/python3.10/site-packages (from pandas->fog_x) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from pandas->fog_x) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.1 in /home/lifelong/.local/lib/python3.10/site-packages (from pandas->fog_x) (2023.3)\n", + "Requirement already satisfied: wrapt in /home/lifelong/.local/lib/python3.10/site-packages (from smart-open->fog_x) (1.16.0)\n", + "Requirement already satisfied: six>=1.5 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->fog_x) (1.16.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Requirement already satisfied: tensorflow[and-cuda] in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (2.16.1)\n", + "Requirement already satisfied: absl-py>=1.0.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (2.0.0)\n", + "Requirement already satisfied: astunparse>=1.6.0 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (1.6.3)\n", + "Requirement already satisfied: flatbuffers>=23.5.26 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (24.3.25)\n", + "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (0.5.4)\n", + "Requirement already satisfied: google-pasta>=0.1.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (0.2.0)\n", + "Requirement already satisfied: h5py>=3.10.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (3.10.0)\n", + "Requirement already satisfied: libclang>=13.0.0 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (18.1.1)\n", + "Requirement already satisfied: ml-dtypes~=0.3.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (0.3.2)\n", + "Requirement already satisfied: opt-einsum>=2.3.2 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (3.3.0)\n", + "Requirement already satisfied: packaging in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (24.1)\n", + "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (3.20.3)\n", + "Requirement already satisfied: requests<3,>=2.21.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (2.31.0)\n", + "Requirement already satisfied: setuptools in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (69.0.2)\n", + "Requirement already satisfied: six>=1.12.0 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (1.16.0)\n", + "Requirement already satisfied: termcolor>=1.1.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (2.4.0)\n", + "Requirement already satisfied: typing-extensions>=3.6.6 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (4.9.0)\n", + "Requirement already satisfied: wrapt>=1.11.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (1.16.0)\n", + "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (1.60.0)\n", + "Requirement already satisfied: tensorboard<2.17,>=2.16 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (2.16.2)\n", + "Requirement already satisfied: keras>=3.0.0 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (3.3.3)\n", + "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (0.37.0)\n", + "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow[and-cuda]) (1.26.2)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.3.4.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.3.4.1)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.3.101 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.3.101)\n", + "Requirement already satisfied: nvidia-cuda-nvcc-cu12==12.3.107 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.3.107)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.3.107 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.3.107)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.3.101 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.3.101)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.7.29 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (8.9.7.29)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.12.1 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (11.0.12.1)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.4.107 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (10.3.4.107)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.5.4.101 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (11.5.4.101)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.2.0.103 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.2.0.103)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (2.19.3)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.3.101 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow[and-cuda]) (12.3.101)\n", + "Requirement already satisfied: wheel<1.0,>=0.23.0 in /home/lifelong/.local/lib/python3.10/site-packages (from astunparse>=1.6.0->tensorflow[and-cuda]) (0.42.0)\n", + "Requirement already satisfied: rich in /home/lifelong/.local/lib/python3.10/site-packages (from keras>=3.0.0->tensorflow[and-cuda]) (13.7.0)\n", + "Requirement already satisfied: namex in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from keras>=3.0.0->tensorflow[and-cuda]) (0.0.8)\n", + "Requirement already satisfied: optree in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from keras>=3.0.0->tensorflow[and-cuda]) (0.11.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/lifelong/.local/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow[and-cuda]) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow[and-cuda]) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/lifelong/.local/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow[and-cuda]) (2.1.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow[and-cuda]) (2024.6.2)\n", + "Requirement already satisfied: markdown>=2.6.8 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorboard<2.17,>=2.16->tensorflow[and-cuda]) (3.5.1)\n", + "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorboard<2.17,>=2.16->tensorflow[and-cuda]) (0.7.2)\n", + "Requirement already satisfied: werkzeug>=1.0.1 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorboard<2.17,>=2.16->tensorflow[and-cuda]) (3.0.1)\n", + "Requirement already satisfied: MarkupSafe>=2.1.1 in /home/lifelong/.local/lib/python3.10/site-packages (from werkzeug>=1.0.1->tensorboard<2.17,>=2.16->tensorflow[and-cuda]) (2.1.3)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/lifelong/.local/lib/python3.10/site-packages (from rich->keras>=3.0.0->tensorflow[and-cuda]) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/lifelong/.local/lib/python3.10/site-packages (from rich->keras>=3.0.0->tensorflow[and-cuda]) (2.17.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /home/lifelong/.local/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.0.0->tensorflow[and-cuda]) (0.1.2)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Collecting tensorflow_datasets\n", + " Downloading tensorflow_datasets-4.9.6-py3-none-any.whl.metadata (9.5 kB)\n", + "Requirement already satisfied: absl-py in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (2.0.0)\n", + "Requirement already satisfied: click in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (8.1.7)\n", + "Collecting dm-tree (from tensorflow_datasets)\n", + " Downloading dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.9 kB)\n", + "Collecting immutabledict (from tensorflow_datasets)\n", + " Downloading immutabledict-4.2.0-py3-none-any.whl.metadata (3.4 kB)\n", + "Requirement already satisfied: numpy in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (1.26.2)\n", + "Collecting promise (from tensorflow_datasets)\n", + " Downloading promise-2.3.tar.gz (19 kB)\n", + " Preparing metadata (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25hRequirement already satisfied: protobuf>=3.20 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (3.20.3)\n", + "Requirement already satisfied: psutil in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (5.9.7)\n", + "Requirement already satisfied: pyarrow in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from tensorflow_datasets) (16.1.0)\n", + "Requirement already satisfied: requests>=2.19.0 in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (2.31.0)\n", + "Collecting simple-parsing (from tensorflow_datasets)\n", + " Downloading simple_parsing-0.1.5-py3-none-any.whl.metadata (7.7 kB)\n", + "Collecting tensorflow-metadata (from tensorflow_datasets)\n", + " Downloading tensorflow_metadata-1.15.0-py3-none-any.whl.metadata (2.4 kB)\n", + "Requirement already satisfied: termcolor in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (2.4.0)\n", + "Collecting toml (from tensorflow_datasets)\n", + " Downloading toml-0.10.2-py2.py3-none-any.whl.metadata (7.1 kB)\n", + "Requirement already satisfied: tqdm in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (4.66.1)\n", + "Requirement already satisfied: wrapt in /home/lifelong/.local/lib/python3.10/site-packages (from tensorflow_datasets) (1.16.0)\n", + "Collecting array-record>=0.5.0 (from tensorflow_datasets)\n", + " Downloading array_record-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (699 bytes)\n", + "Collecting etils>=1.6.0 (from etils[enp,epath,epy,etree]>=1.6.0; python_version < \"3.11\"->tensorflow_datasets)\n", + " Downloading etils-1.7.0-py3-none-any.whl.metadata (6.4 kB)\n", + "Requirement already satisfied: typing_extensions in /home/lifelong/.local/lib/python3.10/site-packages (from etils[enp,epath,epy,etree]>=1.6.0; python_version < \"3.11\"->tensorflow_datasets) (4.9.0)\n", + "Collecting fsspec (from etils[enp,epath,epy,etree]>=1.6.0; python_version < \"3.11\"->tensorflow_datasets)\n", + " Downloading fsspec-2024.6.0-py3-none-any.whl.metadata (11 kB)\n", + "Collecting importlib_resources (from etils[enp,epath,epy,etree]>=1.6.0; python_version < \"3.11\"->tensorflow_datasets)\n", + " Downloading importlib_resources-6.4.0-py3-none-any.whl.metadata (3.9 kB)\n", + "Requirement already satisfied: zipp in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from etils[enp,epath,epy,etree]>=1.6.0; python_version < \"3.11\"->tensorflow_datasets) (3.19.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/lifelong/.local/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow_datasets) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow_datasets) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/lifelong/.local/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow_datasets) (2.1.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from requests>=2.19.0->tensorflow_datasets) (2024.6.2)\n", + "Requirement already satisfied: six in /home/lifelong/anaconda3/envs/fog_rtx_env/lib/python3.10/site-packages (from promise->tensorflow_datasets) (1.16.0)\n", + "Requirement already satisfied: docstring-parser~=0.15 in /home/lifelong/.local/lib/python3.10/site-packages (from simple-parsing->tensorflow_datasets) (0.15)\n", + "Downloading tensorflow_datasets-4.9.6-py3-none-any.whl (5.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.1/5.1 MB\u001b[0m \u001b[31m46.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading array_record-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading etils-1.7.0-py3-none-any.whl (152 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.4/152.4 kB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (152 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m37.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading immutabledict-4.2.0-py3-none-any.whl (4.7 kB)\n", + "Downloading simple_parsing-0.1.5-py3-none-any.whl (113 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.6/113.6 kB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tensorflow_metadata-1.15.0-py3-none-any.whl (28 kB)\n", + "Downloading toml-0.10.2-py2.py3-none-any.whl (16 kB)\n", + "Downloading fsspec-2024.6.0-py3-none-any.whl (176 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.9/176.9 kB\u001b[0m \u001b[31m33.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading importlib_resources-6.4.0-py3-none-any.whl (38 kB)\n", + "Building wheels for collected packages: promise\n", + " Building wheel for promise (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for promise: filename=promise-2.3-py3-none-any.whl size=21483 sha256=065b0237f470f3d985da7b209d3094b4ea457b0e3ed64c9cf8f7b0440c45cce6\n", + " Stored in directory: /home/lifelong/.cache/pip/wheels/54/4e/28/3ed0e1c8a752867445bab994d2340724928aa3ab059c57c8db\n", + "Successfully built promise\n", + "Installing collected packages: dm-tree, toml, tensorflow-metadata, simple-parsing, promise, importlib_resources, immutabledict, fsspec, etils, array-record, tensorflow_datasets\n", + "Successfully installed array-record-0.5.1 dm-tree-0.1.8 etils-1.7.0 fsspec-2024.6.0 immutabledict-4.2.0 importlib_resources-6.4.0 promise-2.3 simple-parsing-0.1.5 tensorflow-metadata-1.15.0 tensorflow_datasets-4.9.6 toml-0.10.2\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install fog_x\n", + "!pip install tensorflow[and-cuda]\n", + "!pip install tensorflow_datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "import fog_x \n", + "import os\n", + "from logging import getLogger\n", + "import numpy as np\n", + "import os\n", + "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # or any {'0', '1', '2'}\n", + "import tensorflow as tf\n", + "\n", + "class BaseLoader():\n", + " def __init__(self, data_path):\n", + " super(BaseLoader, self).__init__()\n", + " self.data_dir = data_path\n", + " self.logger = getLogger(__name__)\n", + " self.index = 0\n", + "\n", + "\n", + " def __len__(self):\n", + " raise NotImplementedError\n", + "\n", + " def __iter___(self):\n", + " raise NotImplementedError\n", + "\n", + "class BaseExporter():\n", + " def __init__(self):\n", + " super(BaseExporter, self).__init__()\n", + " self.logger = getLogger(__name__)\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " raise NotImplementedError" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "class RTXLoader(BaseLoader):\n", + " def __init__(self, data_path, split):\n", + " super(RTXLoader, self).__init__(data_path)\n", + " import tensorflow_datasets as tfds\n", + "\n", + " builder = tfds.builder_from_directory(data_path)\n", + "\n", + " self.ds = builder.as_dataset(split=split)\n", + " # https://www.determined.ai/blog/tf-dataset-the-bad-parts\n", + "\n", + " def __len__(self):\n", + " return len(self.ds)\n", + " \n", + " def __next__(self):\n", + " if self.index < len(self.ds):\n", + " self.index += 1\n", + " nested_dataset = self.ds.__iter__()\n", + " trajectory = list(nested_dataset)[0][\"steps\"]\n", + " ret = []\n", + " # Iterate through the outer dataset\n", + " for step_data in trajectory:\n", + " step = {}\n", + " for dataset_key, element in step_data.items():\n", + " # print(np.array(element))\n", + " if dataset_key == \"observation\":\n", + " step[\"observation\"] = {}\n", + " for obs_key, obs_element in element.items():\n", + " step[\"observation\"][obs_key] = np.array(obs_element)\n", + " elif dataset_key == \"action\":\n", + " step[\"action\"] = {}\n", + " for action_key, action_element in element.items():\n", + " step[\"action\"][action_key] = np.array(action_element)\n", + " else:\n", + " step[dataset_key] = np.array(element)\n", + " ret.append(step)\n", + " return ret\n", + " else:\n", + " raise StopIteration\n", + " \n", + "\n", + " \n", + " def __iter__(self):\n", + " return self\n", + "\n", + "\n", + "class RTXExporter(BaseExporter):\n", + " def __init__(self):\n", + " super(RTXExporter, self).__init__()\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " raise NotImplementedError" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "import h5py\n", + "class HDF5Loader(BaseLoader):\n", + " def __init__(self, data_path):\n", + " super(HDF5Loader, self).__init__(data_path)\n", + " self.ds = []\n", + " with h5py.File(data_path, \"r\") as f:\n", + " print(\"Trajectories: %s\" % f.keys())\n", + " trajectories = []\n", + " for trajectory_key in f.keys():\n", + " traj_hdf5 = f[trajectory_key]\n", + " print(\"Steps: %s\" % traj_hdf5.keys())\n", + " steps = []\n", + " for step_key in traj_hdf5.keys():\n", + " step = {}\n", + " step_hdf5 = traj_hdf5[step_key]\n", + " for item_key,item_value in step_hdf5.items():\n", + " if(item_key == 'action'):\n", + " action_dict = {}\n", + " for action_key,action_value in item_value.items():\n", + " if(action_key == 'gripper_closedness_action'):\n", + " action_dict['gripper_closedness_action'] = action_value[()]\n", + " elif(action_key == 'rotation_delta'):\n", + " action_dict['rotation_delta'] = action_value[()]\n", + " elif(action_key == 'terminate_episode'):\n", + " action_dict['terminate_episode'] = action_value[()]\n", + " elif(action_key == 'world_vector'):\n", + " action_dict['world_vector'] = action_value[()]\n", + " step['action'] = action_dict\n", + " elif(item_key == 'is_first'):\n", + " step['is_first'] = item_value[()]\n", + " elif(item_key == 'is_last'):\n", + " step['is_last'] = item_value[()]\n", + " elif(item_key == 'is_terminal'):\n", + " step['is_terminal'] = item_value[()]\n", + " elif(item_key == 'observation'):\n", + " observation_dict = {}\n", + " for observation_key,observation_value in item_value.items():\n", + " if(observation_key == 'hand_image'):\n", + " observation_dict['hand_image'] = observation_value[()]\n", + " elif(observation_key == 'image'):\n", + " observation_dict['image'] = observation_value[()]\n", + " elif(observation_key == 'image_with_depth'):\n", + " observation_dict['image_with_depth'] = observation_value[()]\n", + " elif(observation_key == 'natural_language_embedding'):\n", + " observation_dict['natural_language_embedding'] = observation_value[()]\n", + " elif(observation_key == 'natural_language_instruction'):\n", + " observation_dict['natural_language_instruction'] = observation_value[()]\n", + " elif(observation_key == 'robot_state'):\n", + " observation_dict['robot_state'] = observation_value[()]\n", + " step['observation'] = observation_dict\n", + " elif(item_key == 'reward'):\n", + " step['reward'] = item_value[()]\n", + " steps.append(step)\n", + " trajectories.append(steps)\n", + " self.ds = trajectories\n", + " \n", + " def __len__(self):\n", + " return len(self.ds)\n", + " \n", + " def __iter__(self):\n", + " return self.ds.__iter__()\n", + " \n", + "class HDF5Exporter(BaseExporter):\n", + " def __init__(self):\n", + " super(HDF5Exporter, self).__init__()\n", + "\n", + " def export(self, loader: BaseLoader, output_path: str):\n", + " with h5py.File(output_path, 'w') as hdf:\n", + " for i,trajectory in enumerate(loader):\n", + " traj_key = \"traj\" + str(i).zfill(5)\n", + " traj_hdf5 = hdf.create_group(traj_key)\n", + " for j,data in enumerate(trajectory):\n", + " step_key = \"step\" + str(j).zfill(5)\n", + " step_hdf5 = traj_hdf5.create_group(step_key)\n", + " for key,value in data.items():\n", + " if(key == 'action'):\n", + " action_hdf5 = step_hdf5.create_group(\"action\")\n", + " for action_key,action_value in value.items():\n", + " if(action_key == 'gripper_closedness_action'):\n", + " action_hdf5.create_dataset('gripper_closedness_action',data=action_value)\n", + " elif(action_key == 'rotation_delta'):\n", + " action_hdf5.create_dataset('rotation_delta',data=action_value)\n", + " elif(action_key == 'terminate_episode'):\n", + " action_hdf5.create_dataset('terminate_episode',data=action_value)\n", + " elif(action_key == 'world_vector'):\n", + " action_hdf5.create_dataset('world_vector',data=action_value)\n", + " elif(key == 'is_first'):\n", + " step_hdf5.create_dataset('is_first',data=np.array(value))\n", + " elif(key=='is_last'):\n", + " step_hdf5.create_dataset('is_last',data=value)\n", + " elif(key=='is_terminal'):\n", + " step_hdf5.create_dataset('is_terminal',data=value)\n", + " elif(key=='observation'):\n", + " observation_hdf5 = step_hdf5.create_group(\"observation\")\n", + " for observation_key,observation_value in value.items():\n", + " if(observation_key == 'hand_image'):\n", + " observation_hdf5.create_dataset('hand_image',data=observation_value)\n", + " elif(observation_key == 'image'):\n", + " observation_hdf5.create_dataset('image',data=observation_value)\n", + " elif(observation_key == 'image_with_depth'):\n", + " observation_hdf5.create_dataset('image_with_depth',data=observation_value)\n", + " elif(observation_key == 'natural_language_embedding'):\n", + " observation_hdf5.create_dataset('natural_language_embedding',data=observation_value)\n", + " elif(observation_key == 'natural_language_instruction'):\n", + " observation_hdf5.create_dataset('natural_language_instruction',data=observation_value)\n", + " elif(observation_key == 'robot_state'):\n", + " observation_hdf5.create_dataset('robot_state',data=observation_value)\n", + " elif(key=='reward'):\n", + " step_hdf5.create_dataset('reward',data=value)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I 2024-06-23 14:37:21,456 dataset_info.py:617] Load dataset info from /home/lifelong/fog_x/datasets/0.1.0\n", + "I 2024-06-23 14:37:21,461 reader.py:261] Creating a tf.data.Dataset reading 1 files located in folders: /home/lifelong/fog_x/datasets/0.1.0.\n", + "I 2024-06-23 14:37:21,500 logging_logger.py:49] Constructing tf.data.Dataset berkeley_autolab_ur5 for split train[:3], from /home/lifelong/fog_x/datasets/0.1.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trajectories: \n", + "Steps: \n", + "Steps: \n", + "Steps: \n", + "3\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00038095, 0. , 0.00057143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00228571, 0.00685714, 0.00354286], dtype=float32)}, 'is_first': True, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 47, 33, 26],\n", + " [ 49, 35, 29],\n", + " [ 48, 34, 27],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 19, 12, 11],\n", + " [ 22, 16, 15]],\n", + "\n", + " [[ 48, 34, 27],\n", + " [ 48, 34, 27],\n", + " [ 47, 33, 26],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " [[ 48, 34, 27],\n", + " [ 48, 34, 27],\n", + " [ 47, 33, 26],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[207, 233, 169],\n", + " [207, 233, 169],\n", + " [207, 233, 169],\n", + " ...,\n", + " [158, 176, 198],\n", + " [159, 177, 199],\n", + " [159, 177, 199]],\n", + "\n", + " [[207, 233, 169],\n", + " [207, 233, 169],\n", + " [208, 234, 170],\n", + " ...,\n", + " [159, 177, 199],\n", + " [158, 176, 198],\n", + " [158, 176, 198]],\n", + "\n", + " [[207, 233, 169],\n", + " [208, 234, 170],\n", + " [208, 234, 170],\n", + " ...,\n", + " [158, 176, 198],\n", + " [157, 175, 197],\n", + " [159, 177, 199]]], dtype=uint8), 'image': array([[[112, 105, 76],\n", + " [114, 107, 78],\n", + " [ 53, 77, 65],\n", + " ...,\n", + " [ 98, 92, 94],\n", + " [ 99, 91, 90],\n", + " [ 99, 91, 90]],\n", + "\n", + " [[124, 105, 78],\n", + " [122, 103, 76],\n", + " [ 68, 93, 76],\n", + " ...,\n", + " [ 97, 92, 90],\n", + " [ 97, 92, 88],\n", + " [ 97, 92, 88]],\n", + "\n", + " [[119, 88, 61],\n", + " [129, 98, 71],\n", + " [ 79, 107, 82],\n", + " ...,\n", + " [ 97, 89, 86],\n", + " [ 99, 92, 86],\n", + " [ 96, 89, 83]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 74, 58],\n", + " [ 74, 77, 61],\n", + " [ 70, 75, 61],\n", + " ...,\n", + " [ 70, 95, 102],\n", + " [ 72, 97, 104],\n", + " [ 75, 100, 107]],\n", + "\n", + " [[ 61, 64, 46],\n", + " [ 60, 63, 45],\n", + " [ 58, 61, 43],\n", + " ...,\n", + " [ 72, 97, 102],\n", + " [ 71, 96, 101],\n", + " [ 73, 98, 103]],\n", + "\n", + " [[ 56, 58, 40],\n", + " [ 53, 55, 37],\n", + " [ 52, 55, 37],\n", + " ...,\n", + " [ 68, 96, 100],\n", + " [ 70, 95, 100],\n", + " [ 71, 96, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2969947 , -1.6309904 , 2.4249759 , -2.406821 , -1.5576328 ,\n", + " 2.7911022 , 0.36736205, 0.05681828, -0.01433784, 0.6336503 ,\n", + " 0.77331245, 0.01563 , 0.01519585, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00114286, 0. , 0.01580952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00954286, 0.01148571, 0.00508571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 49, 33, 26],\n", + " [ 51, 35, 29],\n", + " [ 52, 36, 30],\n", + " ...,\n", + " [ 18, 10, 12],\n", + " [ 20, 13, 14],\n", + " [ 20, 13, 14]],\n", + "\n", + " [[ 49, 33, 26],\n", + " [ 49, 33, 26],\n", + " [ 49, 33, 26],\n", + " ...,\n", + " [ 20, 13, 14],\n", + " [ 21, 14, 15],\n", + " [ 21, 14, 15]],\n", + "\n", + " [[ 47, 33, 26],\n", + " [ 47, 33, 26],\n", + " [ 49, 33, 26],\n", + " ...,\n", + " [ 20, 13, 14],\n", + " [ 21, 14, 15],\n", + " [ 21, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[209, 234, 173],\n", + " [209, 234, 173],\n", + " [208, 233, 172],\n", + " ...,\n", + " [158, 176, 198],\n", + " [159, 177, 199],\n", + " [159, 177, 199]],\n", + "\n", + " [[208, 233, 172],\n", + " [208, 233, 172],\n", + " [210, 235, 174],\n", + " ...,\n", + " [161, 178, 200],\n", + " [161, 178, 200],\n", + " [161, 178, 200]],\n", + "\n", + " [[208, 233, 172],\n", + " [209, 234, 173],\n", + " [211, 236, 175],\n", + " ...,\n", + " [161, 178, 200],\n", + " [161, 178, 200],\n", + " [161, 178, 200]]], dtype=uint8), 'image': array([[[116, 108, 82],\n", + " [111, 103, 77],\n", + " [ 49, 74, 67],\n", + " ...,\n", + " [ 97, 90, 82],\n", + " [ 99, 92, 84],\n", + " [ 99, 92, 84]],\n", + "\n", + " [[127, 110, 83],\n", + " [117, 100, 73],\n", + " [ 66, 94, 79],\n", + " ...,\n", + " [ 95, 88, 80],\n", + " [ 99, 92, 84],\n", + " [ 96, 89, 81]],\n", + "\n", + " [[120, 92, 64],\n", + " [127, 99, 71],\n", + " [ 81, 108, 86],\n", + " ...,\n", + " [ 95, 88, 78],\n", + " [ 92, 85, 75],\n", + " [ 89, 82, 72]],\n", + "\n", + " ...,\n", + "\n", + " [[ 74, 77, 61],\n", + " [ 79, 82, 66],\n", + " [ 75, 78, 60],\n", + " ...,\n", + " [ 72, 98, 98],\n", + " [ 72, 98, 98],\n", + " [ 72, 98, 98]],\n", + "\n", + " [[ 65, 69, 48],\n", + " [ 67, 71, 50],\n", + " [ 61, 65, 44],\n", + " ...,\n", + " [ 72, 98, 98],\n", + " [ 73, 97, 95],\n", + " [ 73, 97, 95]],\n", + "\n", + " [[ 59, 62, 41],\n", + " [ 58, 61, 40],\n", + " [ 51, 55, 34],\n", + " ...,\n", + " [ 70, 96, 96],\n", + " [ 72, 96, 96],\n", + " [ 73, 97, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2966354 , -1.6166404 , 2.418114 , -2.4144275 , -1.5576924 ,\n", + " 2.7917013 , 0.3713127 , 0.05632875, -0.01605012, 0.63372785,\n", + " 0.7732474 , 0.01564922, 0.01525001, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00095238, 0.01561905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01108571, 0.01057143, 0.00765714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 38, 29, 22],\n", + " [ 42, 33, 25],\n", + " [ 42, 33, 25],\n", + " ...,\n", + " [ 21, 15, 14],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " [[ 37, 28, 20],\n", + " [ 38, 29, 22],\n", + " [ 39, 30, 23],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 21, 15, 14],\n", + " [ 21, 15, 14]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 35, 28, 22],\n", + " [ 38, 29, 22],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 21, 15, 14],\n", + " [ 21, 15, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[199, 224, 170],\n", + " [198, 223, 169],\n", + " [196, 224, 167],\n", + " ...,\n", + " [159, 177, 199],\n", + " [159, 177, 199],\n", + " [161, 178, 200]],\n", + "\n", + " [[201, 226, 172],\n", + " [198, 223, 169],\n", + " [196, 224, 167],\n", + " ...,\n", + " [161, 178, 200],\n", + " [159, 177, 199],\n", + " [161, 178, 200]],\n", + "\n", + " [[201, 226, 172],\n", + " [201, 226, 172],\n", + " [197, 225, 168],\n", + " ...,\n", + " [159, 177, 199],\n", + " [158, 176, 198],\n", + " [159, 177, 199]]], dtype=uint8), 'image': array([[[119, 106, 66],\n", + " [116, 103, 63],\n", + " [ 52, 71, 59],\n", + " ...,\n", + " [ 84, 75, 68],\n", + " [ 83, 74, 67],\n", + " [ 80, 71, 64]],\n", + "\n", + " [[128, 108, 68],\n", + " [128, 108, 68],\n", + " [ 64, 89, 68],\n", + " ...,\n", + " [ 81, 71, 59],\n", + " [ 80, 71, 62],\n", + " [ 79, 70, 61]],\n", + "\n", + " [[120, 93, 58],\n", + " [130, 103, 68],\n", + " [ 80, 106, 80],\n", + " ...,\n", + " [ 78, 68, 54],\n", + " [ 76, 68, 54],\n", + " [ 78, 70, 56]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 75, 54],\n", + " [ 70, 73, 52],\n", + " [ 74, 74, 54],\n", + " ...,\n", + " [ 78, 101, 106],\n", + " [ 77, 100, 105],\n", + " [ 74, 97, 102]],\n", + "\n", + " [[ 63, 66, 43],\n", + " [ 63, 66, 43],\n", + " [ 61, 64, 41],\n", + " ...,\n", + " [ 72, 97, 102],\n", + " [ 72, 97, 102],\n", + " [ 71, 96, 101]],\n", + "\n", + " [[ 55, 55, 31],\n", + " [ 55, 55, 31],\n", + " [ 52, 55, 32],\n", + " ...,\n", + " [ 66, 95, 96],\n", + " [ 70, 95, 100],\n", + " [ 69, 94, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3029208 , -1.591381 , 2.4043546 , -2.4257529 , -1.5580639 ,\n", + " 2.7932708 , 0.37898293, 0.05273136, -0.01882746, 0.63674 ,\n", + " 0.7707726 , 0.0154203 , 0.01529887, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0192381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01171429, 0.00902857, 0.00897143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 16, 12, 8],\n", + " [ 17, 14, 10],\n", + " [ 15, 11, 7],\n", + " ...,\n", + " [ 16, 12, 10],\n", + " [ 18, 14, 12],\n", + " [ 19, 15, 14]],\n", + "\n", + " [[ 20, 14, 11],\n", + " [ 18, 12, 8],\n", + " [ 16, 12, 8],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14]],\n", + "\n", + " [[ 20, 14, 11],\n", + " [ 18, 12, 8],\n", + " [ 18, 12, 8],\n", + " ...,\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[211, 232, 185],\n", + " [210, 230, 184],\n", + " [210, 230, 184],\n", + " ...,\n", + " [159, 174, 197],\n", + " [160, 175, 198],\n", + " [160, 175, 198]],\n", + "\n", + " [[212, 233, 186],\n", + " [211, 232, 185],\n", + " [211, 232, 185],\n", + " ...,\n", + " [161, 177, 199],\n", + " [161, 177, 199],\n", + " [161, 177, 199]],\n", + "\n", + " [[212, 233, 186],\n", + " [212, 233, 186],\n", + " [212, 233, 186],\n", + " ...,\n", + " [161, 177, 199],\n", + " [160, 175, 198],\n", + " [161, 177, 199]]], dtype=uint8), 'image': array([[[111, 105, 81],\n", + " [108, 102, 78],\n", + " [ 49, 76, 71],\n", + " ...,\n", + " [ 69, 69, 47],\n", + " [ 70, 68, 49],\n", + " [ 70, 68, 49]],\n", + "\n", + " [[121, 105, 82],\n", + " [123, 107, 84],\n", + " [ 65, 92, 83],\n", + " ...,\n", + " [ 72, 71, 54],\n", + " [ 72, 67, 49],\n", + " [ 78, 73, 55]],\n", + "\n", + " [[117, 90, 68],\n", + " [129, 102, 80],\n", + " [ 78, 107, 89],\n", + " ...,\n", + " [ 75, 74, 59],\n", + " [ 76, 74, 55],\n", + " [ 81, 79, 60]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 75, 52],\n", + " [ 83, 86, 63],\n", + " [ 78, 81, 58],\n", + " ...,\n", + " [ 70, 100, 105],\n", + " [ 67, 99, 106],\n", + " [ 66, 98, 105]],\n", + "\n", + " [[ 60, 63, 40],\n", + " [ 56, 59, 36],\n", + " [ 57, 61, 40],\n", + " ...,\n", + " [ 68, 93, 100],\n", + " [ 68, 96, 102],\n", + " [ 68, 96, 102]],\n", + "\n", + " [[ 51, 54, 33],\n", + " [ 51, 54, 33],\n", + " [ 51, 54, 36],\n", + " ...,\n", + " [ 69, 94, 99],\n", + " [ 71, 97, 99],\n", + " [ 71, 97, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.314199 , -1.5623087 , 2.39123 , -2.4421766 , -1.557824 ,\n", + " 2.7923725 , 0.3873519 , 0.0471875 , -0.0229582 , 0.64069533,\n", + " 0.7674803 , 0.01556755, 0.01552759, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01411429, 0.00897143, 0.01274286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 20, 17, 13],\n", + " [ 18, 15, 11],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 17, 13, 11],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " [[ 19, 16, 12],\n", + " [ 18, 15, 11],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 17, 13, 11],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " [[ 21, 16, 10],\n", + " [ 20, 15, 9],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 230, 185],\n", + " [210, 230, 185],\n", + " [209, 228, 184],\n", + " ...,\n", + " [156, 172, 190],\n", + " [156, 172, 190],\n", + " [158, 175, 192]],\n", + "\n", + " [[210, 230, 185],\n", + " [210, 230, 185],\n", + " [211, 231, 187],\n", + " ...,\n", + " [158, 175, 192],\n", + " [158, 175, 192],\n", + " [158, 175, 192]],\n", + "\n", + " [[211, 231, 187],\n", + " [211, 231, 187],\n", + " [211, 231, 187],\n", + " ...,\n", + " [158, 175, 192],\n", + " [158, 175, 192],\n", + " [157, 174, 191]]], dtype=uint8), 'image': array([[[110, 105, 72],\n", + " [108, 103, 70],\n", + " [ 48, 77, 74],\n", + " ...,\n", + " [ 17, 20, 21],\n", + " [ 19, 26, 20],\n", + " [ 36, 43, 37]],\n", + "\n", + " [[118, 103, 73],\n", + " [121, 106, 76],\n", + " [ 55, 87, 76],\n", + " ...,\n", + " [ 13, 19, 17],\n", + " [ 36, 44, 31],\n", + " [ 47, 55, 42]],\n", + "\n", + " [[116, 89, 63],\n", + " [126, 99, 73],\n", + " [ 74, 106, 87],\n", + " ...,\n", + " [ 25, 34, 27],\n", + " [ 56, 65, 45],\n", + " [ 61, 70, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 53],\n", + " [ 75, 78, 60],\n", + " [ 73, 75, 64],\n", + " ...,\n", + " [ 67, 97, 100],\n", + " [ 68, 99, 99],\n", + " [ 67, 98, 98]],\n", + "\n", + " [[ 61, 65, 44],\n", + " [ 64, 68, 47],\n", + " [ 59, 61, 43],\n", + " ...,\n", + " [ 66, 96, 101],\n", + " [ 66, 96, 99],\n", + " [ 66, 96, 99]],\n", + "\n", + " [[ 60, 59, 42],\n", + " [ 57, 56, 39],\n", + " [ 49, 51, 33],\n", + " ...,\n", + " [ 65, 95, 100],\n", + " [ 61, 94, 98],\n", + " [ 61, 94, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3272011 , -1.5332474 , 2.3811233 , -2.4609396 , -1.5572494 ,\n", + " 2.7913055 , 0.3946254 , 0.04107245, -0.02822383, 0.6452304 ,\n", + " 0.7636719 , 0.01568247, 0.01539803, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.01790476], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01222857, 0.00857143, 0.01382857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 26, 18, 13],\n", + " [ 26, 18, 13],\n", + " [ 26, 18, 14],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 18, 13, 14],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 24, 17, 12],\n", + " [ 23, 16, 10],\n", + " [ 23, 16, 10],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 24, 17, 12],\n", + " [ 23, 16, 10],\n", + " [ 23, 16, 10],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[206, 228, 182],\n", + " [206, 228, 182],\n", + " [205, 227, 180],\n", + " ...,\n", + " [153, 170, 188],\n", + " [154, 171, 189],\n", + " [154, 171, 189]],\n", + "\n", + " [[207, 230, 183],\n", + " [207, 230, 183],\n", + " [207, 230, 183],\n", + " ...,\n", + " [153, 170, 188],\n", + " [153, 170, 188],\n", + " [153, 170, 188]],\n", + "\n", + " [[206, 228, 182],\n", + " [207, 230, 183],\n", + " [207, 230, 183],\n", + " ...,\n", + " [153, 170, 188],\n", + " [154, 171, 189],\n", + " [153, 170, 188]]], dtype=uint8), 'image': array([[[108, 102, 78],\n", + " [108, 102, 78],\n", + " [ 55, 82, 75],\n", + " ...,\n", + " [ 71, 79, 66],\n", + " [ 69, 77, 64],\n", + " [ 70, 78, 65]],\n", + "\n", + " [[111, 94, 76],\n", + " [123, 106, 88],\n", + " [ 66, 94, 81],\n", + " ...,\n", + " [ 69, 77, 66],\n", + " [ 67, 80, 67],\n", + " [ 67, 80, 67]],\n", + "\n", + " [[113, 85, 68],\n", + " [130, 102, 85],\n", + " [ 80, 108, 95],\n", + " ...,\n", + " [ 70, 78, 67],\n", + " [ 68, 83, 69],\n", + " [ 68, 83, 69]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 73, 45],\n", + " [ 72, 77, 49],\n", + " [ 69, 76, 50],\n", + " ...,\n", + " [ 63, 93, 98],\n", + " [ 68, 96, 102],\n", + " [ 68, 96, 102]],\n", + "\n", + " [[ 60, 65, 35],\n", + " [ 61, 66, 36],\n", + " [ 58, 63, 33],\n", + " ...,\n", + " [ 62, 92, 97],\n", + " [ 64, 89, 96],\n", + " [ 63, 88, 95]],\n", + "\n", + " [[ 57, 58, 31],\n", + " [ 56, 57, 30],\n", + " [ 53, 57, 27],\n", + " ...,\n", + " [ 59, 89, 94],\n", + " [ 62, 87, 94],\n", + " [ 59, 84, 91]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3431368 , -1.4991382 , 2.3722975 , -2.4858325 , -1.5566143 ,\n", + " 2.7856972 , 0.40199557, 0.03360351, -0.03550917, 0.6491278 ,\n", + " 0.76036435, 0.01573002, 0.01522294, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00019048, 0. , 0.01238095], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01257143, 0.01074286, 0.01348571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 3, 4, 2],\n", + " [ 6, 7, 4],\n", + " [ 9, 7, 5],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 17, 12, 13],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 9, 7, 5],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 8, 6, 4],\n", + " [ 9, 7, 5],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[203, 227, 182],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [158, 174, 187],\n", + " [161, 174, 188],\n", + " [161, 174, 188]],\n", + "\n", + " [[201, 226, 181],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [160, 176, 189],\n", + " [162, 176, 189],\n", + " [161, 174, 188]],\n", + "\n", + " [[203, 227, 182],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [161, 177, 190],\n", + " [162, 176, 189],\n", + " [161, 174, 188]]], dtype=uint8), 'image': array([[[115, 109, 85],\n", + " [112, 106, 82],\n", + " [ 56, 83, 78],\n", + " ...,\n", + " [ 80, 93, 88],\n", + " [ 77, 92, 87],\n", + " [ 78, 93, 88]],\n", + "\n", + " [[124, 108, 85],\n", + " [126, 110, 87],\n", + " [ 54, 85, 71],\n", + " ...,\n", + " [ 77, 92, 87],\n", + " [ 77, 92, 89],\n", + " [ 74, 89, 86]],\n", + "\n", + " [[116, 89, 67],\n", + " [137, 110, 88],\n", + " [ 69, 101, 82],\n", + " ...,\n", + " [ 74, 92, 86],\n", + " [ 72, 90, 84],\n", + " [ 69, 87, 81]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 72, 54],\n", + " [ 73, 75, 57],\n", + " [ 69, 70, 57],\n", + " ...,\n", + " [ 82, 89, 83],\n", + " [ 82, 87, 77],\n", + " [ 81, 86, 76]],\n", + "\n", + " [[ 60, 66, 49],\n", + " [ 62, 68, 51],\n", + " [ 59, 65, 48],\n", + " ...,\n", + " [ 82, 86, 79],\n", + " [ 84, 86, 77],\n", + " [ 84, 86, 77]],\n", + "\n", + " [[ 53, 58, 46],\n", + " [ 56, 61, 49],\n", + " [ 49, 55, 38],\n", + " ...,\n", + " [ 86, 87, 81],\n", + " [ 90, 88, 80],\n", + " [ 90, 88, 80]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3573966 , -1.4627742 , 2.364023 , -2.5137966 , -1.5562791 ,\n", + " 2.7826176 , 0.40912914, 0.02653525, -0.04385389, 0.6533296 ,\n", + " 0.756757 , 0.0157517 , 0.01520285, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00133333, -0.01009524, 0.01047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01257143, 0.01045714, 0.01451429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 34, 26, 21],\n", + " [ 33, 25, 20],\n", + " [ 29, 24, 20],\n", + " ...,\n", + " [ 19, 15, 14],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " [ 31, 24, 18],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " ...,\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [205, 226, 182],\n", + " ...,\n", + " [154, 171, 189],\n", + " [156, 172, 190],\n", + " [156, 172, 190]],\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [206, 228, 183],\n", + " ...,\n", + " [154, 171, 189],\n", + " [154, 171, 189],\n", + " [156, 172, 190]],\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [206, 228, 183],\n", + " ...,\n", + " [156, 172, 190],\n", + " [156, 172, 190],\n", + " [154, 171, 189]]], dtype=uint8), 'image': array([[[112, 105, 84],\n", + " [109, 102, 81],\n", + " [ 38, 66, 70],\n", + " ...,\n", + " [ 43, 54, 35],\n", + " [ 42, 52, 40],\n", + " [ 41, 51, 39]],\n", + "\n", + " [[122, 104, 82],\n", + " [123, 105, 83],\n", + " [ 61, 93, 86],\n", + " ...,\n", + " [ 43, 54, 33],\n", + " [ 40, 50, 36],\n", + " [ 40, 50, 36]],\n", + "\n", + " [[118, 91, 69],\n", + " [127, 100, 78],\n", + " [ 75, 106, 90],\n", + " ...,\n", + " [ 42, 53, 32],\n", + " [ 41, 49, 36],\n", + " [ 41, 49, 36]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 75, 52],\n", + " [ 69, 79, 56],\n", + " [ 64, 73, 55],\n", + " ...,\n", + " [ 92, 93, 80],\n", + " [ 97, 98, 87],\n", + " [ 95, 96, 85]],\n", + "\n", + " [[ 55, 66, 45],\n", + " [ 55, 66, 45],\n", + " [ 54, 65, 46],\n", + " ...,\n", + " [ 97, 94, 78],\n", + " [ 99, 97, 85],\n", + " [ 99, 97, 85]],\n", + "\n", + " [[ 54, 64, 48],\n", + " [ 48, 58, 42],\n", + " [ 47, 55, 40],\n", + " ...,\n", + " [ 99, 96, 80],\n", + " [ 99, 98, 83],\n", + " [101, 100, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3698246 , -1.4250783 , 2.3521204 , -2.5393493 , -1.5560521 ,\n", + " 2.7788906 , 0.41707364, 0.0197808 , -0.05207061, 0.6565796 ,\n", + " 0.75394094, 0.01570738, 0.0151506 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00933333, 0.008 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01325714, 0.01108571, 0.0148 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 33, 26, 18],\n", + " [ 33, 26, 18],\n", + " [ 33, 26, 18],\n", + " ...,\n", + " [ 21, 17, 16],\n", + " [ 21, 17, 17],\n", + " [ 20, 16, 16]],\n", + "\n", + " [[ 30, 23, 17],\n", + " [ 30, 23, 17],\n", + " [ 29, 22, 16],\n", + " ...,\n", + " [ 20, 16, 15],\n", + " [ 21, 17, 17],\n", + " [ 21, 17, 17]],\n", + "\n", + " [[ 27, 22, 16],\n", + " [ 26, 21, 15],\n", + " [ 26, 21, 15],\n", + " ...,\n", + " [ 20, 16, 16],\n", + " [ 21, 17, 17],\n", + " [ 21, 17, 17]],\n", + "\n", + " ...,\n", + "\n", + " [[205, 229, 184],\n", + " [205, 229, 184],\n", + " [204, 228, 183],\n", + " ...,\n", + " [164, 175, 189],\n", + " [165, 176, 190],\n", + " [165, 176, 190]],\n", + "\n", + " [[205, 229, 184],\n", + " [205, 229, 184],\n", + " [205, 229, 184],\n", + " ...,\n", + " [165, 176, 190],\n", + " [165, 176, 190],\n", + " [165, 176, 190]],\n", + "\n", + " [[205, 229, 184],\n", + " [206, 230, 185],\n", + " [206, 230, 185],\n", + " ...,\n", + " [164, 175, 189],\n", + " [165, 176, 190],\n", + " [164, 175, 189]]], dtype=uint8), 'image': array([[[110, 107, 75],\n", + " [111, 108, 76],\n", + " [ 41, 73, 64],\n", + " ...,\n", + " [ 76, 50, 50],\n", + " [ 80, 51, 41],\n", + " [ 84, 55, 45]],\n", + "\n", + " [[121, 106, 78],\n", + " [126, 111, 83],\n", + " [ 59, 92, 77],\n", + " ...,\n", + " [ 80, 54, 52],\n", + " [ 84, 55, 43],\n", + " [ 84, 55, 43]],\n", + "\n", + " [[116, 89, 65],\n", + " [130, 103, 79],\n", + " [ 73, 107, 87],\n", + " ...,\n", + " [ 84, 57, 51],\n", + " [ 86, 56, 49],\n", + " [ 87, 57, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 74, 73, 58],\n", + " [ 76, 75, 60],\n", + " [ 72, 71, 56],\n", + " ...,\n", + " [104, 103, 88],\n", + " [104, 103, 88],\n", + " [105, 104, 89]],\n", + "\n", + " [[ 63, 65, 47],\n", + " [ 65, 67, 49],\n", + " [ 62, 64, 48],\n", + " ...,\n", + " [105, 104, 87],\n", + " [106, 108, 92],\n", + " [109, 111, 95]],\n", + "\n", + " [[ 56, 58, 40],\n", + " [ 54, 56, 38],\n", + " [ 49, 51, 33],\n", + " ...,\n", + " [105, 108, 90],\n", + " [103, 108, 94],\n", + " [103, 108, 94]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3830068 , -1.3866261 , 2.3424206 , -2.5729144 , -1.5544709 ,\n", + " 2.7728264 , 0.42537263, 0.01276265, -0.0608653 , 0.65915596,\n", + " 0.7516113 , 0.01741757, 0.01704437, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.0047619 , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01417143, 0.01262857, 0.01457143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[145, 165, 180],\n", + " [147, 167, 182],\n", + " [147, 167, 182],\n", + " ...,\n", + " [ 21, 18, 19],\n", + " [ 21, 17, 17],\n", + " [ 23, 18, 19]],\n", + "\n", + " [[147, 167, 182],\n", + " [147, 167, 182],\n", + " [147, 167, 182],\n", + " ...,\n", + " [ 19, 17, 17],\n", + " [ 23, 18, 19],\n", + " [ 23, 18, 19]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 21, 17, 17],\n", + " [ 23, 18, 19],\n", + " [ 23, 18, 19]],\n", + "\n", + " ...,\n", + "\n", + " [[208, 230, 185],\n", + " [208, 230, 185],\n", + " [207, 229, 184],\n", + " ...,\n", + " [158, 173, 189],\n", + " [161, 174, 190],\n", + " [161, 174, 190]],\n", + "\n", + " [[208, 230, 185],\n", + " [208, 230, 185],\n", + " [208, 230, 185],\n", + " ...,\n", + " [158, 173, 189],\n", + " [160, 172, 189],\n", + " [160, 172, 189]],\n", + "\n", + " [[207, 229, 184],\n", + " [208, 230, 185],\n", + " [209, 231, 187],\n", + " ...,\n", + " [158, 173, 189],\n", + " [160, 172, 189],\n", + " [160, 172, 189]]], dtype=uint8), 'image': array([[[111, 104, 73],\n", + " [111, 104, 73],\n", + " [ 48, 75, 68],\n", + " ...,\n", + " [ 79, 86, 90],\n", + " [ 81, 88, 94],\n", + " [ 82, 89, 95]],\n", + "\n", + " [[120, 101, 72],\n", + " [118, 99, 70],\n", + " [ 66, 94, 79],\n", + " ...,\n", + " [ 78, 83, 88],\n", + " [ 85, 87, 93],\n", + " [ 86, 88, 94]],\n", + "\n", + " [[118, 86, 62],\n", + " [129, 97, 73],\n", + " [ 79, 108, 86],\n", + " ...,\n", + " [ 85, 87, 95],\n", + " [ 85, 87, 93],\n", + " [ 87, 89, 95]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 52],\n", + " [ 72, 79, 57],\n", + " [ 71, 77, 58],\n", + " ...,\n", + " [113, 111, 99],\n", + " [113, 111, 101],\n", + " [111, 109, 99]],\n", + "\n", + " [[ 59, 66, 44],\n", + " [ 58, 65, 43],\n", + " [ 56, 62, 43],\n", + " ...,\n", + " [114, 112, 100],\n", + " [113, 111, 99],\n", + " [114, 112, 100]],\n", + "\n", + " [[ 52, 55, 39],\n", + " [ 51, 54, 38],\n", + " [ 50, 53, 37],\n", + " ...,\n", + " [114, 112, 100],\n", + " [114, 112, 100],\n", + " [115, 113, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.396764 , -1.3478969 , 2.3317156 , -2.606684 , -1.5517172 ,\n", + " 2.7645218 , 0.4339046 , 0.00549856, -0.06976949, 0.6610677 ,\n", + " 0.74982053, 0.01980009, 0.01914801, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00438095, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01577143, 0.01262857, 0.01222857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[156, 168, 180],\n", + " [155, 167, 178],\n", + " [153, 167, 178],\n", + " ...,\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21]],\n", + "\n", + " [[155, 167, 178],\n", + " [155, 167, 178],\n", + " [151, 166, 177],\n", + " ...,\n", + " [ 20, 20, 20],\n", + " [ 20, 20, 20],\n", + " [ 20, 20, 20]],\n", + "\n", + " [[153, 167, 178],\n", + " [151, 166, 177],\n", + " [151, 166, 177],\n", + " ...,\n", + " [ 20, 20, 20],\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21]],\n", + "\n", + " ...,\n", + "\n", + " [[204, 229, 182],\n", + " [204, 229, 182],\n", + " [203, 228, 180],\n", + " ...,\n", + " [148, 168, 190],\n", + " [148, 168, 190],\n", + " [148, 168, 190]],\n", + "\n", + " [[204, 229, 182],\n", + " [203, 228, 180],\n", + " [203, 228, 180],\n", + " ...,\n", + " [150, 170, 192],\n", + " [149, 169, 191],\n", + " [147, 167, 189]],\n", + "\n", + " [[204, 229, 182],\n", + " [204, 229, 182],\n", + " [203, 228, 180],\n", + " ...,\n", + " [150, 170, 192],\n", + " [150, 170, 192],\n", + " [150, 170, 192]]], dtype=uint8), 'image': array([[[111, 103, 89],\n", + " [107, 99, 85],\n", + " [ 44, 74, 79],\n", + " ...,\n", + " [106, 102, 105],\n", + " [ 99, 102, 103],\n", + " [ 99, 102, 103]],\n", + "\n", + " [[123, 107, 84],\n", + " [122, 106, 83],\n", + " [ 61, 93, 86],\n", + " ...,\n", + " [104, 104, 100],\n", + " [102, 102, 102],\n", + " [101, 101, 101]],\n", + "\n", + " [[124, 95, 72],\n", + " [130, 101, 78],\n", + " [ 73, 104, 88],\n", + " ...,\n", + " [105, 102, 97],\n", + " [102, 102, 96],\n", + " [102, 102, 96]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 52],\n", + " [ 74, 77, 59],\n", + " [ 69, 73, 52],\n", + " ...,\n", + " [117, 119, 103],\n", + " [119, 118, 101],\n", + " [113, 112, 95]],\n", + "\n", + " [[ 59, 62, 44],\n", + " [ 60, 63, 45],\n", + " [ 60, 63, 45],\n", + " ...,\n", + " [115, 117, 99],\n", + " [118, 120, 104],\n", + " [119, 121, 105]],\n", + "\n", + " [[ 51, 54, 36],\n", + " [ 53, 56, 38],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [117, 119, 103],\n", + " [119, 121, 105],\n", + " [119, 121, 105]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4100306e+00, -1.3073558e+00, 2.3148079e+00, -2.6330740e+00,\n", + " -1.5499452e+00, 2.7541919e+00, 4.4335869e-01, -2.3262841e-03,\n", + " -7.8644946e-02, 6.6207409e-01, 7.4887121e-01, 2.1015564e-02,\n", + " 2.0203222e-02, 0.0000000e+00, 0.0000000e+00], dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00152381, -0.00171429, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01742857, 0.01297143, 0.01085714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[147, 166, 183],\n", + " [147, 166, 183],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[148, 167, 184],\n", + " [148, 167, 184],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[148, 167, 184],\n", + " [148, 167, 184],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 228, 182],\n", + " [210, 228, 182],\n", + " [209, 227, 179],\n", + " ...,\n", + " [156, 172, 192],\n", + " [156, 172, 192],\n", + " [156, 172, 192]],\n", + "\n", + " [[211, 229, 183],\n", + " [211, 229, 183],\n", + " [210, 228, 180],\n", + " ...,\n", + " [158, 174, 194],\n", + " [157, 173, 193],\n", + " [157, 173, 193]],\n", + "\n", + " [[211, 229, 183],\n", + " [211, 229, 183],\n", + " [211, 230, 181],\n", + " ...,\n", + " [156, 172, 192],\n", + " [156, 172, 192],\n", + " [154, 170, 191]]], dtype=uint8), 'image': array([[[110, 111, 82],\n", + " [104, 105, 76],\n", + " [ 35, 70, 73],\n", + " ...,\n", + " [111, 113, 102],\n", + " [115, 116, 110],\n", + " [115, 116, 110]],\n", + "\n", + " [[124, 112, 81],\n", + " [119, 107, 76],\n", + " [ 49, 84, 76],\n", + " ...,\n", + " [111, 113, 102],\n", + " [114, 115, 109],\n", + " [114, 115, 109]],\n", + "\n", + " [[121, 98, 66],\n", + " [123, 100, 68],\n", + " [ 74, 108, 88],\n", + " ...,\n", + " [110, 112, 103],\n", + " [115, 116, 110],\n", + " [116, 117, 111]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 48],\n", + " [ 71, 78, 52],\n", + " [ 65, 74, 54],\n", + " ...,\n", + " [122, 123, 112],\n", + " [121, 123, 112],\n", + " [121, 123, 112]],\n", + "\n", + " [[ 60, 68, 39],\n", + " [ 60, 68, 39],\n", + " [ 52, 64, 40],\n", + " ...,\n", + " [120, 121, 108],\n", + " [124, 125, 112],\n", + " [124, 125, 112]],\n", + "\n", + " [[ 55, 61, 42],\n", + " [ 51, 57, 38],\n", + " [ 44, 53, 35],\n", + " ...,\n", + " [122, 123, 110],\n", + " [122, 123, 110],\n", + " [122, 123, 110]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4241116 , -1.2713832 , 2.294727 , -2.6514275 , -1.5482687 ,\n", + " 2.7406254 , 0.4530185 , -0.01090289, -0.08589576, 0.66217947,\n", + " 0.74872 , 0.02204888, 0.02122683, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00380952, -0.0007619 , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01782857, 0.01325714, 0.01085714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[149, 166, 182],\n", + " [149, 166, 182],\n", + " [151, 166, 182],\n", + " ...,\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22],\n", + " [ 25, 23, 23]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 24, 23, 21],\n", + " [ 24, 22, 22],\n", + " [ 25, 23, 23]],\n", + "\n", + " ...,\n", + "\n", + " [[212, 228, 180],\n", + " [212, 228, 180],\n", + " [211, 226, 180],\n", + " ...,\n", + " [159, 173, 191],\n", + " [159, 173, 191],\n", + " [159, 173, 191]],\n", + "\n", + " [[213, 229, 181],\n", + " [212, 228, 180],\n", + " [213, 228, 183],\n", + " ...,\n", + " [158, 172, 190],\n", + " [158, 172, 190],\n", + " [158, 172, 190]],\n", + "\n", + " [[213, 229, 181],\n", + " [213, 229, 181],\n", + " [213, 228, 183],\n", + " ...,\n", + " [156, 171, 189],\n", + " [158, 172, 190],\n", + " [158, 172, 190]]], dtype=uint8), 'image': array([[[105, 107, 73],\n", + " [107, 109, 75],\n", + " [ 37, 72, 64],\n", + " ...,\n", + " [183, 184, 175],\n", + " [171, 167, 170],\n", + " [167, 163, 166]],\n", + "\n", + " [[116, 105, 71],\n", + " [120, 109, 75],\n", + " [ 53, 86, 71],\n", + " ...,\n", + " [179, 176, 173],\n", + " [170, 166, 169],\n", + " [163, 159, 162]],\n", + "\n", + " [[124, 101, 71],\n", + " [124, 101, 71],\n", + " [ 75, 107, 88],\n", + " ...,\n", + " [179, 173, 175],\n", + " [167, 166, 168],\n", + " [160, 159, 161]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 73, 54],\n", + " [ 72, 78, 59],\n", + " [ 70, 76, 59],\n", + " ...,\n", + " [126, 127, 114],\n", + " [126, 127, 116],\n", + " [123, 124, 113]],\n", + "\n", + " [[ 58, 64, 45],\n", + " [ 57, 63, 44],\n", + " [ 53, 62, 42],\n", + " ...,\n", + " [127, 126, 111],\n", + " [127, 125, 113],\n", + " [128, 126, 114]],\n", + "\n", + " [[ 54, 57, 39],\n", + " [ 51, 54, 36],\n", + " [ 44, 53, 33],\n", + " ...,\n", + " [129, 128, 113],\n", + " [127, 128, 115],\n", + " [127, 128, 115]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4396536 , -1.2382988 , 2.2719407 , -2.66238 , -1.5463403 ,\n", + " 2.7252848 , 0.46257424, -0.020392 , -0.09226454, 0.66218996,\n", + " 0.74868333, 0.02275231, 0.02144971, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00285714, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01782857, 0.01342857, 0.01022857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[137, 159, 181],\n", + " [137, 159, 181],\n", + " [138, 160, 182],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 20, 18],\n", + " [ 28, 24, 23]],\n", + "\n", + " [[137, 159, 181],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[137, 159, 181],\n", + " [137, 159, 181],\n", + " [137, 159, 181],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22]],\n", + "\n", + " ...,\n", + "\n", + " [[211, 227, 179],\n", + " [211, 227, 179],\n", + " [211, 227, 179],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 168, 190],\n", + " [150, 168, 190]],\n", + "\n", + " [[211, 227, 179],\n", + " [211, 227, 179],\n", + " [211, 227, 179],\n", + " ...,\n", + " [151, 169, 191],\n", + " [149, 166, 189],\n", + " [149, 166, 189]],\n", + "\n", + " [[210, 226, 178],\n", + " [210, 226, 178],\n", + " [211, 227, 179],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 168, 190],\n", + " [148, 165, 187]]], dtype=uint8), 'image': array([[[111, 106, 75],\n", + " [107, 102, 71],\n", + " [ 53, 78, 69],\n", + " ...,\n", + " [162, 162, 162],\n", + " [167, 167, 169],\n", + " [173, 173, 175]],\n", + "\n", + " [[119, 104, 74],\n", + " [121, 106, 76],\n", + " [ 65, 92, 74],\n", + " ...,\n", + " [161, 161, 161],\n", + " [167, 167, 167],\n", + " [175, 175, 175]],\n", + "\n", + " [[111, 88, 58],\n", + " [125, 102, 72],\n", + " [ 79, 106, 84],\n", + " ...,\n", + " [159, 162, 161],\n", + " [168, 168, 168],\n", + " [174, 174, 174]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 73, 44],\n", + " [ 70, 78, 49],\n", + " [ 69, 76, 52],\n", + " ...,\n", + " [131, 132, 121],\n", + " [134, 132, 122],\n", + " [133, 131, 121]],\n", + "\n", + " [[ 60, 68, 39],\n", + " [ 58, 66, 37],\n", + " [ 56, 66, 41],\n", + " ...,\n", + " [133, 131, 119],\n", + " [136, 133, 119],\n", + " [139, 136, 122]],\n", + "\n", + " [[ 55, 59, 34],\n", + " [ 50, 54, 29],\n", + " [ 46, 56, 33],\n", + " ...,\n", + " [137, 134, 118],\n", + " [141, 135, 120],\n", + " [139, 133, 118]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4565248 , -1.2037634 , 2.2444725 , -2.6683176 , -1.5432638 ,\n", + " 2.7085657 , 0.47299433, -0.03085847, -0.0988073 , 0.6621888 ,\n", + " 0.74867994, 0.02358894, 0.0206851 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00380952, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01788571, 0.01325714, 0.01051429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[143, 159, 179],\n", + " [145, 161, 181],\n", + " [144, 160, 180],\n", + " ...,\n", + " [ 27, 23, 22],\n", + " [ 27, 23, 22],\n", + " [ 27, 23, 22]],\n", + "\n", + " [[143, 159, 179],\n", + " [144, 160, 180],\n", + " [143, 159, 179],\n", + " ...,\n", + " [ 24, 20, 18],\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19]],\n", + "\n", + " [[141, 159, 179],\n", + " [141, 159, 179],\n", + " [141, 159, 179],\n", + " ...,\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 227, 176],\n", + " [210, 227, 176],\n", + " [208, 225, 175],\n", + " ...,\n", + " [147, 164, 186],\n", + " [150, 165, 187],\n", + " [150, 165, 187]],\n", + "\n", + " [[208, 225, 175],\n", + " [208, 225, 175],\n", + " [208, 225, 175],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 165, 187],\n", + " [150, 165, 187]],\n", + "\n", + " [[207, 224, 174],\n", + " [208, 225, 175],\n", + " [208, 225, 175],\n", + " ...,\n", + " [149, 166, 189],\n", + " [149, 164, 186],\n", + " [150, 165, 187]]], dtype=uint8), 'image': array([[[108, 104, 68],\n", + " [109, 105, 69],\n", + " [ 45, 71, 56],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " [[120, 103, 66],\n", + " [125, 108, 71],\n", + " [ 66, 93, 71],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " [[119, 89, 57],\n", + " [128, 98, 66],\n", + " [ 80, 108, 81],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 73, 50],\n", + " [ 76, 80, 57],\n", + " [ 73, 76, 58],\n", + " ...,\n", + " [137, 133, 124],\n", + " [141, 134, 124],\n", + " [137, 130, 120]],\n", + "\n", + " [[ 59, 62, 37],\n", + " [ 60, 63, 38],\n", + " [ 60, 63, 38],\n", + " ...,\n", + " [136, 134, 124],\n", + " [136, 134, 124],\n", + " [138, 136, 126]],\n", + "\n", + " [[ 53, 56, 33],\n", + " [ 48, 51, 28],\n", + " [ 48, 51, 26],\n", + " ...,\n", + " [136, 137, 126],\n", + " [139, 137, 129],\n", + " [140, 138, 130]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.471241 , -1.1722492 , 2.2166338 , -2.6705124 , -1.5408925 ,\n", + " 2.6938715 , 0.4829263 , -0.04071204, -0.10468176, 0.6621572 ,\n", + " 0.74871343, 0.02410153, 0.0198809 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0047619, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01811429, 0.01285714, 0.01 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[136, 158, 179],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 21, 19, 17],\n", + " [ 21, 19, 17],\n", + " [ 23, 21, 19]],\n", + "\n", + " [[137, 159, 181],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18],\n", + " [ 23, 21, 19]],\n", + "\n", + " [[138, 161, 180],\n", + " [138, 161, 180],\n", + " [137, 160, 179],\n", + " ...,\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[204, 221, 170],\n", + " [204, 221, 170],\n", + " [203, 220, 169],\n", + " ...,\n", + " [141, 158, 181],\n", + " [142, 159, 182],\n", + " [142, 159, 182]],\n", + "\n", + " [[204, 221, 170],\n", + " [203, 220, 169],\n", + " [203, 220, 169],\n", + " ...,\n", + " [142, 159, 182],\n", + " [142, 159, 182],\n", + " [141, 158, 181]],\n", + "\n", + " [[204, 221, 170],\n", + " [203, 220, 169],\n", + " [204, 221, 170],\n", + " ...,\n", + " [141, 158, 181],\n", + " [142, 159, 182],\n", + " [141, 158, 181]]], dtype=uint8), 'image': array([[[107, 101, 63],\n", + " [109, 103, 65],\n", + " [ 47, 75, 62],\n", + " ...,\n", + " [155, 155, 157],\n", + " [155, 155, 151],\n", + " [156, 156, 152]],\n", + "\n", + " [[118, 102, 68],\n", + " [122, 106, 72],\n", + " [ 57, 86, 68],\n", + " ...,\n", + " [157, 157, 159],\n", + " [156, 156, 152],\n", + " [156, 156, 152]],\n", + "\n", + " [[123, 94, 71],\n", + " [125, 96, 73],\n", + " [ 66, 98, 77],\n", + " ...,\n", + " [158, 157, 159],\n", + " [156, 156, 152],\n", + " [156, 156, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 70, 55],\n", + " [ 77, 76, 61],\n", + " [ 76, 78, 60],\n", + " ...,\n", + " [140, 141, 130],\n", + " [140, 141, 128],\n", + " [134, 135, 122]],\n", + "\n", + " [[ 63, 62, 45],\n", + " [ 65, 64, 47],\n", + " [ 58, 57, 42],\n", + " ...,\n", + " [143, 141, 133],\n", + " [140, 141, 128],\n", + " [137, 138, 125]],\n", + "\n", + " [[ 56, 53, 39],\n", + " [ 57, 54, 40],\n", + " [ 53, 51, 39],\n", + " ...,\n", + " [142, 138, 129],\n", + " [142, 140, 128],\n", + " [143, 141, 129]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4855387 , -1.1409982 , 2.1876225 , -2.6713645 , -1.5381626 ,\n", + " 2.679764 , 0.49294764, -0.05065339, -0.11059836, 0.66218203,\n", + " 0.74868876, 0.02483457, 0.0190603 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00533333, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01862857, 0.01217143, 0.00902857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 23, 19, 17]],\n", + "\n", + " [[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " [[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[199, 216, 166],\n", + " [199, 216, 166],\n", + " [198, 215, 164],\n", + " ...,\n", + " [135, 150, 172],\n", + " [136, 151, 174],\n", + " [136, 151, 174]],\n", + "\n", + " [[198, 215, 164],\n", + " [198, 215, 164],\n", + " [198, 215, 164],\n", + " ...,\n", + " [136, 151, 174],\n", + " [136, 151, 174],\n", + " [136, 151, 174]],\n", + "\n", + " [[198, 215, 164],\n", + " [198, 215, 164],\n", + " [198, 215, 164],\n", + " ...,\n", + " [136, 151, 174],\n", + " [136, 151, 174],\n", + " [136, 151, 174]]], dtype=uint8), 'image': array([[[113, 102, 79],\n", + " [112, 101, 78],\n", + " [ 54, 74, 69],\n", + " ...,\n", + " [224, 220, 209],\n", + " [224, 220, 211],\n", + " [225, 221, 212]],\n", + "\n", + " [[120, 101, 76],\n", + " [118, 99, 74],\n", + " [ 61, 85, 71],\n", + " ...,\n", + " [224, 220, 211],\n", + " [226, 219, 211],\n", + " [228, 221, 213]],\n", + "\n", + " [[116, 84, 62],\n", + " [128, 96, 74],\n", + " [ 76, 101, 82],\n", + " ...,\n", + " [224, 220, 211],\n", + " [226, 222, 213],\n", + " [227, 223, 214]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 73, 50],\n", + " [ 73, 76, 53],\n", + " [ 69, 72, 49],\n", + " ...,\n", + " [140, 140, 134],\n", + " [139, 139, 133],\n", + " [140, 140, 134]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 58, 62, 37],\n", + " [ 59, 63, 38],\n", + " ...,\n", + " [142, 140, 132],\n", + " [141, 138, 133],\n", + " [145, 142, 137]],\n", + "\n", + " [[ 52, 55, 32],\n", + " [ 49, 52, 29],\n", + " [ 48, 52, 31],\n", + " ...,\n", + " [145, 143, 133],\n", + " [142, 137, 133],\n", + " [146, 141, 137]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.500686 , -1.1089195 , 2.1561332 , -2.6699607 , -1.5347141 ,\n", + " 2.6646018 , 0.5029847 , -0.06102105, -0.11659307, 0.6621161 ,\n", + " 0.7487434 , 0.02574205, 0.0179702 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00761905, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01902857, 0.0112 , 0.00971429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[118, 147, 169],\n", + " [118, 147, 169],\n", + " [119, 148, 170],\n", + " ...,\n", + " [ 18, 17, 15],\n", + " [ 20, 16, 15],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[119, 145, 168],\n", + " [118, 144, 167],\n", + " [117, 145, 168],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 20, 16, 15],\n", + " [ 23, 19, 17]],\n", + "\n", + " [[118, 144, 167],\n", + " [118, 144, 167],\n", + " [117, 145, 168],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[177, 172, 159],\n", + " [178, 173, 160],\n", + " [179, 177, 157],\n", + " ...,\n", + " [138, 151, 174],\n", + " [138, 151, 174],\n", + " [138, 151, 174]],\n", + "\n", + " [[176, 171, 158],\n", + " [177, 172, 159],\n", + " [178, 176, 156],\n", + " ...,\n", + " [139, 152, 175],\n", + " [138, 151, 174],\n", + " [139, 152, 175]],\n", + "\n", + " [[173, 169, 156],\n", + " [175, 170, 157],\n", + " [177, 175, 154],\n", + " ...,\n", + " [140, 153, 176],\n", + " [140, 153, 176],\n", + " [141, 154, 177]]], dtype=uint8), 'image': array([[[107, 106, 78],\n", + " [101, 100, 72],\n", + " [ 36, 74, 57],\n", + " ...,\n", + " [207, 203, 192],\n", + " [198, 190, 187],\n", + " [186, 178, 175]],\n", + "\n", + " [[120, 108, 81],\n", + " [110, 98, 71],\n", + " [ 58, 95, 72],\n", + " ...,\n", + " [205, 201, 192],\n", + " [198, 192, 194],\n", + " [186, 180, 182]],\n", + "\n", + " [[108, 84, 57],\n", + " [122, 98, 71],\n", + " [ 72, 106, 82],\n", + " ...,\n", + " [207, 199, 196],\n", + " [197, 191, 195],\n", + " [186, 180, 184]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 53],\n", + " [ 72, 75, 57],\n", + " [ 70, 72, 56],\n", + " ...,\n", + " [146, 142, 133],\n", + " [144, 140, 131],\n", + " [142, 138, 129]],\n", + "\n", + " [[ 60, 64, 43],\n", + " [ 60, 64, 43],\n", + " [ 59, 63, 42],\n", + " ...,\n", + " [148, 141, 133],\n", + " [150, 143, 135],\n", + " [148, 141, 133]],\n", + "\n", + " [[ 54, 57, 36],\n", + " [ 54, 57, 36],\n", + " [ 48, 51, 33],\n", + " ...,\n", + " [149, 142, 134],\n", + " [148, 141, 133],\n", + " [149, 142, 134]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5155468 , -1.0804976 , 2.1259022 , -2.6659186 , -1.5311936 ,\n", + " 2.6497624 , 0.51242495, -0.07151319, -0.12196171, 0.66212773,\n", + " 0.74872553, 0.02683827, 0.01663197, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00590476, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01891429, 0.01205714, 0.01451429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 150, 173],\n", + " [120, 149, 172],\n", + " [121, 150, 173],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 16, 14, 14],\n", + " [ 17, 15, 15]],\n", + "\n", + " [[119, 148, 170],\n", + " [119, 148, 170],\n", + " [118, 147, 169],\n", + " ...,\n", + " [ 15, 12, 13],\n", + " [ 16, 14, 14],\n", + " [ 16, 14, 14]],\n", + "\n", + " [[121, 147, 170],\n", + " [121, 147, 170],\n", + " [120, 146, 169],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 160, 168],\n", + " [154, 160, 168],\n", + " [154, 159, 170],\n", + " ...,\n", + " [152, 157, 175],\n", + " [152, 157, 175],\n", + " [152, 157, 175]],\n", + "\n", + " [[154, 160, 168],\n", + " [153, 159, 167],\n", + " [154, 159, 170],\n", + " ...,\n", + " [152, 157, 175],\n", + " [152, 157, 175],\n", + " [152, 157, 175]],\n", + "\n", + " [[154, 160, 168],\n", + " [154, 160, 168],\n", + " [153, 158, 169],\n", + " ...,\n", + " [151, 156, 173],\n", + " [151, 156, 173],\n", + " [151, 156, 173]]], dtype=uint8), 'image': array([[[111, 100, 77],\n", + " [113, 102, 79],\n", + " [ 51, 71, 68],\n", + " ...,\n", + " [137, 132, 128],\n", + " [136, 136, 130],\n", + " [138, 138, 132]],\n", + "\n", + " [[120, 101, 76],\n", + " [124, 105, 80],\n", + " [ 62, 86, 74],\n", + " ...,\n", + " [139, 136, 131],\n", + " [138, 135, 130],\n", + " [140, 137, 132]],\n", + "\n", + " [[120, 86, 62],\n", + " [130, 96, 72],\n", + " [ 80, 102, 84],\n", + " ...,\n", + " [138, 135, 130],\n", + " [140, 137, 132],\n", + " [141, 138, 133]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 72, 49],\n", + " [ 74, 77, 54],\n", + " [ 73, 76, 55],\n", + " ...,\n", + " [148, 144, 137],\n", + " [148, 144, 135],\n", + " [143, 139, 130]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 60, 64, 39],\n", + " [ 59, 62, 39],\n", + " ...,\n", + " [148, 144, 135],\n", + " [149, 145, 136],\n", + " [147, 143, 134]],\n", + "\n", + " [[ 53, 56, 31],\n", + " [ 52, 55, 30],\n", + " [ 45, 49, 24],\n", + " ...,\n", + " [149, 145, 136],\n", + " [149, 145, 136],\n", + " [149, 145, 136]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5310063 , -1.052627 , 2.096078 , -2.660988 , -1.5265115 ,\n", + " 2.6343114 , 0.5215374 , -0.08259036, -0.12778392, 0.66204727,\n", + " 0.74878216, 0.0282663 , 0.01482305, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00361905, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01805714, 0.01268571, 0.01908571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 145, 168],\n", + " [120, 144, 167],\n", + " [120, 144, 167],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 19, 15, 14],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[120, 144, 167],\n", + " [120, 144, 167],\n", + " [118, 142, 166],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 19, 15, 14],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[120, 144, 167],\n", + " [120, 144, 167],\n", + " [118, 142, 166],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 157, 175],\n", + " [154, 157, 175],\n", + " [154, 158, 173],\n", + " ...,\n", + " [150, 157, 176],\n", + " [150, 157, 176],\n", + " [150, 157, 176]],\n", + "\n", + " [[150, 153, 171],\n", + " [150, 153, 171],\n", + " [151, 155, 171],\n", + " ...,\n", + " [151, 158, 177],\n", + " [151, 158, 177],\n", + " [150, 157, 176]],\n", + "\n", + " [[147, 150, 168],\n", + " [148, 151, 169],\n", + " [149, 153, 168],\n", + " ...,\n", + " [152, 159, 178],\n", + " [151, 158, 177],\n", + " [151, 158, 177]]], dtype=uint8), 'image': array([[[113, 102, 79],\n", + " [107, 96, 73],\n", + " [ 49, 73, 71],\n", + " ...,\n", + " [159, 158, 158],\n", + " [158, 157, 161],\n", + " [158, 157, 161]],\n", + "\n", + " [[118, 99, 74],\n", + " [119, 100, 75],\n", + " [ 61, 86, 77],\n", + " ...,\n", + " [159, 159, 161],\n", + " [157, 157, 161],\n", + " [154, 154, 158]],\n", + "\n", + " [[109, 77, 53],\n", + " [135, 103, 79],\n", + " [ 77, 104, 86],\n", + " ...,\n", + " [158, 157, 164],\n", + " [152, 151, 160],\n", + " [147, 146, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 69, 49],\n", + " [ 72, 72, 52],\n", + " [ 66, 69, 46],\n", + " ...,\n", + " [148, 146, 138],\n", + " [147, 144, 141],\n", + " [146, 143, 140]],\n", + "\n", + " [[ 63, 63, 41],\n", + " [ 62, 62, 40],\n", + " [ 57, 57, 37],\n", + " ...,\n", + " [145, 146, 137],\n", + " [146, 143, 140],\n", + " [146, 143, 140]],\n", + "\n", + " [[ 57, 55, 34],\n", + " [ 56, 54, 33],\n", + " [ 50, 50, 28],\n", + " ...,\n", + " [148, 149, 138],\n", + " [148, 145, 140],\n", + " [149, 146, 141]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5453887 , -1.0169486 , 2.0630584 , -2.6606886 , -1.5223564 ,\n", + " 2.6199625 , 0.531208 , -0.0935498 , -0.13631946, 0.6620134 ,\n", + " 0.748793 , 0.02956677, 0.01315534, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0.0007619 , 0.00152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01308571, 0.01091429, 0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[100, 127, 159],\n", + " [ 98, 126, 158],\n", + " [100, 127, 159],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 12, 12, 12],\n", + " [ 12, 12, 12]],\n", + "\n", + " [[ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " [[100, 127, 159],\n", + " [ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " ...,\n", + "\n", + " [[146, 148, 168],\n", + " [146, 148, 168],\n", + " [142, 147, 167],\n", + " ...,\n", + " [149, 153, 175],\n", + " [149, 153, 175],\n", + " [149, 153, 175]],\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [145, 149, 169],\n", + " ...,\n", + " [149, 153, 175],\n", + " [149, 153, 175],\n", + " [149, 153, 175]],\n", + "\n", + " [[146, 148, 168],\n", + " [147, 149, 169],\n", + " [145, 149, 169],\n", + " ...,\n", + " [152, 156, 178],\n", + " [149, 153, 175],\n", + " [149, 153, 175]]], dtype=uint8), 'image': array([[[105, 98, 77],\n", + " [103, 96, 75],\n", + " [ 48, 72, 74],\n", + " ...,\n", + " [158, 158, 154],\n", + " [157, 152, 148],\n", + " [159, 154, 150]],\n", + "\n", + " [[119, 101, 79],\n", + " [113, 95, 73],\n", + " [ 53, 80, 75],\n", + " ...,\n", + " [156, 155, 155],\n", + " [156, 153, 150],\n", + " [156, 153, 150]],\n", + "\n", + " [[124, 94, 76],\n", + " [123, 93, 75],\n", + " [ 69, 97, 84],\n", + " ...,\n", + " [156, 155, 157],\n", + " [148, 147, 149],\n", + " [148, 147, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 71, 54],\n", + " [ 78, 77, 60],\n", + " [ 76, 75, 58],\n", + " ...,\n", + " [151, 144, 134],\n", + " [147, 145, 135],\n", + " [145, 143, 133]],\n", + "\n", + " [[ 59, 63, 42],\n", + " [ 57, 61, 40],\n", + " [ 55, 58, 37],\n", + " ...,\n", + " [151, 144, 134],\n", + " [148, 144, 135],\n", + " [148, 144, 135]],\n", + "\n", + " [[ 50, 53, 35],\n", + " [ 52, 55, 37],\n", + " [ 48, 51, 33],\n", + " ...,\n", + " [151, 144, 134],\n", + " [149, 145, 136],\n", + " [149, 145, 136]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.556693 , -0.9751294 , 2.0289373 , -2.666302 , -1.5200332 ,\n", + " 2.6086464 , 0.5411583 , -0.10370405, -0.14767449, 0.66197366,\n", + " 0.748819 , 0.03023103, 0.01212799, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0.00095238, 0.01142857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00805714, 0. , 0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[102, 127, 159],\n", + " [100, 126, 158],\n", + " [102, 128, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " [[102, 127, 159],\n", + " [100, 126, 158],\n", + " [100, 127, 157],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14]],\n", + "\n", + " [[104, 128, 158],\n", + " [104, 128, 158],\n", + " [102, 128, 158],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [149, 152, 170],\n", + " [150, 153, 171],\n", + " [150, 153, 171]],\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [149, 152, 170],\n", + " [148, 151, 169],\n", + " [149, 152, 170]],\n", + "\n", + " [[146, 148, 168],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " [148, 151, 169]]], dtype=uint8), 'image': array([[[108, 103, 74],\n", + " [105, 100, 71],\n", + " [ 44, 71, 62],\n", + " ...,\n", + " [158, 154, 155],\n", + " [158, 154, 157],\n", + " [157, 153, 156]],\n", + "\n", + " [[119, 102, 73],\n", + " [123, 106, 77],\n", + " [ 62, 93, 77],\n", + " ...,\n", + " [155, 154, 154],\n", + " [156, 155, 157],\n", + " [154, 153, 155]],\n", + "\n", + " [[120, 91, 66],\n", + " [127, 98, 73],\n", + " [ 71, 100, 80],\n", + " ...,\n", + " [152, 152, 156],\n", + " [148, 147, 151],\n", + " [148, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 68, 49],\n", + " [ 67, 73, 54],\n", + " [ 64, 71, 49],\n", + " ...,\n", + " [150, 146, 137],\n", + " [147, 143, 134],\n", + " [142, 138, 129]],\n", + "\n", + " [[ 56, 63, 41],\n", + " [ 56, 63, 41],\n", + " [ 57, 64, 42],\n", + " ...,\n", + " [148, 144, 135],\n", + " [148, 144, 135],\n", + " [148, 144, 135]],\n", + "\n", + " [[ 50, 56, 39],\n", + " [ 50, 56, 39],\n", + " [ 46, 52, 33],\n", + " ...,\n", + " [148, 146, 136],\n", + " [147, 145, 135],\n", + " [147, 145, 135]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5630758 , -0.9333814 , 1.9979777 , -2.6762102 , -1.5196503 ,\n", + " 2.6031442 , 0.5501979 , -0.11132966, -0.16009057, 0.66227996,\n", + " 0.7485525 , 0.03024558, 0.01181715, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00704762, 0. , 0.02038095], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01777143, 0. , 0.00914286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[112, 130, 150],\n", + " [ 97, 115, 135],\n", + " [ 83, 101, 109],\n", + " ...,\n", + " [ 17, 15, 15],\n", + " [ 18, 16, 16],\n", + " [ 18, 16, 16]],\n", + "\n", + " [[112, 129, 151],\n", + " [102, 120, 142],\n", + " [ 89, 106, 117],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15]],\n", + "\n", + " [[113, 133, 155],\n", + " [105, 125, 147],\n", + " [ 97, 113, 126],\n", + " ...,\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 148, 168],\n", + " [144, 148, 168],\n", + " [144, 148, 168],\n", + " ...,\n", + " [139, 149, 172],\n", + " [139, 149, 172],\n", + " [139, 149, 172]],\n", + "\n", + " [[139, 144, 163],\n", + " [140, 145, 165],\n", + " [139, 144, 163],\n", + " ...,\n", + " [140, 150, 174],\n", + " [139, 149, 172],\n", + " [140, 150, 174]],\n", + "\n", + " [[139, 144, 163],\n", + " [139, 144, 163],\n", + " [139, 144, 163],\n", + " ...,\n", + " [140, 150, 174],\n", + " [140, 150, 174],\n", + " [139, 149, 172]]], dtype=uint8), 'image': array([[[107, 100, 71],\n", + " [109, 102, 73],\n", + " [ 46, 69, 65],\n", + " ...,\n", + " [152, 155, 158],\n", + " [153, 156, 157],\n", + " [152, 155, 156]],\n", + "\n", + " [[114, 95, 68],\n", + " [122, 103, 76],\n", + " [ 52, 78, 65],\n", + " ...,\n", + " [150, 153, 156],\n", + " [152, 155, 156],\n", + " [151, 154, 155]],\n", + "\n", + " [[118, 86, 64],\n", + " [127, 95, 73],\n", + " [ 76, 101, 82],\n", + " ...,\n", + " [150, 152, 158],\n", + " [149, 152, 155],\n", + " [148, 151, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 73, 76, 53],\n", + " [ 75, 78, 55],\n", + " [ 68, 72, 47],\n", + " ...,\n", + " [150, 148, 140],\n", + " [148, 146, 138],\n", + " [141, 139, 131]],\n", + "\n", + " [[ 58, 61, 38],\n", + " [ 59, 62, 39],\n", + " [ 55, 58, 35],\n", + " ...,\n", + " [150, 146, 139],\n", + " [150, 146, 137],\n", + " [147, 143, 134]],\n", + "\n", + " [[ 53, 53, 33],\n", + " [ 51, 51, 31],\n", + " [ 49, 52, 31],\n", + " ...,\n", + " [148, 144, 137],\n", + " [148, 144, 137],\n", + " [150, 146, 139]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.569471 , -0.8993924 , 1.9853354 , -2.6972144 , -1.5198654 ,\n", + " 2.60294 , 0.5531312 , -0.11650646, -0.1728949 , 0.6645783 ,\n", + " 0.7465192 , 0.03010343, 0.01176925, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00095238, -0.00552381, 0.01314286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01902857, -0.00474286, 0.00297143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 150, 168],\n", + " [147, 150, 168],\n", + " [147, 150, 168],\n", + " ...,\n", + " [135, 146, 169],\n", + " [136, 147, 170],\n", + " [136, 147, 170]],\n", + "\n", + " [[148, 151, 169],\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " ...,\n", + " [133, 143, 167],\n", + " [134, 144, 168],\n", + " [134, 144, 168]],\n", + "\n", + " [[149, 152, 170],\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " ...,\n", + " [133, 143, 167],\n", + " [134, 144, 168],\n", + " [134, 144, 168]]], dtype=uint8), 'image': array([[[109, 101, 77],\n", + " [109, 101, 77],\n", + " [ 42, 67, 62],\n", + " ...,\n", + " [155, 155, 151],\n", + " [156, 151, 149],\n", + " [157, 152, 150]],\n", + "\n", + " [[122, 103, 78],\n", + " [124, 105, 80],\n", + " [ 62, 86, 72],\n", + " ...,\n", + " [152, 152, 152],\n", + " [152, 151, 151],\n", + " [153, 152, 152]],\n", + "\n", + " [[126, 93, 76],\n", + " [127, 94, 77],\n", + " [ 78, 105, 85],\n", + " ...,\n", + " [150, 153, 154],\n", + " [151, 151, 153],\n", + " [146, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 70, 48],\n", + " [ 67, 74, 52],\n", + " [ 66, 73, 51],\n", + " ...,\n", + " [148, 146, 136],\n", + " [146, 143, 138],\n", + " [143, 140, 135]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 57, 63, 44],\n", + " [ 60, 63, 45],\n", + " ...,\n", + " [149, 144, 140],\n", + " [149, 144, 140],\n", + " [147, 142, 138]],\n", + "\n", + " [[ 51, 52, 41],\n", + " [ 52, 53, 42],\n", + " [ 47, 50, 34],\n", + " ...,\n", + " [152, 145, 139],\n", + " [149, 144, 140],\n", + " [148, 143, 139]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5539145 , -0.88672715, 1.9930334 , -2.7198975 , -1.5237097 ,\n", + " 2.630835 , 0.5523699 , -0.10781911, -0.17971404, 0.6691864 ,\n", + " 0.7424202 , 0.02894053, 0.0128143 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00266667, -0.00590476, 0.0112381 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01988572, -0.00342857, 0.0024 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 150, 168],\n", + " [147, 150, 168],\n", + " [147, 150, 168],\n", + " ...,\n", + " [134, 144, 168],\n", + " [134, 144, 168],\n", + " [134, 144, 168]],\n", + "\n", + " [[148, 151, 169],\n", + " [148, 151, 169],\n", + " [147, 150, 168],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [132, 142, 165]],\n", + "\n", + " [[148, 151, 169],\n", + " [147, 150, 168],\n", + " [146, 149, 166],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [132, 142, 165]]], dtype=uint8), 'image': array([[[104, 101, 71],\n", + " [101, 98, 68],\n", + " [ 43, 70, 63],\n", + " ...,\n", + " [157, 157, 161],\n", + " [155, 154, 156],\n", + " [154, 153, 155]],\n", + "\n", + " [[116, 101, 73],\n", + " [119, 104, 76],\n", + " [ 63, 91, 76],\n", + " ...,\n", + " [154, 154, 158],\n", + " [154, 153, 157],\n", + " [152, 151, 155]],\n", + "\n", + " [[110, 86, 59],\n", + " [125, 101, 74],\n", + " [ 75, 102, 80],\n", + " ...,\n", + " [152, 155, 158],\n", + " [146, 146, 150],\n", + " [147, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 69, 46],\n", + " [ 69, 72, 49],\n", + " [ 65, 68, 52],\n", + " ...,\n", + " [146, 145, 130],\n", + " [147, 143, 134],\n", + " [146, 142, 133]],\n", + "\n", + " [[ 62, 65, 40],\n", + " [ 61, 64, 39],\n", + " [ 55, 57, 41],\n", + " ...,\n", + " [146, 144, 132],\n", + " [146, 143, 138],\n", + " [147, 144, 139]],\n", + "\n", + " [[ 55, 55, 35],\n", + " [ 54, 54, 34],\n", + " [ 46, 48, 30],\n", + " ...,\n", + " [146, 144, 136],\n", + " [143, 143, 137],\n", + " [143, 143, 137]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5357125 , -0.88817865, 2.0125475 , -2.7420037 , -1.524392 ,\n", + " 2.6583688 , 0.5487908 , -0.09566072, -0.18201101, 0.6725909 ,\n", + " 0.7392859 , 0.02962112, 0.01415604, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00285714, -0.00609524, 0.01180952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01977143, -0.00291429, 0.00234286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 13, 11],\n", + " [ 15, 13, 11],\n", + " [ 14, 12, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 14, 12, 10],\n", + " [ 14, 12, 10],\n", + " [ 12, 11, 9],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14]],\n", + "\n", + " [[ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 10, 11, 9],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [139, 146, 167],\n", + " [138, 146, 170],\n", + " [138, 146, 170]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [138, 144, 166],\n", + " [136, 144, 168],\n", + " [135, 143, 167]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [138, 144, 166],\n", + " [135, 143, 167],\n", + " [135, 143, 167]]], dtype=uint8), 'image': array([[[109, 97, 68],\n", + " [112, 100, 71],\n", + " [ 49, 69, 64],\n", + " ...,\n", + " [153, 152, 152],\n", + " [156, 155, 155],\n", + " [155, 154, 154]],\n", + "\n", + " [[119, 97, 71],\n", + " [124, 102, 76],\n", + " [ 59, 83, 69],\n", + " ...,\n", + " [151, 151, 151],\n", + " [150, 150, 150],\n", + " [150, 150, 150]],\n", + "\n", + " [[117, 83, 59],\n", + " [128, 94, 70],\n", + " [ 74, 99, 80],\n", + " ...,\n", + " [150, 150, 154],\n", + " [145, 145, 147],\n", + " [144, 144, 146]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 71, 43],\n", + " [ 72, 77, 49],\n", + " [ 67, 71, 48],\n", + " ...,\n", + " [141, 142, 133],\n", + " [140, 140, 134],\n", + " [139, 139, 133]],\n", + "\n", + " [[ 54, 59, 31],\n", + " [ 55, 60, 32],\n", + " [ 52, 59, 33],\n", + " ...,\n", + " [143, 144, 133],\n", + " [143, 144, 135],\n", + " [143, 144, 135]],\n", + "\n", + " [[ 53, 57, 32],\n", + " [ 51, 55, 30],\n", + " [ 45, 52, 28],\n", + " ...,\n", + " [143, 144, 135],\n", + " [143, 144, 135],\n", + " [143, 144, 135]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5170557 , -0.8910335 , 2.0314274 , -2.7631152 , -1.5247868 ,\n", + " 2.6843548 , 0.545448 , -0.08352495, -0.18336658, 0.67522 ,\n", + " 0.7368057 , 0.03068829, 0.01592391, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00419048, -0.0072381 , 0.0127619 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01982857, -0.00034286, 0.00194286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 15, 15],\n", + " [ 14, 14, 14],\n", + " [ 13, 13, 13],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 19, 18, 16],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 12, 12, 12],\n", + " [ 12, 12, 12],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 10, 12, 12],\n", + " [ 10, 12, 12],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 157, 170],\n", + " [153, 157, 170],\n", + " [151, 155, 171],\n", + " ...,\n", + " [147, 149, 169],\n", + " [148, 149, 172],\n", + " [148, 149, 172]],\n", + "\n", + " [[154, 158, 171],\n", + " [154, 158, 171],\n", + " [154, 158, 173],\n", + " ...,\n", + " [148, 150, 170],\n", + " [148, 149, 172],\n", + " [149, 151, 173]],\n", + "\n", + " [[153, 157, 170],\n", + " [154, 158, 171],\n", + " [154, 158, 173],\n", + " ...,\n", + " [148, 150, 170],\n", + " [147, 148, 171],\n", + " [148, 149, 172]]], dtype=uint8), 'image': array([[[103, 96, 79],\n", + " [105, 98, 81],\n", + " [ 48, 72, 72],\n", + " ...,\n", + " [153, 154, 148],\n", + " [152, 152, 148],\n", + " [152, 152, 148]],\n", + "\n", + " [[112, 95, 77],\n", + " [112, 95, 77],\n", + " [ 58, 83, 76],\n", + " ...,\n", + " [152, 153, 149],\n", + " [151, 151, 151],\n", + " [149, 149, 149]],\n", + "\n", + " [[112, 82, 64],\n", + " [121, 91, 73],\n", + " [ 75, 101, 86],\n", + " ...,\n", + " [148, 151, 152],\n", + " [147, 147, 151],\n", + " [145, 145, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 69, 46],\n", + " [ 68, 71, 48],\n", + " [ 65, 68, 43],\n", + " ...,\n", + " [139, 137, 129],\n", + " [142, 140, 130],\n", + " [139, 137, 127]],\n", + "\n", + " [[ 58, 61, 38],\n", + " [ 54, 57, 34],\n", + " [ 49, 52, 27],\n", + " ...,\n", + " [141, 139, 131],\n", + " [142, 140, 130],\n", + " [142, 140, 130]],\n", + "\n", + " [[ 51, 51, 31],\n", + " [ 51, 51, 31],\n", + " [ 45, 49, 21],\n", + " ...,\n", + " [140, 138, 130],\n", + " [142, 140, 130],\n", + " [143, 141, 131]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4967704 , -0.89257985, 2.048069 , -2.7829902 , -1.5259126 ,\n", + " 2.7121851 , 0.5425027 , -0.0710614 , -0.1848708 , 0.67793345,\n", + " 0.734238 , 0.03148263, 0.01759424, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00457143, -0.00704762, 0.01142857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, -5.7142857e-05, 2.0571428e-03], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 17, 16],\n", + " [ 11, 16, 15],\n", + " [ 10, 12, 12],\n", + " ...,\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15],\n", + " [ 15, 16, 14]],\n", + "\n", + " [[ 11, 13, 13],\n", + " [ 11, 13, 13],\n", + " [ 12, 12, 12],\n", + " ...,\n", + " [ 15, 16, 14],\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15]],\n", + "\n", + " [[ 11, 13, 13],\n", + " [ 11, 13, 13],\n", + " [ 12, 12, 12],\n", + " ...,\n", + " [ 15, 16, 14],\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [149, 151, 172],\n", + " [148, 153, 173],\n", + " [148, 153, 173]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [150, 153, 173],\n", + " [147, 152, 172],\n", + " [147, 152, 172]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [151, 154, 174],\n", + " [147, 152, 172],\n", + " [147, 152, 172]]], dtype=uint8), 'image': array([[[103, 99, 61],\n", + " [104, 100, 62],\n", + " [ 49, 76, 58],\n", + " ...,\n", + " [153, 154, 150],\n", + " [152, 152, 148],\n", + " [152, 152, 148]],\n", + "\n", + " [[114, 101, 63],\n", + " [117, 104, 66],\n", + " [ 52, 80, 55],\n", + " ...,\n", + " [150, 153, 152],\n", + " [151, 151, 151],\n", + " [150, 150, 150]],\n", + "\n", + " [[113, 87, 56],\n", + " [123, 97, 66],\n", + " [ 75, 104, 74],\n", + " ...,\n", + " [146, 152, 154],\n", + " [147, 147, 149],\n", + " [145, 145, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 45],\n", + " [ 70, 73, 48],\n", + " [ 65, 69, 46],\n", + " ...,\n", + " [138, 136, 124],\n", + " [134, 136, 120],\n", + " [133, 135, 119]],\n", + "\n", + " [[ 58, 61, 36],\n", + " [ 56, 59, 34],\n", + " [ 54, 57, 34],\n", + " ...,\n", + " [138, 137, 122],\n", + " [138, 137, 122],\n", + " [138, 137, 122]],\n", + "\n", + " [[ 54, 57, 32],\n", + " [ 51, 54, 29],\n", + " [ 47, 51, 30],\n", + " ...,\n", + " [139, 138, 123],\n", + " [138, 137, 122],\n", + " [138, 137, 122]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4744623 , -0.8928798 , 2.0601006 , -2.8009937 , -1.5276612 ,\n", + " 2.7427585 , 0.54119325, -0.05844606, -0.18594792, 0.6808814 ,\n", + " 0.7314025 , 0.03243729, 0.01996658, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00514286, -0.00666667, 0.01314286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01982857, 0. , 0.00194286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 14, 14],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12],\n", + " [ 17, 16, 14]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 8, 11, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[143, 150, 169],\n", + " [143, 150, 169],\n", + " [143, 150, 169],\n", + " ...,\n", + " [145, 149, 171],\n", + " [145, 149, 169],\n", + " [145, 149, 169]],\n", + "\n", + " [[145, 152, 172],\n", + " [145, 152, 172],\n", + " [145, 152, 172],\n", + " ...,\n", + " [145, 149, 171],\n", + " [146, 151, 170],\n", + " [145, 149, 169]],\n", + "\n", + " [[145, 152, 172],\n", + " [146, 153, 173],\n", + " [146, 153, 173],\n", + " ...,\n", + " [145, 149, 171],\n", + " [146, 151, 170],\n", + " [145, 149, 169]]], dtype=uint8), 'image': array([[[111, 99, 81],\n", + " [106, 94, 76],\n", + " [ 49, 73, 73],\n", + " ...,\n", + " [156, 156, 150],\n", + " [152, 153, 149],\n", + " [152, 153, 149]],\n", + "\n", + " [[121, 101, 81],\n", + " [115, 95, 75],\n", + " [ 55, 82, 75],\n", + " ...,\n", + " [152, 153, 149],\n", + " [152, 152, 152],\n", + " [150, 150, 150]],\n", + "\n", + " [[120, 88, 68],\n", + " [125, 93, 73],\n", + " [ 68, 96, 81],\n", + " ...,\n", + " [149, 152, 151],\n", + " [146, 146, 148],\n", + " [145, 145, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 68, 45],\n", + " [ 66, 70, 47],\n", + " [ 62, 65, 40],\n", + " ...,\n", + " [131, 129, 121],\n", + " [134, 130, 121],\n", + " [132, 128, 119]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 57, 61, 36],\n", + " [ 54, 58, 35],\n", + " ...,\n", + " [133, 129, 120],\n", + " [133, 129, 120],\n", + " [134, 130, 121]],\n", + "\n", + " [[ 55, 59, 36],\n", + " [ 50, 54, 31],\n", + " [ 45, 49, 28],\n", + " ...,\n", + " [134, 130, 121],\n", + " [133, 129, 120],\n", + " [135, 131, 122]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.451041 , -0.89231616, 2.0696073 , -2.817558 , -1.5302237 ,\n", + " 2.7739055 , 0.54045534, -0.0457982 , -0.18693021, 0.6836287 ,\n", + " 0.728724 , 0.03323772, 0.02254771, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00457143, -0.00438095, 0.01333333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01977143, 0. , 0.00228571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 14, 14],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 13, 14, 11],\n", + " [ 13, 14, 11],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 14, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " ...,\n", + "\n", + " [[142, 151, 172],\n", + " [142, 151, 172],\n", + " [141, 149, 171],\n", + " ...,\n", + " [143, 145, 167],\n", + " [144, 146, 168],\n", + " [144, 146, 168]],\n", + "\n", + " [[141, 149, 171],\n", + " [141, 149, 171],\n", + " [141, 149, 171],\n", + " ...,\n", + " [144, 146, 168],\n", + " [142, 144, 166],\n", + " [141, 142, 165]],\n", + "\n", + " [[142, 151, 172],\n", + " [142, 151, 172],\n", + " [142, 151, 172],\n", + " ...,\n", + " [144, 146, 168],\n", + " [143, 145, 167],\n", + " [143, 145, 167]]], dtype=uint8), 'image': array([[[109, 102, 71],\n", + " [111, 104, 73],\n", + " [ 44, 69, 62],\n", + " ...,\n", + " [151, 155, 150],\n", + " [151, 154, 153],\n", + " [149, 152, 151]],\n", + "\n", + " [[117, 95, 69],\n", + " [121, 99, 73],\n", + " [ 66, 90, 76],\n", + " ...,\n", + " [150, 153, 154],\n", + " [147, 150, 151],\n", + " [147, 150, 151]],\n", + "\n", + " [[114, 80, 56],\n", + " [124, 90, 66],\n", + " [ 77, 99, 79],\n", + " ...,\n", + " [148, 151, 152],\n", + " [143, 146, 149],\n", + " [143, 146, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 59, 71, 47],\n", + " [ 62, 74, 50],\n", + " [ 61, 70, 50],\n", + " ...,\n", + " [125, 123, 113],\n", + " [123, 124, 113],\n", + " [120, 121, 110]],\n", + "\n", + " [[ 47, 62, 35],\n", + " [ 48, 63, 36],\n", + " [ 52, 64, 40],\n", + " ...,\n", + " [124, 125, 114],\n", + " [126, 124, 114],\n", + " [125, 123, 113]],\n", + "\n", + " [[ 45, 57, 31],\n", + " [ 43, 55, 29],\n", + " [ 42, 54, 28],\n", + " ...,\n", + " [127, 128, 117],\n", + " [128, 126, 116],\n", + " [127, 125, 115]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4267576 , -0.8909252 , 2.0777245 , -2.832958 , -1.5336002 ,\n", + " 2.8063924 , 0.5396934 , -0.03303649, -0.18807867, 0.6865643 ,\n", + " 0.7258565 , 0.03373102, 0.02497944, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00495238, -0.0047619 , 0.01333333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01977143, 0. , 0.00177143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 15, 12],\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 14, 12, 10]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " [[ 8, 12, 9],\n", + " [ 8, 12, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 155, 172],\n", + " [149, 155, 172],\n", + " [148, 154, 171],\n", + " ...,\n", + " [144, 148, 168],\n", + " [145, 149, 169],\n", + " [145, 149, 169]],\n", + "\n", + " [[149, 155, 172],\n", + " [151, 156, 173],\n", + " [151, 156, 173],\n", + " ...,\n", + " [144, 148, 168],\n", + " [144, 148, 168],\n", + " [145, 149, 169]],\n", + "\n", + " [[149, 155, 172],\n", + " [148, 154, 171],\n", + " [148, 154, 171],\n", + " ...,\n", + " [145, 149, 169],\n", + " [145, 149, 169],\n", + " [145, 149, 169]]], dtype=uint8), 'image': array([[[111, 104, 71],\n", + " [103, 96, 63],\n", + " [ 54, 74, 67],\n", + " ...,\n", + " [154, 153, 153],\n", + " [153, 152, 152],\n", + " [153, 152, 152]],\n", + "\n", + " [[120, 105, 68],\n", + " [111, 96, 59],\n", + " [ 62, 83, 70],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[120, 89, 60],\n", + " [122, 91, 62],\n", + " [ 78, 100, 82],\n", + " ...,\n", + " [152, 152, 156],\n", + " [146, 145, 152],\n", + " [144, 143, 150]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 48],\n", + " [ 69, 76, 50],\n", + " [ 64, 71, 47],\n", + " ...,\n", + " [115, 112, 96],\n", + " [115, 114, 97],\n", + " [114, 113, 96]],\n", + "\n", + " [[ 54, 58, 33],\n", + " [ 57, 61, 36],\n", + " [ 60, 67, 41],\n", + " ...,\n", + " [119, 118, 103],\n", + " [118, 117, 102],\n", + " [117, 116, 101]],\n", + "\n", + " [[ 52, 55, 34],\n", + " [ 48, 51, 30],\n", + " [ 45, 49, 24],\n", + " ...,\n", + " [120, 119, 104],\n", + " [118, 117, 102],\n", + " [118, 117, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4022958 , -0.88859874, 2.0833273 , -2.8456123 , -1.5374321 ,\n", + " 2.8394403 , 0.5391687 , -0.02050408, -0.18934578, 0.6896574 ,\n", + " 0.7228375 , 0.03391662, 0.02698694, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00438095, -0.00533333, 0.012 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.02 , 0. , 0.0008], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 14, 13, 8],\n", + " [ 14, 13, 8],\n", + " [ 14, 13, 8]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[148, 154, 171],\n", + " [148, 154, 171],\n", + " [149, 151, 172],\n", + " ...,\n", + " [140, 147, 168],\n", + " [142, 148, 170],\n", + " [142, 148, 170]],\n", + "\n", + " [[147, 153, 170],\n", + " [146, 151, 169],\n", + " [149, 151, 172],\n", + " ...,\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " [142, 148, 170]],\n", + "\n", + " [[146, 151, 169],\n", + " [147, 153, 170],\n", + " [149, 151, 172],\n", + " ...,\n", + " [142, 148, 170],\n", + " [143, 149, 171],\n", + " [142, 148, 170]]], dtype=uint8), 'image': array([[[103, 98, 69],\n", + " [105, 100, 71],\n", + " [ 44, 71, 66],\n", + " ...,\n", + " [154, 153, 157],\n", + " [156, 152, 157],\n", + " [154, 150, 155]],\n", + "\n", + " [[111, 94, 69],\n", + " [123, 106, 81],\n", + " [ 63, 90, 81],\n", + " ...,\n", + " [152, 152, 156],\n", + " [151, 150, 154],\n", + " [152, 151, 155]],\n", + "\n", + " [[111, 79, 57],\n", + " [128, 96, 74],\n", + " [ 75, 100, 83],\n", + " ...,\n", + " [147, 149, 155],\n", + " [146, 146, 150],\n", + " [144, 144, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 66, 41],\n", + " [ 67, 70, 45],\n", + " [ 68, 69, 40],\n", + " ...,\n", + " [105, 104, 89],\n", + " [108, 107, 90],\n", + " [108, 107, 90]],\n", + "\n", + " [[ 56, 60, 30],\n", + " [ 59, 63, 33],\n", + " [ 60, 65, 28],\n", + " ...,\n", + " [110, 109, 94],\n", + " [111, 110, 95],\n", + " [107, 106, 91]],\n", + "\n", + " [[ 51, 53, 21],\n", + " [ 51, 53, 21],\n", + " [ 49, 52, 13],\n", + " ...,\n", + " [112, 111, 96],\n", + " [113, 112, 97],\n", + " [111, 110, 95]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.377918 , -0.88633186, 2.08747 , -2.8563583 , -1.5414795 ,\n", + " 2.8719716 , 0.5387577 , -0.00821607, -0.19039896, 0.6925935 ,\n", + " 0.7199444 , 0.03404387, 0.02890096, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.0047619 , -0.00609524, 0.01047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , -0.00022857, 0.0004 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 14, 13, 8]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 8, 11, 10],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [151, 155, 172],\n", + " ...,\n", + " [134, 142, 165],\n", + " [135, 143, 167],\n", + " [135, 143, 167]],\n", + "\n", + " [[148, 151, 169],\n", + " [150, 153, 171],\n", + " [150, 153, 171],\n", + " ...,\n", + " [136, 144, 168],\n", + " [136, 144, 168],\n", + " [136, 144, 168]],\n", + "\n", + " [[143, 146, 164],\n", + " [147, 150, 168],\n", + " [146, 149, 166],\n", + " ...,\n", + " [136, 144, 168],\n", + " [136, 144, 168],\n", + " [136, 144, 168]]], dtype=uint8), 'image': array([[[108, 98, 66],\n", + " [108, 98, 66],\n", + " [ 51, 73, 57],\n", + " ...,\n", + " [154, 153, 153],\n", + " [153, 153, 149],\n", + " [152, 152, 148]],\n", + "\n", + " [[113, 97, 65],\n", + " [113, 97, 65],\n", + " [ 62, 88, 64],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[110, 82, 54],\n", + " [121, 93, 65],\n", + " [ 74, 100, 74],\n", + " ...,\n", + " [149, 151, 157],\n", + " [145, 147, 153],\n", + " [145, 147, 153]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 69, 42],\n", + " [ 72, 73, 46],\n", + " [ 71, 71, 49],\n", + " ...,\n", + " [ 95, 88, 78],\n", + " [ 93, 89, 78],\n", + " [ 93, 89, 78]],\n", + "\n", + " [[ 62, 63, 36],\n", + " [ 61, 62, 35],\n", + " [ 58, 58, 36],\n", + " ...,\n", + " [ 94, 91, 77],\n", + " [ 94, 92, 80],\n", + " [ 97, 95, 83]],\n", + "\n", + " [[ 57, 55, 34],\n", + " [ 52, 50, 29],\n", + " [ 48, 48, 28],\n", + " ...,\n", + " [ 98, 95, 81],\n", + " [ 98, 94, 83],\n", + " [100, 96, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3518052 , -0.8849166 , 2.0911217 , -2.8662055 , -1.5458258 ,\n", + " 2.9062974 , 0.53834844, 0.00483379, -0.19096382, 0.6955367 ,\n", + " 0.71699756, 0.0343527 , 0.03103506, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00609524, -0.00457143, 0.00780952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " ...,\n", + " [ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [151, 155, 172],\n", + " [151, 155, 172],\n", + " ...,\n", + " [127, 137, 161],\n", + " [128, 139, 162],\n", + " [128, 139, 162]],\n", + "\n", + " [[149, 152, 170],\n", + " [151, 155, 172],\n", + " [151, 155, 172],\n", + " ...,\n", + " [128, 139, 162],\n", + " [127, 137, 161],\n", + " [127, 137, 161]],\n", + "\n", + " [[149, 152, 170],\n", + " [149, 152, 170],\n", + " [150, 153, 171],\n", + " ...,\n", + " [128, 139, 162],\n", + " [127, 137, 161],\n", + " [127, 137, 161]]], dtype=uint8), 'image': array([[[104, 104, 71],\n", + " [100, 100, 67],\n", + " [ 47, 79, 68],\n", + " ...,\n", + " [158, 152, 154],\n", + " [156, 150, 154],\n", + " [156, 150, 154]],\n", + "\n", + " [[111, 97, 62],\n", + " [114, 100, 65],\n", + " [ 61, 93, 72],\n", + " ...,\n", + " [156, 153, 150],\n", + " [154, 150, 151],\n", + " [153, 149, 150]],\n", + "\n", + " [[105, 79, 48],\n", + " [122, 96, 65],\n", + " [ 77, 105, 78],\n", + " ...,\n", + " [153, 153, 149],\n", + " [148, 147, 147],\n", + " [147, 146, 146]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 47],\n", + " [ 70, 73, 50],\n", + " [ 67, 71, 50],\n", + " ...,\n", + " [ 77, 77, 71],\n", + " [ 75, 75, 71],\n", + " [ 74, 74, 70]],\n", + "\n", + " [[ 56, 59, 34],\n", + " [ 59, 62, 37],\n", + " [ 53, 55, 37],\n", + " ...,\n", + " [ 80, 78, 68],\n", + " [ 78, 75, 70],\n", + " [ 79, 76, 71]],\n", + "\n", + " [[ 52, 50, 31],\n", + " [ 54, 52, 33],\n", + " [ 48, 47, 32],\n", + " ...,\n", + " [ 83, 80, 66],\n", + " [ 83, 79, 72],\n", + " [ 86, 82, 75]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3262916 , -0.88400537, 2.0940309 , -2.8750207 , -1.5502566 ,\n", + " 2.9384649 , 0.5379629 , 0.01745152, -0.19117709, 0.69790804,\n", + " 0.71456426, 0.03479953, 0.03333521, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00666667, -0.00095238, 0.00533333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 14, 13],\n", + " [ 5, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 5, 10, 9],\n", + " [ 5, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 7, 10, 9],\n", + " [ 7, 10, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [153, 156, 173],\n", + " ...,\n", + " [134, 142, 165],\n", + " [134, 142, 165],\n", + " [134, 142, 165]],\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [153, 156, 173],\n", + " ...,\n", + " [131, 139, 163],\n", + " [133, 140, 164],\n", + " [133, 140, 164]],\n", + "\n", + " [[153, 156, 173],\n", + " [154, 157, 175],\n", + " [154, 157, 175],\n", + " ...,\n", + " [133, 140, 164],\n", + " [133, 140, 164],\n", + " [131, 139, 163]]], dtype=uint8), 'image': array([[[106, 97, 60],\n", + " [107, 98, 61],\n", + " [ 44, 68, 56],\n", + " ...,\n", + " [152, 150, 157],\n", + " [152, 152, 156],\n", + " [152, 152, 156]],\n", + "\n", + " [[116, 95, 64],\n", + " [123, 102, 71],\n", + " [ 60, 85, 68],\n", + " ...,\n", + " [152, 151, 158],\n", + " [150, 150, 154],\n", + " [151, 151, 155]],\n", + "\n", + " [[113, 82, 55],\n", + " [127, 96, 69],\n", + " [ 78, 103, 82],\n", + " ...,\n", + " [152, 152, 156],\n", + " [145, 148, 151],\n", + " [141, 144, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 66, 45],\n", + " [ 67, 71, 50],\n", + " [ 61, 71, 46],\n", + " ...,\n", + " [ 44, 73, 85],\n", + " [ 43, 71, 86],\n", + " [ 48, 76, 91]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 53, 60, 36],\n", + " [ 50, 61, 31],\n", + " ...,\n", + " [ 50, 79, 89],\n", + " [ 44, 73, 85],\n", + " [ 45, 74, 86]],\n", + "\n", + " [[ 50, 54, 31],\n", + " [ 50, 54, 31],\n", + " [ 42, 52, 25],\n", + " ...,\n", + " [ 65, 97, 106],\n", + " [ 57, 88, 102],\n", + " [ 52, 83, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.299629 , -0.8831771 , 2.0949645 , -2.8811018 , -1.5560879 ,\n", + " 2.9703782 , 0.5377479 , 0.03023179, -0.1911558 , 0.6998129 ,\n", + " 0.71258867, 0.03465674, 0.03575221, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.004 , 0. , 0.00552381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 10, 8],\n", + " [ 10, 9, 7],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 11, 10, 8],\n", + " [ 11, 10, 8],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[160, 160, 174],\n", + " [160, 160, 174],\n", + " [157, 159, 173],\n", + " ...,\n", + " [137, 143, 165],\n", + " [137, 143, 165],\n", + " [137, 143, 165]],\n", + "\n", + " [[159, 159, 173],\n", + " [160, 160, 174],\n", + " [159, 162, 175],\n", + " ...,\n", + " [137, 143, 165],\n", + " [136, 142, 164],\n", + " [136, 142, 164]],\n", + "\n", + " [[158, 158, 171],\n", + " [158, 158, 171],\n", + " [157, 159, 173],\n", + " ...,\n", + " [136, 142, 164],\n", + " [136, 142, 164],\n", + " [135, 141, 163]]], dtype=uint8), 'image': array([[[111, 101, 71],\n", + " [104, 94, 64],\n", + " [ 49, 69, 66],\n", + " ...,\n", + " [155, 154, 156],\n", + " [153, 152, 152],\n", + " [151, 150, 150]],\n", + "\n", + " [[119, 101, 69],\n", + " [119, 101, 69],\n", + " [ 62, 84, 68],\n", + " ...,\n", + " [152, 155, 156],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[118, 87, 58],\n", + " [126, 95, 66],\n", + " [ 78, 101, 78],\n", + " ...,\n", + " [149, 154, 159],\n", + " [144, 147, 150],\n", + " [143, 146, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 71, 49],\n", + " [ 65, 72, 50],\n", + " [ 63, 71, 42],\n", + " ...,\n", + " [ 64, 89, 94],\n", + " [ 63, 91, 97],\n", + " [ 63, 91, 97]],\n", + "\n", + " [[ 53, 60, 36],\n", + " [ 51, 58, 34],\n", + " [ 49, 56, 30],\n", + " ...,\n", + " [ 59, 87, 93],\n", + " [ 63, 87, 97],\n", + " [ 64, 88, 98]],\n", + "\n", + " [[ 53, 57, 34],\n", + " [ 52, 56, 33],\n", + " [ 42, 49, 25],\n", + " ...,\n", + " [ 55, 84, 92],\n", + " [ 57, 84, 93],\n", + " [ 57, 84, 93]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2725835 , -0.88199025, 2.093504 , -2.8825166 , -1.5627569 ,\n", + " 3.0007691 , 0.53741384, 0.0429778 , -0.19125551, 0.70110106,\n", + " 0.7112887 , 0.03353317, 0.03743315, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00647619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 12],\n", + " [ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 7, 7, 7],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 7, 7, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[155, 156, 172],\n", + " [155, 156, 172],\n", + " [155, 155, 171],\n", + " ...,\n", + " [129, 140, 163],\n", + " [131, 141, 164],\n", + " [131, 141, 164]],\n", + "\n", + " [[152, 154, 169],\n", + " [153, 155, 171],\n", + " [157, 156, 172],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [131, 141, 164]],\n", + "\n", + " [[155, 156, 172],\n", + " [153, 155, 171],\n", + " [155, 155, 171],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [131, 141, 164]]], dtype=uint8), 'image': array([[[107, 96, 71],\n", + " [107, 96, 71],\n", + " [ 51, 74, 66],\n", + " ...,\n", + " [151, 154, 155],\n", + " [153, 152, 154],\n", + " [153, 152, 154]],\n", + "\n", + " [[112, 95, 68],\n", + " [118, 101, 74],\n", + " [ 58, 83, 66],\n", + " ...,\n", + " [150, 153, 152],\n", + " [151, 150, 152],\n", + " [151, 150, 152]],\n", + "\n", + " [[118, 90, 62],\n", + " [130, 102, 74],\n", + " [ 69, 94, 73],\n", + " ...,\n", + " [150, 153, 154],\n", + " [146, 146, 150],\n", + " [147, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 70, 48],\n", + " [ 66, 73, 51],\n", + " [ 62, 69, 45],\n", + " ...,\n", + " [ 66, 94, 100],\n", + " [ 64, 92, 98],\n", + " [ 62, 90, 96]],\n", + "\n", + " [[ 55, 62, 38],\n", + " [ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " ...,\n", + " [ 68, 93, 100],\n", + " [ 64, 89, 96],\n", + " [ 64, 89, 96]],\n", + "\n", + " [[ 47, 51, 30],\n", + " [ 51, 55, 34],\n", + " [ 42, 49, 27],\n", + " ...,\n", + " [ 65, 90, 97],\n", + " [ 63, 88, 95],\n", + " [ 63, 88, 95]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2453115 , -0.88051444, 2.0899243 , -2.8813412 , -1.5684208 ,\n", + " 3.031338 , 0.53737307, 0.05604258, -0.19126037, 0.70236003,\n", + " 0.7100228 , 0.03269348, 0.03858953, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00666667], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 14, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 14, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [155, 155, 173],\n", + " ...,\n", + " [134, 142, 165],\n", + " [134, 142, 164],\n", + " [134, 142, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [135, 143, 167],\n", + " [134, 142, 164],\n", + " [134, 142, 164]],\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [156, 156, 175],\n", + " ...,\n", + " [135, 143, 167],\n", + " [134, 142, 164],\n", + " [134, 142, 164]]], dtype=uint8), 'image': array([[[109, 98, 73],\n", + " [108, 97, 72],\n", + " [ 55, 75, 68],\n", + " ...,\n", + " [154, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[114, 94, 72],\n", + " [121, 101, 79],\n", + " [ 62, 84, 68],\n", + " ...,\n", + " [153, 152, 154],\n", + " [151, 151, 153],\n", + " [150, 150, 152]],\n", + "\n", + " [[109, 77, 55],\n", + " [126, 94, 72],\n", + " [ 78, 101, 78],\n", + " ...,\n", + " [151, 151, 155],\n", + " [147, 147, 149],\n", + " [146, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 71, 46],\n", + " [ 69, 73, 48],\n", + " [ 67, 71, 48],\n", + " ...,\n", + " [ 63, 96, 98],\n", + " [ 62, 92, 95],\n", + " [ 61, 91, 94]],\n", + "\n", + " [[ 61, 65, 37],\n", + " [ 62, 66, 38],\n", + " [ 57, 61, 40],\n", + " ...,\n", + " [ 61, 94, 96],\n", + " [ 62, 92, 95],\n", + " [ 60, 90, 93]],\n", + "\n", + " [[ 53, 56, 31],\n", + " [ 53, 56, 31],\n", + " [ 46, 50, 29],\n", + " ...,\n", + " [ 61, 94, 96],\n", + " [ 62, 92, 95],\n", + " [ 61, 91, 94]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.219752 , -0.8787039 , 2.0854707 , -2.8789427 , -1.5713781 ,\n", + " 3.0608265 , 0.53732604, 0.06888904, -0.19135544, 0.70380074,\n", + " 0.7085868 , 0.03269861, 0.03873029, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00552381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 7, 14, 13],\n", + " [ 3, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 3, 10, 9],\n", + " [ 2, 9, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 5, 10, 9],\n", + " [ 4, 9, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 151, 172],\n", + " [149, 151, 172],\n", + " [148, 150, 170],\n", + " ...,\n", + " [126, 139, 162],\n", + " [127, 140, 163],\n", + " [127, 140, 163]],\n", + "\n", + " [[151, 154, 174],\n", + " [153, 155, 175],\n", + " [151, 154, 174],\n", + " ...,\n", + " [127, 140, 163],\n", + " [126, 139, 162],\n", + " [127, 140, 163]],\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [153, 155, 175],\n", + " ...,\n", + " [129, 141, 164],\n", + " [127, 140, 163],\n", + " [127, 140, 163]]], dtype=uint8), 'image': array([[[104, 99, 66],\n", + " [102, 97, 64],\n", + " [ 47, 71, 57],\n", + " ...,\n", + " [155, 154, 154],\n", + " [153, 152, 152],\n", + " [152, 151, 151]],\n", + "\n", + " [[114, 99, 71],\n", + " [118, 103, 75],\n", + " [ 57, 84, 66],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [147, 147, 149]],\n", + "\n", + " [[116, 89, 65],\n", + " [131, 104, 80],\n", + " [ 68, 95, 73],\n", + " ...,\n", + " [153, 152, 159],\n", + " [147, 147, 151],\n", + " [146, 146, 150]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 48],\n", + " [ 72, 75, 52],\n", + " [ 75, 76, 49],\n", + " ...,\n", + " [ 66, 91, 98],\n", + " [ 65, 90, 97],\n", + " [ 64, 89, 96]],\n", + "\n", + " [[ 57, 61, 36],\n", + " [ 55, 59, 34],\n", + " [ 51, 54, 29],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 90, 96],\n", + " [ 62, 90, 96]],\n", + "\n", + " [[ 55, 58, 33],\n", + " [ 51, 54, 29],\n", + " [ 48, 51, 26],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 62, 92, 95],\n", + " [ 63, 93, 96]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.194349 , -0.8765333 , 2.0798793 , -2.8753808 , -1.5737132 ,\n", + " 3.09029 , 0.5372993 , 0.08182718, -0.19148241, 0.7052792 ,\n", + " 0.7071188 , 0.03283391, 0.03855143, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 14, 13],\n", + " [ 5, 10, 9],\n", + " [ 7, 10, 9],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 7, 10, 9],\n", + " [ 6, 9, 8],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 7, 11, 8],\n", + " [ 6, 9, 7],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [155, 155, 173],\n", + " ...,\n", + " [129, 140, 163],\n", + " [131, 141, 164],\n", + " [131, 141, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [129, 140, 163],\n", + " [129, 140, 163],\n", + " [131, 141, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [129, 140, 163],\n", + " [129, 140, 163],\n", + " [129, 140, 163]]], dtype=uint8), 'image': array([[[106, 98, 72],\n", + " [103, 95, 69],\n", + " [ 52, 74, 73],\n", + " ...,\n", + " [155, 154, 154],\n", + " [153, 152, 154],\n", + " [154, 153, 155]],\n", + "\n", + " [[115, 97, 65],\n", + " [113, 95, 63],\n", + " [ 64, 88, 74],\n", + " ...,\n", + " [153, 153, 155],\n", + " [154, 153, 157],\n", + " [151, 150, 154]],\n", + "\n", + " [[111, 84, 51],\n", + " [120, 93, 60],\n", + " [ 83, 106, 83],\n", + " ...,\n", + " [152, 152, 156],\n", + " [150, 149, 156],\n", + " [146, 145, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 69, 52],\n", + " [ 69, 75, 58],\n", + " [ 65, 71, 52],\n", + " ...,\n", + " [ 71, 96, 101],\n", + " [ 78, 100, 103],\n", + " [ 75, 97, 100]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 57, 63, 44],\n", + " [ 56, 63, 41],\n", + " ...,\n", + " [ 67, 95, 101],\n", + " [ 75, 100, 105],\n", + " [ 75, 100, 105]],\n", + "\n", + " [[ 47, 50, 34],\n", + " [ 50, 53, 37],\n", + " [ 45, 52, 30],\n", + " ...,\n", + " [ 64, 91, 102],\n", + " [ 69, 96, 105],\n", + " [ 71, 98, 107]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1694727 , -0.87412244, 2.0735097 , -2.8713624 , -1.5762395 ,\n", + " 3.1185782 , 0.53730077, 0.09447004, -0.19158448, 0.7065261 ,\n", + " 0.7058712 , 0.03291051, 0.03851758, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.004], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 10, 10],\n", + " [ 8, 8, 8],\n", + " [ 7, 8, 5],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [158, 157, 173],\n", + " ...,\n", + " [122, 136, 161],\n", + " [123, 137, 162],\n", + " [123, 137, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [159, 158, 174],\n", + " ...,\n", + " [123, 137, 162],\n", + " [123, 137, 162],\n", + " [123, 137, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [158, 157, 173],\n", + " ...,\n", + " [123, 137, 162],\n", + " [124, 138, 163],\n", + " [124, 138, 163]]], dtype=uint8), 'image': array([[[106, 103, 71],\n", + " [104, 101, 69],\n", + " [ 49, 72, 66],\n", + " ...,\n", + " [155, 156, 150],\n", + " [155, 156, 147],\n", + " [153, 154, 145]],\n", + "\n", + " [[117, 102, 74],\n", + " [119, 104, 76],\n", + " [ 59, 85, 70],\n", + " ...,\n", + " [151, 155, 148],\n", + " [153, 155, 146],\n", + " [152, 154, 145]],\n", + "\n", + " [[110, 82, 54],\n", + " [128, 100, 72],\n", + " [ 74, 99, 78],\n", + " ...,\n", + " [149, 153, 148],\n", + " [145, 149, 142],\n", + " [142, 146, 139]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 67, 49],\n", + " [ 67, 70, 52],\n", + " [ 66, 69, 51],\n", + " ...,\n", + " [ 61, 91, 96],\n", + " [ 60, 93, 97],\n", + " [ 58, 91, 95]],\n", + "\n", + " [[ 57, 61, 40],\n", + " [ 58, 62, 41],\n", + " [ 49, 55, 36],\n", + " ...,\n", + " [ 59, 92, 96],\n", + " [ 60, 93, 97],\n", + " [ 59, 92, 96]],\n", + "\n", + " [[ 52, 56, 35],\n", + " [ 50, 54, 33],\n", + " [ 44, 50, 31],\n", + " ...,\n", + " [ 59, 88, 96],\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1447768 , -0.8713878 , 2.066051 , -2.8665173 , -1.5785502 ,\n", + " 3.1462178 , 0.53738934, 0.10712075, -0.19166769, 0.70760185,\n", + " 0.7047916 , 0.03304661, 0.03842265, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 15, 12],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [156, 156, 175],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [157, 158, 176],\n", + " ...,\n", + " [113, 129, 156],\n", + " [112, 128, 155],\n", + " [113, 129, 156]],\n", + "\n", + " [[157, 158, 176],\n", + " [156, 156, 175],\n", + " [157, 158, 176],\n", + " ...,\n", + " [115, 131, 158],\n", + " [113, 129, 156],\n", + " [114, 130, 157]]], dtype=uint8), 'image': array([[[112, 100, 73],\n", + " [112, 100, 73],\n", + " [ 53, 71, 69],\n", + " ...,\n", + " [150, 154, 149],\n", + " [149, 153, 146],\n", + " [150, 154, 147]],\n", + "\n", + " [[120, 101, 72],\n", + " [117, 98, 69],\n", + " [ 67, 88, 75],\n", + " ...,\n", + " [149, 153, 148],\n", + " [148, 152, 147],\n", + " [148, 152, 147]],\n", + "\n", + " [[110, 82, 54],\n", + " [121, 93, 65],\n", + " [ 77, 99, 81],\n", + " ...,\n", + " [149, 152, 151],\n", + " [144, 147, 146],\n", + " [143, 146, 145]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 72, 48],\n", + " [ 64, 76, 52],\n", + " [ 63, 73, 50],\n", + " ...,\n", + " [ 60, 90, 95],\n", + " [ 58, 83, 90],\n", + " [ 58, 83, 90]],\n", + "\n", + " [[ 53, 63, 38],\n", + " [ 51, 61, 36],\n", + " [ 52, 59, 37],\n", + " ...,\n", + " [ 60, 90, 95],\n", + " [ 58, 83, 90],\n", + " [ 56, 81, 88]],\n", + "\n", + " [[ 48, 52, 29],\n", + " [ 50, 54, 31],\n", + " [ 46, 53, 31],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 87, 92],\n", + " [ 55, 80, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1198535 , -0.8682216 , 2.057539 , -2.860892 , -1.5811008 ,\n", + " 3.1736052 , 0.53747594, 0.11990292, -0.19179024, 0.70851207,\n", + " 0.7038783 , 0.03306015, 0.03838015, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 154, 174],\n", + " [149, 154, 174],\n", + " [150, 153, 171],\n", + " ...,\n", + " [112, 132, 161],\n", + " [112, 132, 161],\n", + " [112, 132, 161]],\n", + "\n", + " [[151, 155, 175],\n", + " [151, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [110, 130, 159],\n", + " [109, 129, 157],\n", + " [111, 131, 160]],\n", + "\n", + " [[152, 156, 176],\n", + " [152, 156, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [112, 132, 161],\n", + " [113, 133, 162],\n", + " [115, 135, 163]]], dtype=uint8), 'image': array([[[104, 100, 62],\n", + " [ 97, 93, 55],\n", + " [ 39, 68, 67],\n", + " ...,\n", + " [152, 155, 154],\n", + " [151, 154, 153],\n", + " [150, 153, 152]],\n", + "\n", + " [[116, 101, 62],\n", + " [111, 96, 57],\n", + " [ 54, 84, 74],\n", + " ...,\n", + " [152, 155, 156],\n", + " [149, 152, 153],\n", + " [147, 150, 151]],\n", + "\n", + " [[118, 87, 60],\n", + " [122, 91, 64],\n", + " [ 73, 102, 84],\n", + " ...,\n", + " [150, 152, 160],\n", + " [144, 146, 154],\n", + " [144, 146, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 69, 46],\n", + " [ 67, 71, 48],\n", + " [ 64, 68, 47],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " [ 56, 62, 43],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 98]],\n", + "\n", + " [[ 43, 52, 32],\n", + " [ 43, 52, 32],\n", + " [ 42, 50, 35],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 61, 93, 100],\n", + " [ 58, 90, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0955417 , -0.8648394 , 2.0481286 , -2.8546073 , -1.5833639 ,\n", + " 3.2002604 , 0.53765064, 0.1325151 , -0.191861 , 0.7093723 ,\n", + " 0.70301116, 0.03317269, 0.03828627, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00514286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00011429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 8, 16, 12],\n", + " [ 4, 12, 9],\n", + " [ 8, 12, 9],\n", + " ...,\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " [[ 5, 11, 8],\n", + " [ 5, 11, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 9, 11, 6],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " [[ 7, 11, 8],\n", + " [ 7, 11, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 9, 11, 6],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[155, 158, 176],\n", + " [155, 158, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [112, 127, 156],\n", + " [112, 127, 156],\n", + " [113, 128, 157]],\n", + "\n", + " [[154, 157, 175],\n", + " [155, 158, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [114, 129, 159],\n", + " [114, 129, 159],\n", + " [113, 128, 157]],\n", + "\n", + " [[155, 158, 176],\n", + " [156, 159, 177],\n", + " [155, 158, 176],\n", + " ...,\n", + " [117, 133, 162],\n", + " [119, 134, 163],\n", + " [117, 133, 162]]], dtype=uint8), 'image': array([[[108, 97, 74],\n", + " [108, 97, 74],\n", + " [ 43, 66, 60],\n", + " ...,\n", + " [151, 151, 151],\n", + " [156, 153, 150],\n", + " [157, 154, 151]],\n", + "\n", + " [[111, 91, 69],\n", + " [121, 101, 79],\n", + " [ 65, 89, 75],\n", + " ...,\n", + " [152, 151, 155],\n", + " [156, 152, 153],\n", + " [155, 151, 152]],\n", + "\n", + " [[106, 77, 56],\n", + " [125, 96, 75],\n", + " [ 81, 106, 87],\n", + " ...,\n", + " [152, 151, 160],\n", + " [151, 149, 156],\n", + " [147, 145, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 63, 50],\n", + " [ 67, 69, 56],\n", + " [ 63, 69, 50],\n", + " ...,\n", + " [ 66, 96, 99],\n", + " [ 66, 96, 99],\n", + " [ 65, 95, 98]],\n", + "\n", + " [[ 57, 60, 44],\n", + " [ 59, 62, 46],\n", + " [ 59, 62, 44],\n", + " ...,\n", + " [ 65, 95, 98],\n", + " [ 61, 91, 94],\n", + " [ 62, 92, 95]],\n", + "\n", + " [[ 51, 54, 38],\n", + " [ 47, 50, 34],\n", + " [ 46, 49, 31],\n", + " ...,\n", + " [ 65, 95, 98],\n", + " [ 60, 90, 95],\n", + " [ 62, 92, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0713017 , -0.8611215 , 2.037857 , -2.8476746 , -1.585627 ,\n", + " 3.2267466 , 0.5377978 , 0.14518696, -0.19194975, 0.7101977 ,\n", + " 0.7021796 , 0.03325085, 0.03817584, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00514286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00022857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 12],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[151, 154, 174],\n", + " [153, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [120, 136, 163]],\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [154, 157, 175],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [119, 135, 162]]], dtype=uint8), 'image': array([[[104, 103, 75],\n", + " [ 94, 93, 65],\n", + " [ 41, 71, 63],\n", + " ...,\n", + " [148, 153, 158],\n", + " [147, 152, 157],\n", + " [148, 153, 158]],\n", + "\n", + " [[116, 106, 78],\n", + " [106, 96, 68],\n", + " [ 58, 91, 74],\n", + " ...,\n", + " [146, 152, 154],\n", + " [147, 152, 157],\n", + " [146, 151, 156]],\n", + "\n", + " [[114, 90, 67],\n", + " [114, 90, 67],\n", + " [ 69, 101, 80],\n", + " ...,\n", + " [147, 152, 157],\n", + " [143, 148, 153],\n", + " [141, 146, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 71, 46],\n", + " [ 67, 77, 52],\n", + " [ 66, 73, 51],\n", + " ...,\n", + " [ 67, 96, 104],\n", + " [ 63, 95, 102],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 53, 63, 38],\n", + " [ 55, 65, 40],\n", + " [ 53, 60, 36],\n", + " ...,\n", + " [ 66, 96, 101],\n", + " [ 64, 96, 103],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 46, 56, 31],\n", + " [ 47, 57, 32],\n", + " [ 43, 50, 26],\n", + " ...,\n", + " [ 66, 94, 100],\n", + " [ 63, 92, 102],\n", + " [ 65, 94, 104]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.046966 , -0.8571275 , 2.026507 , -2.8399389 , -1.5878781 ,\n", + " 3.254046 , 0.5379952 , 0.15803353, -0.19198048, 0.71127206,\n", + " 0.7010918 , 0.03336547, 0.03806647, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, 0.0000000e+00, -5.7142857e-05], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 16, 14],\n", + " [ 8, 9, 7],\n", + " [ 7, 8, 5],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 9, 8, 4],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [153, 155, 175],\n", + " ...,\n", + " [121, 135, 160],\n", + " [121, 134, 162],\n", + " [121, 134, 162]],\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [154, 156, 176],\n", + " ...,\n", + " [116, 130, 155],\n", + " [115, 128, 156],\n", + " [117, 131, 158]],\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [154, 156, 176],\n", + " ...,\n", + " [114, 128, 153],\n", + " [114, 127, 155],\n", + " [116, 130, 157]]], dtype=uint8), 'image': array([[[104, 102, 83],\n", + " [ 95, 93, 74],\n", + " [ 44, 68, 78],\n", + " ...,\n", + " [151, 154, 155],\n", + " [150, 153, 154],\n", + " [150, 153, 154]],\n", + "\n", + " [[110, 96, 77],\n", + " [114, 100, 81],\n", + " [ 54, 80, 80],\n", + " ...,\n", + " [147, 153, 155],\n", + " [147, 153, 155],\n", + " [148, 154, 156]],\n", + "\n", + " [[109, 81, 64],\n", + " [130, 102, 85],\n", + " [ 73, 100, 91],\n", + " ...,\n", + " [146, 151, 158],\n", + " [145, 151, 153],\n", + " [140, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 68, 45],\n", + " [ 66, 70, 47],\n", + " [ 64, 68, 47],\n", + " ...,\n", + " [ 61, 94, 98],\n", + " [ 63, 96, 100],\n", + " [ 63, 96, 100]],\n", + "\n", + " [[ 57, 61, 36],\n", + " [ 58, 62, 37],\n", + " [ 55, 62, 36],\n", + " ...,\n", + " [ 64, 97, 101],\n", + " [ 64, 97, 101],\n", + " [ 61, 94, 98]],\n", + "\n", + " [[ 52, 57, 29],\n", + " [ 50, 55, 27],\n", + " [ 46, 51, 23],\n", + " ...,\n", + " [ 65, 98, 102],\n", + " [ 64, 97, 101],\n", + " [ 61, 94, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.02317 , -0.8530019 , 2.0143075 , -2.8314588 , -1.5901531 ,\n", + " 3.280961 , 0.5382555 , 0.17071289, -0.1919237 , 0.7123984 ,\n", + " 0.69994664, 0.03346113, 0.0379937 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.004], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [120, 136, 163],\n", + " [121, 137, 164],\n", + " [121, 137, 164]],\n", + "\n", + " [[150, 153, 173],\n", + " [151, 154, 174],\n", + " [151, 155, 172],\n", + " ...,\n", + " [121, 137, 164],\n", + " [122, 138, 165],\n", + " [123, 139, 166]],\n", + "\n", + " [[150, 153, 173],\n", + " [151, 154, 174],\n", + " [153, 156, 173],\n", + " ...,\n", + " [123, 139, 166],\n", + " [124, 140, 167],\n", + " [124, 140, 167]]], dtype=uint8), 'image': array([[[101, 98, 84],\n", + " [106, 103, 89],\n", + " [ 37, 68, 68],\n", + " ...,\n", + " [148, 156, 153],\n", + " [147, 155, 152],\n", + " [147, 155, 152]],\n", + "\n", + " [[111, 98, 85],\n", + " [118, 105, 92],\n", + " [ 46, 80, 75],\n", + " ...,\n", + " [148, 156, 155],\n", + " [147, 153, 151],\n", + " [148, 154, 152]],\n", + "\n", + " [[105, 80, 71],\n", + " [122, 97, 88],\n", + " [ 61, 93, 84],\n", + " ...,\n", + " [148, 153, 158],\n", + " [143, 149, 151],\n", + " [141, 147, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 57, 68, 47],\n", + " [ 63, 74, 53],\n", + " [ 59, 75, 52],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 59, 87, 93],\n", + " [ 56, 84, 90]],\n", + "\n", + " [[ 50, 62, 38],\n", + " [ 49, 61, 37],\n", + " [ 48, 60, 36],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 59, 87, 93],\n", + " [ 57, 85, 91]],\n", + "\n", + " [[ 47, 57, 34],\n", + " [ 47, 57, 34],\n", + " [ 41, 53, 29],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 56, 86, 89],\n", + " [ 55, 85, 88]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9994097 , -0.84833604, 2.0012221 , -2.822284 , -1.5922841 ,\n", + " 3.3077435 , 0.5384106 , 0.18353929, -0.19198695, 0.7134904 ,\n", + " 0.6988444 , 0.03347312, 0.0377824 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[152, 156, 178],\n", + " [152, 156, 178],\n", + " [154, 156, 176],\n", + " ...,\n", + " [116, 132, 159],\n", + " [117, 133, 160],\n", + " [117, 133, 160]],\n", + "\n", + " [[152, 156, 178],\n", + " [153, 157, 179],\n", + " [155, 157, 177],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[152, 156, 178],\n", + " [153, 157, 179],\n", + " [155, 157, 177],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]]], dtype=uint8), 'image': array([[[102, 95, 74],\n", + " [107, 100, 79],\n", + " [ 37, 68, 66],\n", + " ...,\n", + " [147, 152, 159],\n", + " [152, 151, 160],\n", + " [151, 150, 159]],\n", + "\n", + " [[109, 93, 72],\n", + " [121, 105, 84],\n", + " [ 52, 84, 75],\n", + " ...,\n", + " [147, 152, 159],\n", + " [150, 152, 160],\n", + " [145, 147, 155]],\n", + "\n", + " [[110, 82, 63],\n", + " [124, 96, 77],\n", + " [ 66, 99, 84],\n", + " ...,\n", + " [147, 152, 161],\n", + " [147, 149, 157],\n", + " [144, 146, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 57, 65, 50],\n", + " [ 62, 70, 55],\n", + " [ 64, 69, 57],\n", + " ...,\n", + " [ 86, 118, 125],\n", + " [ 80, 114, 120],\n", + " [ 67, 101, 107]],\n", + "\n", + " [[ 56, 62, 45],\n", + " [ 57, 63, 46],\n", + " [ 55, 57, 44],\n", + " ...,\n", + " [ 67, 99, 108],\n", + " [ 77, 111, 119],\n", + " [ 73, 107, 115]],\n", + "\n", + " [[ 52, 55, 37],\n", + " [ 51, 54, 36],\n", + " [ 45, 47, 34],\n", + " ...,\n", + " [ 60, 92, 101],\n", + " [ 59, 93, 101],\n", + " [ 70, 104, 112]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9757934 , -0.84310704, 1.9867959 , -2.8126876 , -1.5945233 ,\n", + " 3.334057 , 0.53880805, 0.19645281, -0.19202068, 0.71446204,\n", + " 0.69784665, 0.03360009, 0.03775069, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 9, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[151, 155, 175],\n", + " [151, 155, 175],\n", + " [149, 154, 174],\n", + " ...,\n", + " [125, 138, 161],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[148, 153, 173],\n", + " [148, 153, 173],\n", + " [148, 153, 173],\n", + " ...,\n", + " [126, 139, 162],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[148, 153, 173],\n", + " [147, 152, 172],\n", + " [149, 154, 174],\n", + " ...,\n", + " [127, 140, 163],\n", + " [127, 140, 163],\n", + " [127, 140, 163]]], dtype=uint8), 'image': array([[[107, 95, 75],\n", + " [108, 96, 76],\n", + " [ 49, 68, 72],\n", + " ...,\n", + " [155, 157, 163],\n", + " [153, 152, 159],\n", + " [152, 151, 158]],\n", + "\n", + " [[115, 95, 73],\n", + " [122, 102, 80],\n", + " [ 62, 85, 79],\n", + " ...,\n", + " [152, 154, 162],\n", + " [151, 153, 159],\n", + " [149, 151, 157]],\n", + "\n", + " [[118, 85, 68],\n", + " [128, 95, 78],\n", + " [ 78, 102, 90],\n", + " ...,\n", + " [148, 153, 162],\n", + " [148, 150, 158],\n", + " [145, 147, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 65, 51],\n", + " [ 67, 72, 58],\n", + " [ 64, 69, 55],\n", + " ...,\n", + " [ 83, 111, 126],\n", + " [ 89, 121, 132],\n", + " [ 82, 114, 125]],\n", + "\n", + " [[ 55, 61, 42],\n", + " [ 53, 59, 40],\n", + " [ 54, 57, 41],\n", + " ...,\n", + " [ 62, 90, 105],\n", + " [ 73, 102, 112],\n", + " [ 82, 111, 121]],\n", + "\n", + " [[ 48, 50, 32],\n", + " [ 52, 54, 36],\n", + " [ 47, 51, 30],\n", + " ...,\n", + " [ 58, 90, 101],\n", + " [ 60, 89, 99],\n", + " [ 58, 87, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9526203 , -0.8375662 , 1.9717474 , -2.8025048 , -1.5965704 ,\n", + " 3.3599634 , 0.539095 , 0.20931895, -0.1921031 , 0.7154446 ,\n", + " 0.6968444 , 0.03366781, 0.03759455, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, 0.0000000e+00, -5.7142857e-05], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 14, 10],\n", + " [ 10, 12, 7],\n", + " [ 8, 10, 5],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 7],\n", + " [ 11, 10, 8]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[145, 151, 173],\n", + " [145, 151, 173],\n", + " [144, 150, 172],\n", + " ...,\n", + " [125, 140, 163],\n", + " [125, 140, 163],\n", + " [125, 140, 163]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [125, 140, 163],\n", + " [124, 139, 162],\n", + " [123, 138, 161]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [121, 136, 158],\n", + " [121, 136, 158],\n", + " [121, 136, 158]]], dtype=uint8), 'image': array([[[103, 96, 77],\n", + " [102, 95, 76],\n", + " [ 50, 74, 74],\n", + " ...,\n", + " [150, 155, 160],\n", + " [150, 155, 160],\n", + " [150, 155, 160]],\n", + "\n", + " [[115, 97, 77],\n", + " [112, 94, 74],\n", + " [ 57, 85, 72],\n", + " ...,\n", + " [152, 154, 162],\n", + " [147, 152, 157],\n", + " [147, 152, 157]],\n", + "\n", + " [[120, 91, 68],\n", + " [123, 94, 71],\n", + " [ 73, 102, 82],\n", + " ...,\n", + " [148, 152, 164],\n", + " [143, 148, 155],\n", + " [142, 147, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 68, 51],\n", + " [ 66, 72, 55],\n", + " [ 62, 67, 53],\n", + " ...,\n", + " [ 87, 121, 129],\n", + " [ 74, 108, 116],\n", + " [ 66, 100, 108]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 57, 64, 40],\n", + " [ 53, 58, 44],\n", + " ...,\n", + " [ 85, 119, 127],\n", + " [ 87, 121, 131],\n", + " [ 78, 112, 122]],\n", + "\n", + " [[ 55, 62, 40],\n", + " [ 48, 55, 33],\n", + " [ 45, 51, 34],\n", + " ...,\n", + " [ 64, 98, 104],\n", + " [ 75, 107, 116],\n", + " [ 82, 114, 123]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9290407 , -0.83140117, 1.9549513 , -2.7911823 , -1.5989178 ,\n", + " 3.386706 , 0.53953487, 0.22216113, -0.19215716, 0.7165386 ,\n", + " 0.6957205 , 0.0336923 , 0.03755271, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00057143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 14, 11],\n", + " [ 8, 12, 9],\n", + " [ 8, 10, 5],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " [[ 8, 12, 9],\n", + " [ 8, 12, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[152, 156, 176],\n", + " [152, 156, 176],\n", + " [151, 155, 175],\n", + " ...,\n", + " [126, 139, 162],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[149, 154, 174],\n", + " [149, 154, 174],\n", + " [151, 155, 175],\n", + " ...,\n", + " [123, 135, 158],\n", + " [123, 135, 158],\n", + " [122, 134, 157]],\n", + "\n", + " [[148, 153, 173],\n", + " [148, 153, 173],\n", + " [149, 154, 174],\n", + " ...,\n", + " [120, 133, 156],\n", + " [119, 132, 155],\n", + " [120, 133, 156]]], dtype=uint8), 'image': array([[[107, 95, 77],\n", + " [108, 96, 78],\n", + " [ 49, 73, 73],\n", + " ...,\n", + " [153, 156, 157],\n", + " [152, 155, 156],\n", + " [150, 153, 154]],\n", + "\n", + " [[115, 94, 77],\n", + " [120, 99, 82],\n", + " [ 57, 84, 77],\n", + " ...,\n", + " [150, 158, 159],\n", + " [150, 156, 158],\n", + " [147, 153, 155]],\n", + "\n", + " [[112, 82, 64],\n", + " [125, 95, 77],\n", + " [ 75, 104, 86],\n", + " ...,\n", + " [147, 154, 158],\n", + " [142, 149, 153],\n", + " [140, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 72, 60],\n", + " [ 67, 77, 65],\n", + " [ 62, 71, 62],\n", + " ...,\n", + " [ 64, 98, 108],\n", + " [ 66, 95, 105],\n", + " [ 66, 95, 105]],\n", + "\n", + " [[ 56, 64, 49],\n", + " [ 55, 63, 48],\n", + " [ 54, 61, 53],\n", + " ...,\n", + " [ 72, 106, 114],\n", + " [ 67, 99, 106],\n", + " [ 66, 98, 105]],\n", + "\n", + " [[ 50, 56, 39],\n", + " [ 48, 54, 37],\n", + " [ 39, 47, 34],\n", + " ...,\n", + " [ 89, 121, 128],\n", + " [ 87, 119, 126],\n", + " [ 74, 106, 113]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9072087 , -0.82534486, 1.9385734 , -2.780183 , -1.6008447 ,\n", + " 3.4115925 , 0.5398998 , 0.23507659, -0.19219233, 0.7176641 ,\n", + " 0.6945623 , 0.03376771, 0.03743221, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00361905, 0. , 0.00742857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01845714, -0.00131429, -0.01542857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 10],\n", + " [ 10, 11, 9],\n", + " [ 10, 12, 7],\n", + " ...,\n", + " [ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 5, 5, 3],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [143, 149, 171],\n", + " ...,\n", + " [114, 127, 155],\n", + " [114, 127, 155],\n", + " [114, 127, 155]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [117, 131, 158],\n", + " [117, 131, 158],\n", + " [117, 131, 158]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [140, 147, 168],\n", + " ...,\n", + " [121, 134, 162],\n", + " [121, 134, 162],\n", + " [121, 134, 162]]], dtype=uint8), 'image': array([[[109, 101, 77],\n", + " [110, 102, 78],\n", + " [ 47, 70, 66],\n", + " ...,\n", + " [150, 157, 165],\n", + " [149, 156, 162],\n", + " [148, 155, 161]],\n", + "\n", + " [[117, 101, 80],\n", + " [114, 98, 77],\n", + " [ 61, 86, 77],\n", + " ...,\n", + " [146, 156, 163],\n", + " [146, 156, 161],\n", + " [146, 156, 161]],\n", + "\n", + " [[108, 83, 65],\n", + " [118, 93, 75],\n", + " [ 77, 105, 90],\n", + " ...,\n", + " [146, 155, 165],\n", + " [142, 152, 157],\n", + " [143, 153, 158]],\n", + "\n", + " ...,\n", + "\n", + " [[ 55, 70, 43],\n", + " [ 59, 74, 47],\n", + " [ 58, 73, 46],\n", + " ...,\n", + " [ 61, 91, 96],\n", + " [ 63, 95, 102],\n", + " [ 60, 92, 99]],\n", + "\n", + " [[ 50, 62, 38],\n", + " [ 52, 64, 40],\n", + " [ 51, 62, 32],\n", + " ...,\n", + " [ 58, 91, 93],\n", + " [ 59, 93, 101],\n", + " [ 58, 92, 100]],\n", + "\n", + " [[ 48, 52, 31],\n", + " [ 50, 54, 33],\n", + " [ 45, 52, 26],\n", + " ...,\n", + " [ 67, 100, 102],\n", + " [ 60, 94, 102],\n", + " [ 60, 94, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8851256 , -0.8193839 , 1.9207363 , -2.767865 , -1.60294 ,\n", + " 3.4364538 , 0.5404305 , 0.24793711, -0.1918753 , 0.71865124,\n", + " 0.69353217, 0.03392792, 0.03744877, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.02457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.0012 , -0.00131429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 153, 175],\n", + " [149, 153, 175],\n", + " [151, 154, 174],\n", + " ...,\n", + " [119, 134, 159],\n", + " [123, 135, 160],\n", + " [123, 135, 160]],\n", + "\n", + " [[145, 149, 171],\n", + " [145, 149, 171],\n", + " [147, 149, 169],\n", + " ...,\n", + " [121, 135, 160],\n", + " [123, 135, 160],\n", + " [123, 135, 160]],\n", + "\n", + " [[146, 150, 172],\n", + " [145, 149, 171],\n", + " [147, 149, 169],\n", + " ...,\n", + " [119, 134, 159],\n", + " [123, 135, 160],\n", + " [123, 135, 160]]], dtype=uint8), 'image': array([[[110, 103, 82],\n", + " [104, 97, 76],\n", + " [ 46, 72, 70],\n", + " ...,\n", + " [149, 156, 162],\n", + " [149, 156, 162],\n", + " [150, 157, 163]],\n", + "\n", + " [[117, 102, 76],\n", + " [114, 99, 73],\n", + " [ 64, 92, 79],\n", + " ...,\n", + " [147, 154, 160],\n", + " [148, 155, 161],\n", + " [147, 154, 160]],\n", + "\n", + " [[114, 85, 60],\n", + " [124, 95, 70],\n", + " [ 75, 102, 84],\n", + " ...,\n", + " [145, 152, 158],\n", + " [143, 148, 157],\n", + " [143, 148, 157]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 71, 48],\n", + " [ 63, 73, 50],\n", + " [ 61, 69, 54],\n", + " ...,\n", + " [ 63, 93, 98],\n", + " [ 66, 96, 101],\n", + " [ 65, 95, 100]],\n", + "\n", + " [[ 55, 65, 38],\n", + " [ 54, 64, 37],\n", + " [ 54, 64, 41],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 98],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 47, 57, 32],\n", + " [ 47, 57, 32],\n", + " [ 43, 53, 28],\n", + " ...,\n", + " [ 65, 94, 104],\n", + " [ 62, 91, 101],\n", + " [ 61, 90, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8629944 , -0.83263665, 1.9082379 , -2.740589 , -1.6067833 ,\n", + " 3.4631937 , 0.53934747, 0.260093 , -0.18352933, 0.72030914,\n", + " 0.69181347, 0.03324209, 0.03799796, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00190476, 0.00457143, 0.03009524], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00245714, -0.00251429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 7, 10, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 169, 180],\n", + " [168, 169, 180],\n", + " [167, 168, 179],\n", + " ...,\n", + " [120, 136, 163],\n", + " [120, 137, 161],\n", + " [120, 137, 161]],\n", + "\n", + " [[166, 167, 178],\n", + " [166, 167, 178],\n", + " [167, 168, 179],\n", + " ...,\n", + " [121, 137, 164],\n", + " [123, 140, 165],\n", + " [122, 139, 163]],\n", + "\n", + " [[165, 165, 177],\n", + " [165, 165, 177],\n", + " [165, 165, 177],\n", + " ...,\n", + " [123, 139, 166],\n", + " [124, 141, 166],\n", + " [126, 142, 167]]], dtype=uint8), 'image': array([[[104, 100, 75],\n", + " [104, 100, 75],\n", + " [ 50, 75, 70],\n", + " ...,\n", + " [151, 156, 163],\n", + " [153, 158, 165],\n", + " [151, 156, 163]],\n", + "\n", + " [[114, 101, 77],\n", + " [110, 97, 73],\n", + " [ 61, 89, 76],\n", + " ...,\n", + " [153, 160, 166],\n", + " [154, 159, 166],\n", + " [148, 153, 160]],\n", + "\n", + " [[119, 92, 70],\n", + " [123, 96, 74],\n", + " [ 74, 106, 85],\n", + " ...,\n", + " [146, 158, 162],\n", + " [145, 152, 158],\n", + " [142, 149, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 55],\n", + " [ 71, 74, 58],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 61, 89, 104],\n", + " [ 63, 91, 106],\n", + " [ 65, 93, 108]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 50, 56, 39],\n", + " ...,\n", + " [ 63, 90, 103],\n", + " [ 63, 90, 103],\n", + " [ 63, 90, 103]],\n", + "\n", + " [[ 50, 56, 37],\n", + " [ 46, 52, 33],\n", + " [ 43, 49, 32],\n", + " ...,\n", + " [ 61, 88, 101],\n", + " [ 62, 89, 102],\n", + " [ 61, 88, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8570101 , -0.86013824, 1.9111109 , -2.715855 , -1.6078013 ,\n", + " 3.482787 , 0.5375394 , 0.26292184, -0.17109968, 0.72502565,\n", + " 0.68685734, 0.0334212 , 0.03804962, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00266667, 0.00457143, 0.02761905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00937143, -0.00274286, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 99, 125, 148],\n", + " [ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[100, 124, 147],\n", + " [100, 124, 147],\n", + " [100, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[163, 165, 178],\n", + " [163, 165, 178],\n", + " [163, 165, 178],\n", + " ...,\n", + " [124, 138, 163],\n", + " [125, 139, 166],\n", + " [125, 139, 166]],\n", + "\n", + " [[163, 165, 178],\n", + " [164, 166, 180],\n", + " [165, 167, 181],\n", + " ...,\n", + " [125, 140, 165],\n", + " [123, 137, 164],\n", + " [123, 137, 164]],\n", + "\n", + " [[163, 165, 178],\n", + " [163, 165, 178],\n", + " [164, 166, 180],\n", + " ...,\n", + " [125, 140, 165],\n", + " [125, 139, 166],\n", + " [125, 139, 166]]], dtype=uint8), 'image': array([[[107, 103, 94],\n", + " [102, 98, 89],\n", + " [ 39, 69, 74],\n", + " ...,\n", + " [142, 149, 155],\n", + " [141, 148, 154],\n", + " [141, 148, 154]],\n", + "\n", + " [[120, 105, 95],\n", + " [109, 94, 84],\n", + " [ 42, 76, 71],\n", + " ...,\n", + " [142, 149, 153],\n", + " [139, 146, 150],\n", + " [140, 147, 151]],\n", + "\n", + " [[116, 92, 78],\n", + " [120, 96, 82],\n", + " [ 57, 92, 82],\n", + " ...,\n", + " [144, 151, 155],\n", + " [141, 146, 151],\n", + " [141, 146, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 70, 49],\n", + " [ 71, 75, 54],\n", + " [ 67, 71, 50],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 100],\n", + " [ 62, 91, 101]],\n", + "\n", + " [[ 58, 62, 41],\n", + " [ 57, 61, 40],\n", + " [ 53, 59, 40],\n", + " ...,\n", + " [ 62, 96, 104],\n", + " [ 58, 90, 101],\n", + " [ 57, 89, 100]],\n", + "\n", + " [[ 52, 55, 37],\n", + " [ 50, 53, 35],\n", + " [ 45, 51, 34],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 57, 91, 101],\n", + " [ 59, 93, 103]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8583746 , -0.8919328 , 1.9205203 , -2.6912768 , -1.6058615 ,\n", + " 3.5005364 , 0.5345636 , 0.26166272, -0.15828562, 0.73158187,\n", + " 0.67996943, 0.03315829, 0.03647007, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0032381 , 0.008 , 0.02761905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.0132 , -0.00371429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[106, 130, 153],\n", + " [106, 130, 153],\n", + " [106, 130, 153],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 4, 3, 1],\n", + " [ 4, 3, 1]],\n", + "\n", + " [[106, 130, 153],\n", + " [106, 130, 153],\n", + " [106, 130, 153],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[108, 129, 153],\n", + " [108, 129, 153],\n", + " [108, 129, 153],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [167, 164, 178],\n", + " ...,\n", + " [128, 143, 165],\n", + " [129, 143, 168],\n", + " [129, 143, 168]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [167, 164, 178],\n", + " ...,\n", + " [128, 143, 165],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [166, 163, 177],\n", + " ...,\n", + " [126, 142, 164],\n", + " [126, 141, 166],\n", + " [128, 142, 167]]], dtype=uint8), 'image': array([[[104, 101, 87],\n", + " [ 95, 92, 78],\n", + " [ 33, 69, 75],\n", + " ...,\n", + " [212, 215, 214],\n", + " [207, 207, 207],\n", + " [204, 204, 204]],\n", + "\n", + " [[109, 97, 79],\n", + " [115, 103, 85],\n", + " [ 50, 86, 84],\n", + " ...,\n", + " [212, 215, 214],\n", + " [208, 207, 207],\n", + " [205, 204, 204]],\n", + "\n", + " [[109, 85, 71],\n", + " [122, 98, 84],\n", + " [ 73, 104, 100],\n", + " ...,\n", + " [209, 212, 211],\n", + " [208, 208, 208],\n", + " [206, 206, 206]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 70, 55],\n", + " [ 67, 75, 60],\n", + " [ 66, 72, 55],\n", + " ...,\n", + " [ 61, 90, 102],\n", + " [ 62, 90, 105],\n", + " [ 62, 90, 105]],\n", + "\n", + " [[ 59, 65, 46],\n", + " [ 58, 64, 45],\n", + " [ 51, 57, 40],\n", + " ...,\n", + " [ 62, 91, 101],\n", + " [ 62, 91, 103],\n", + " [ 59, 88, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 49, 55, 36],\n", + " [ 43, 49, 30],\n", + " ...,\n", + " [ 62, 91, 101],\n", + " [ 60, 89, 101],\n", + " [ 60, 89, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.866813 , -0.92632943, 1.9351735 , -2.669913 , -1.602569 ,\n", + " 3.5102549 , 0.5314049 , 0.25643107, -0.14546628, 0.73773456,\n", + " 0.67338413, 0.0331663 , 0.03466382, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.01561905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01651429, -0.0068 , -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[116, 138, 160],\n", + " [115, 137, 158],\n", + " [115, 137, 158],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[115, 137, 158],\n", + " [113, 136, 157],\n", + " [113, 136, 157],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[111, 135, 159],\n", + " [111, 135, 159],\n", + " [111, 135, 159],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 168, 182],\n", + " [168, 168, 182],\n", + " [167, 167, 181],\n", + " ...,\n", + " [127, 144, 168],\n", + " [129, 143, 168],\n", + " [129, 143, 168]],\n", + "\n", + " [[168, 168, 182],\n", + " [167, 167, 181],\n", + " [169, 169, 183],\n", + " ...,\n", + " [128, 145, 169],\n", + " [130, 144, 169],\n", + " [130, 144, 169]],\n", + "\n", + " [[168, 168, 182],\n", + " [168, 168, 182],\n", + " [167, 167, 181],\n", + " ...,\n", + " [129, 146, 170],\n", + " [131, 145, 170],\n", + " [130, 144, 169]]], dtype=uint8), 'image': array([[[102, 100, 77],\n", + " [104, 102, 79],\n", + " [ 38, 72, 69],\n", + " ...,\n", + " [181, 181, 183],\n", + " [182, 182, 182],\n", + " [185, 185, 185]],\n", + "\n", + " [[116, 100, 79],\n", + " [122, 106, 85],\n", + " [ 48, 83, 73],\n", + " ...,\n", + " [180, 180, 184],\n", + " [179, 182, 185],\n", + " [184, 187, 190]],\n", + "\n", + " [[118, 88, 72],\n", + " [133, 103, 87],\n", + " [ 74, 107, 92],\n", + " ...,\n", + " [176, 176, 180],\n", + " [179, 181, 187],\n", + " [185, 187, 193]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 77, 51],\n", + " [ 71, 78, 52],\n", + " [ 68, 70, 52],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 91, 99],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 58, 64, 45],\n", + " [ 58, 61, 45],\n", + " ...,\n", + " [ 64, 92, 98],\n", + " [ 62, 91, 99],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 57, 59, 46],\n", + " [ 52, 54, 41],\n", + " [ 48, 50, 37],\n", + " ...,\n", + " [ 63, 92, 100],\n", + " [ 63, 90, 99],\n", + " [ 63, 90, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8797753 , -0.962189 , 1.9533105 , -2.6487052 , -1.5976962 ,\n", + " 3.5144436 , 0.52728325, 0.24874139, -0.1332232 , 0.7435012 ,\n", + " 0.66718084, 0.03253057, 0.03189772, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00361905, 0.00933333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.0172 , -0.00937143, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[129, 148, 166],\n", + " [129, 148, 166],\n", + " [129, 148, 166],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[129, 148, 166],\n", + " [129, 148, 166],\n", + " [128, 147, 165],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[129, 148, 166],\n", + " [129, 148, 166],\n", + " [128, 147, 165],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 165, 180],\n", + " [168, 165, 180],\n", + " [168, 165, 180],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [168, 165, 180],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [169, 167, 181],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [126, 141, 166]]], dtype=uint8), 'image': array([[[105, 98, 79],\n", + " [108, 101, 82],\n", + " [ 42, 71, 68],\n", + " ...,\n", + " [176, 175, 177],\n", + " [170, 168, 175],\n", + " [168, 166, 173]],\n", + "\n", + " [[119, 100, 83],\n", + " [123, 104, 87],\n", + " [ 54, 84, 76],\n", + " ...,\n", + " [175, 174, 176],\n", + " [173, 172, 176],\n", + " [169, 168, 172]],\n", + "\n", + " [[123, 92, 81],\n", + " [123, 92, 81],\n", + " [ 73, 104, 90],\n", + " ...,\n", + " [175, 174, 174],\n", + " [175, 171, 174],\n", + " [172, 168, 171]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 78, 56],\n", + " [ 71, 78, 56],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 65, 97, 104],\n", + " [ 66, 96, 99],\n", + " [ 62, 92, 95]],\n", + "\n", + " [[ 55, 59, 36],\n", + " [ 59, 63, 40],\n", + " [ 59, 62, 44],\n", + " ...,\n", + " [ 64, 96, 103],\n", + " [ 63, 93, 98],\n", + " [ 62, 92, 97]],\n", + "\n", + " [[ 52, 55, 34],\n", + " [ 52, 55, 34],\n", + " [ 46, 50, 29],\n", + " ...,\n", + " [ 64, 96, 105],\n", + " [ 63, 92, 100],\n", + " [ 61, 90, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8943539 , -1.002066 , 1.9794574 , -2.6350296 , -1.5959238 ,\n", + " 3.5113556 , 0.52145064, 0.23904608, -0.12084077, 0.7473314 ,\n", + " 0.6629031 , 0.0326565 , 0.0314473 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00057143, -0.00780952, 0.0072381 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01051429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[117, 136, 156],\n", + " [119, 137, 157],\n", + " [119, 137, 157],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[119, 137, 157],\n", + " [119, 137, 157],\n", + " [119, 137, 157],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[120, 138, 158],\n", + " [120, 138, 158],\n", + " [120, 138, 158],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [173, 169, 181],\n", + " ...,\n", + " [140, 146, 170],\n", + " [140, 146, 170],\n", + " [140, 146, 170]],\n", + "\n", + " [[175, 171, 183],\n", + " [176, 172, 184],\n", + " [175, 171, 183],\n", + " ...,\n", + " [139, 145, 169],\n", + " [138, 144, 168],\n", + " [139, 145, 169]],\n", + "\n", + " [[176, 172, 184],\n", + " [176, 172, 184],\n", + " [175, 171, 183],\n", + " ...,\n", + " [138, 144, 168],\n", + " [139, 145, 169],\n", + " [139, 145, 169]]], dtype=uint8), 'image': array([[[108, 105, 71],\n", + " [106, 103, 69],\n", + " [ 41, 73, 62],\n", + " ...,\n", + " [129, 132, 133],\n", + " [129, 135, 135],\n", + " [127, 133, 133]],\n", + "\n", + " [[123, 108, 78],\n", + " [121, 106, 76],\n", + " [ 56, 89, 72],\n", + " ...,\n", + " [130, 130, 132],\n", + " [128, 134, 136],\n", + " [129, 135, 137]],\n", + "\n", + " [[112, 86, 57],\n", + " [124, 98, 69],\n", + " [ 69, 101, 80],\n", + " ...,\n", + " [128, 130, 136],\n", + " [129, 135, 137],\n", + " [133, 139, 141]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 71, 49],\n", + " [ 69, 76, 54],\n", + " [ 67, 73, 56],\n", + " ...,\n", + " [ 60, 94, 102],\n", + " [ 59, 93, 103],\n", + " [ 59, 93, 103]],\n", + "\n", + " [[ 57, 64, 40],\n", + " [ 53, 60, 36],\n", + " [ 55, 58, 40],\n", + " ...,\n", + " [ 58, 93, 106],\n", + " [ 56, 91, 104],\n", + " [ 57, 92, 105]],\n", + "\n", + " [[ 53, 56, 35],\n", + " [ 51, 54, 33],\n", + " [ 47, 50, 32],\n", + " ...,\n", + " [ 58, 93, 106],\n", + " [ 57, 92, 107],\n", + " [ 56, 91, 106]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9088843 , -1.0461868 , 2.012751 , -2.6265728 , -1.5953134 ,\n", + " 3.5029898 , 0.5132325 , 0.2287732 , -0.10842803, 0.74936545,\n", + " 0.66052914, 0.03346506, 0.03213668, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00019048, -0.00876191, 0.00685714], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01068571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[119, 134, 152],\n", + " [119, 134, 152],\n", + " [119, 134, 152],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[118, 132, 151],\n", + " [118, 132, 151],\n", + " [118, 132, 151],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[118, 132, 151],\n", + " [118, 132, 151],\n", + " [118, 132, 151],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [137, 144, 170],\n", + " [138, 145, 172],\n", + " [138, 145, 172]],\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [140, 147, 173],\n", + " [138, 145, 172],\n", + " [138, 145, 172]],\n", + "\n", + " [[173, 169, 181],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [141, 148, 174],\n", + " [138, 145, 172],\n", + " [138, 145, 172]]], dtype=uint8), 'image': array([[[109, 104, 73],\n", + " [109, 104, 73],\n", + " [ 49, 76, 67],\n", + " ...,\n", + " [103, 104, 95],\n", + " [106, 103, 98],\n", + " [107, 104, 99]],\n", + "\n", + " [[119, 102, 73],\n", + " [116, 99, 70],\n", + " [ 57, 85, 70],\n", + " ...,\n", + " [104, 105, 96],\n", + " [105, 106, 97],\n", + " [105, 106, 97]],\n", + "\n", + " [[126, 99, 73],\n", + " [120, 93, 67],\n", + " [ 77, 106, 86],\n", + " ...,\n", + " [105, 106, 97],\n", + " [106, 107, 98],\n", + " [108, 109, 100]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 72, 51],\n", + " [ 74, 78, 57],\n", + " [ 73, 75, 62],\n", + " ...,\n", + " [ 63, 97, 107],\n", + " [ 61, 93, 102],\n", + " [ 60, 92, 101]],\n", + "\n", + " [[ 58, 65, 43],\n", + " [ 58, 65, 43],\n", + " [ 56, 61, 49],\n", + " ...,\n", + " [ 63, 97, 107],\n", + " [ 63, 95, 106],\n", + " [ 60, 92, 103]],\n", + "\n", + " [[ 54, 57, 41],\n", + " [ 52, 55, 39],\n", + " [ 45, 50, 36],\n", + " ...,\n", + " [ 60, 94, 104],\n", + " [ 64, 96, 107],\n", + " [ 60, 92, 103]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9227448 , -1.090619 , 2.0471349 , -2.6207187 , -1.5956005 ,\n", + " 3.4939177 , 0.5043737 , 0.21883176, -0.09658621, 0.750923 ,\n", + " 0.65860933, 0.03484912, 0.03367477, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00971429, 0.00666667], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01091429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 140, 158],\n", + " [119, 138, 155],\n", + " [121, 140, 158],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[119, 138, 155],\n", + " [119, 138, 155],\n", + " [119, 138, 155],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[119, 138, 155],\n", + " [119, 138, 155],\n", + " [119, 138, 155],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[166, 165, 181],\n", + " [166, 165, 181],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [136, 146, 172],\n", + " [138, 147, 173]],\n", + "\n", + " [[165, 164, 180],\n", + " [165, 164, 180],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [138, 147, 173],\n", + " [136, 146, 172]],\n", + "\n", + " [[165, 164, 180],\n", + " [165, 164, 180],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [136, 146, 172],\n", + " [138, 147, 173]]], dtype=uint8), 'image': array([[[110, 103, 70],\n", + " [107, 100, 67],\n", + " [ 42, 72, 66],\n", + " ...,\n", + " [ 85, 82, 77],\n", + " [ 86, 83, 78],\n", + " [ 87, 84, 79]],\n", + "\n", + " [[124, 106, 74],\n", + " [117, 99, 67],\n", + " [ 57, 88, 74],\n", + " ...,\n", + " [ 85, 82, 77],\n", + " [ 87, 83, 76],\n", + " [ 90, 86, 79]],\n", + "\n", + " [[117, 90, 57],\n", + " [125, 98, 65],\n", + " [ 70, 102, 81],\n", + " ...,\n", + " [ 84, 81, 76],\n", + " [ 85, 83, 75],\n", + " [ 87, 85, 77]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 55],\n", + " [ 72, 75, 59],\n", + " [ 69, 72, 54],\n", + " ...,\n", + " [ 59, 93, 103],\n", + " [ 60, 94, 102],\n", + " [ 59, 93, 101]],\n", + "\n", + " [[ 60, 62, 49],\n", + " [ 60, 62, 49],\n", + " [ 58, 61, 45],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 63, 92, 102],\n", + " [ 63, 92, 102]],\n", + "\n", + " [[ 56, 57, 46],\n", + " [ 52, 53, 42],\n", + " [ 47, 49, 36],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 61, 90, 98],\n", + " [ 62, 91, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9372277 , -1.138132 , 2.0828366 , -2.614289 , -1.5960315 ,\n", + " 3.4834929 , 0.4947406 , 0.20868209, -0.08435462, 0.7522201 ,\n", + " 0.656926 , 0.03664842, 0.03563798, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01047619, 0.00609524], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.0112 , -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[126, 144, 159],\n", + " [126, 144, 159],\n", + " [125, 143, 158],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[128, 145, 161],\n", + " [126, 144, 159],\n", + " [125, 143, 158],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3]],\n", + "\n", + " [[130, 145, 161],\n", + " [129, 144, 159],\n", + " [126, 144, 159],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 175, 189],\n", + " [175, 175, 189],\n", + " [174, 174, 188],\n", + " ...,\n", + " [145, 151, 166],\n", + " [145, 150, 168],\n", + " [145, 150, 168]],\n", + "\n", + " [[174, 174, 188],\n", + " [174, 174, 188],\n", + " [174, 174, 188],\n", + " ...,\n", + " [146, 152, 167],\n", + " [145, 150, 168],\n", + " [145, 150, 168]],\n", + "\n", + " [[173, 173, 187],\n", + " [174, 174, 188],\n", + " [174, 174, 188],\n", + " ...,\n", + " [146, 152, 167],\n", + " [146, 151, 169],\n", + " [145, 150, 168]]], dtype=uint8), 'image': array([[[101, 102, 89],\n", + " [ 97, 98, 85],\n", + " [ 48, 83, 84],\n", + " ...,\n", + " [ 90, 87, 82],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " [[112, 103, 84],\n", + " [114, 105, 86],\n", + " [ 41, 79, 73],\n", + " ...,\n", + " [ 84, 84, 78],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " [[114, 91, 72],\n", + " [134, 111, 92],\n", + " [ 60, 97, 84],\n", + " ...,\n", + " [ 89, 86, 81],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 70, 51],\n", + " [ 69, 75, 56],\n", + " [ 63, 72, 54],\n", + " ...,\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98],\n", + " [ 60, 89, 97]],\n", + "\n", + " [[ 56, 65, 47],\n", + " [ 55, 64, 46],\n", + " [ 54, 65, 46],\n", + " ...,\n", + " [ 63, 95, 104],\n", + " [ 59, 88, 96],\n", + " [ 60, 89, 97]],\n", + "\n", + " [[ 52, 58, 41],\n", + " [ 50, 56, 39],\n", + " [ 49, 58, 40],\n", + " ...,\n", + " [ 58, 90, 99],\n", + " [ 56, 88, 95],\n", + " [ 57, 89, 96]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9522376 , -1.1864475 , 2.1171021 , -2.6065881 , -1.5964864 ,\n", + " 3.4725657 , 0.48520797, 0.19863266, -0.07217506, 0.75350314,\n", + " 0.65520203, 0.03879603, 0.03792322, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01314286, 0.00419048], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01691429, -0.01308571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[142, 154, 166],\n", + " [142, 154, 166],\n", + " [142, 154, 166],\n", + " ...,\n", + " [ 9, 5, 3],\n", + " [ 8, 4, 0],\n", + " [ 10, 7, 3]],\n", + "\n", + " [[142, 154, 166],\n", + " [142, 154, 166],\n", + " [141, 153, 164],\n", + " ...,\n", + " [ 9, 5, 3],\n", + " [ 10, 7, 3],\n", + " [ 10, 7, 3]],\n", + "\n", + " [[141, 153, 164],\n", + " [141, 153, 164],\n", + " [141, 153, 164],\n", + " ...,\n", + " [ 9, 5, 1],\n", + " [ 10, 7, 3],\n", + " [ 10, 7, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[177, 180, 191],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [196, 212, 171],\n", + " [197, 213, 172],\n", + " [197, 213, 172]],\n", + "\n", + " [[177, 180, 191],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [198, 214, 173],\n", + " [198, 214, 173],\n", + " [197, 213, 172]],\n", + "\n", + " [[176, 179, 190],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [198, 214, 173],\n", + " [198, 214, 173],\n", + " [197, 213, 172]]], dtype=uint8), 'image': array([[[104, 98, 85],\n", + " [103, 97, 84],\n", + " [ 45, 72, 81],\n", + " ...,\n", + " [138, 136, 143],\n", + " [126, 119, 130],\n", + " [113, 106, 117]],\n", + "\n", + " [[114, 99, 87],\n", + " [114, 99, 87],\n", + " [ 52, 83, 83],\n", + " ...,\n", + " [132, 131, 142],\n", + " [125, 120, 130],\n", + " [113, 108, 118]],\n", + "\n", + " [[123, 97, 84],\n", + " [122, 96, 83],\n", + " [ 70, 99, 96],\n", + " ...,\n", + " [133, 134, 149],\n", + " [116, 113, 129],\n", + " [102, 99, 115]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 54],\n", + " [ 71, 74, 58],\n", + " [ 67, 73, 56],\n", + " ...,\n", + " [ 56, 90, 100],\n", + " [ 54, 83, 95],\n", + " [ 61, 90, 102]],\n", + "\n", + " [[ 58, 61, 45],\n", + " [ 59, 62, 46],\n", + " [ 60, 62, 49],\n", + " ...,\n", + " [ 58, 90, 101],\n", + " [ 58, 87, 99],\n", + " [ 54, 83, 95]],\n", + "\n", + " [[ 52, 54, 43],\n", + " [ 50, 52, 41],\n", + " [ 45, 47, 38],\n", + " ...,\n", + " [ 61, 93, 104],\n", + " [ 60, 92, 103],\n", + " [ 54, 86, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9681327 , -1.236956 , 2.1511526 , -2.5970871 , -1.5968579 ,\n", + " 3.4606204 , 0.4752485 , 0.18844554, -0.05988858, 0.7547231 ,\n", + " 0.65350336, 0.0411324 , 0.04042946, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01333333, 0.00342857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01662857, -0.01388571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 63, 60, 69],\n", + " [ 66, 63, 73],\n", + " [ 70, 69, 79],\n", + " ...,\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " [[ 81, 84, 95],\n", + " [ 86, 89, 100],\n", + " [ 87, 93, 103],\n", + " ...,\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4]],\n", + "\n", + " [[101, 110, 124],\n", + " [103, 112, 126],\n", + " [108, 117, 129],\n", + " ...,\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 188, 197],\n", + " [189, 188, 197],\n", + " [189, 188, 197],\n", + " ...,\n", + " [195, 211, 170],\n", + " [194, 212, 171],\n", + " [194, 212, 171]],\n", + "\n", + " [[188, 187, 196],\n", + " [188, 187, 196],\n", + " [188, 187, 196],\n", + " ...,\n", + " [197, 213, 172],\n", + " [196, 215, 173],\n", + " [194, 212, 171]],\n", + "\n", + " [[187, 186, 195],\n", + " [188, 187, 196],\n", + " [188, 187, 196],\n", + " ...,\n", + " [197, 213, 172],\n", + " [195, 214, 172],\n", + " [194, 212, 171]]], dtype=uint8), 'image': array([[[ 98, 101, 80],\n", + " [ 95, 98, 77],\n", + " [ 36, 74, 72],\n", + " ...,\n", + " [155, 157, 165],\n", + " [156, 158, 166],\n", + " [156, 158, 166]],\n", + "\n", + " [[107, 95, 77],\n", + " [114, 102, 84],\n", + " [ 53, 90, 81],\n", + " ...,\n", + " [151, 156, 165],\n", + " [151, 156, 165],\n", + " [151, 156, 165]],\n", + "\n", + " [[118, 92, 79],\n", + " [123, 97, 84],\n", + " [ 67, 102, 92],\n", + " ...,\n", + " [151, 155, 167],\n", + " [148, 152, 164],\n", + " [147, 151, 163]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 68, 53],\n", + " [ 74, 73, 58],\n", + " [ 70, 72, 56],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 63, 95, 102],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 65, 65, 43],\n", + " [ 63, 63, 41],\n", + " [ 58, 58, 38],\n", + " ...,\n", + " [ 60, 92, 99],\n", + " [ 64, 96, 103],\n", + " [ 63, 95, 102]],\n", + "\n", + " [[ 56, 56, 32],\n", + " [ 56, 56, 32],\n", + " [ 50, 50, 28],\n", + " ...,\n", + " [ 65, 94, 102],\n", + " [ 63, 95, 104],\n", + " [ 61, 93, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.98276 , -1.2880267 , 2.1854315 , -2.5885217 , -1.5973486 ,\n", + " 3.4484115 , 0.46462205, 0.17914946, -0.04822387, 0.75541085,\n", + " 0.65233696, 0.04393689, 0.04336553, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01409524, 0.00152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01582857, -0.01577143, -0.0192 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7],\n", + " [ 11, 7, 5]],\n", + "\n", + " [[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 11, 7, 5],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " [[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 11, 7, 5],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 191, 200],\n", + " [189, 191, 200],\n", + " [188, 190, 198],\n", + " ...,\n", + " [201, 218, 174],\n", + " [201, 218, 174],\n", + " [201, 218, 174]],\n", + "\n", + " [[188, 190, 198],\n", + " [188, 190, 198],\n", + " [187, 188, 197],\n", + " ...,\n", + " [199, 216, 173],\n", + " [201, 218, 174],\n", + " [201, 218, 174]],\n", + "\n", + " [[186, 187, 196],\n", + " [187, 188, 197],\n", + " [187, 188, 197],\n", + " ...,\n", + " [201, 218, 174],\n", + " [199, 216, 173],\n", + " [199, 216, 173]]], dtype=uint8), 'image': array([[[106, 103, 89],\n", + " [102, 99, 85],\n", + " [ 43, 73, 78],\n", + " ...,\n", + " [152, 164, 170],\n", + " [150, 160, 165],\n", + " [151, 161, 166]],\n", + "\n", + " [[115, 103, 87],\n", + " [116, 104, 88],\n", + " [ 55, 86, 82],\n", + " ...,\n", + " [152, 162, 169],\n", + " [150, 157, 163],\n", + " [150, 157, 163]],\n", + "\n", + " [[113, 87, 72],\n", + " [122, 96, 81],\n", + " [ 70, 101, 87],\n", + " ...,\n", + " [150, 160, 167],\n", + " [146, 153, 161],\n", + " [147, 154, 162]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 54],\n", + " [ 72, 75, 59],\n", + " [ 64, 72, 59],\n", + " ...,\n", + " [ 61, 93, 104],\n", + " [ 64, 96, 105],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 61, 64, 48],\n", + " [ 60, 63, 47],\n", + " [ 57, 62, 52],\n", + " ...,\n", + " [ 59, 93, 103],\n", + " [ 64, 96, 105],\n", + " [ 60, 92, 101]],\n", + "\n", + " [[ 54, 55, 44],\n", + " [ 52, 53, 42],\n", + " [ 48, 53, 41],\n", + " ...,\n", + " [ 60, 94, 104],\n", + " [ 60, 92, 101],\n", + " [ 61, 93, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9976504 , -1.3427528 , 2.2203817 , -2.5778692 , -1.5979112 ,\n", + " 3.4355202 , 0.45309877, 0.16995318, -0.03630768, 0.7559353 ,\n", + " 0.6512862 , 0.04700414, 0.04667673, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00114286, -0.01142857, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01417143, -0.01828571, -0.01594286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 11, 8, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 7, 4],\n", + " [ 13, 7, 4]],\n", + "\n", + " [[ 10, 7, 3],\n", + " [ 10, 7, 3],\n", + " [ 9, 5, 1],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 9, 5, 1],\n", + " [ 9, 5, 1],\n", + " [ 9, 5, 1],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [184, 190, 198],\n", + " ...,\n", + " [201, 220, 176],\n", + " [202, 221, 177],\n", + " [202, 221, 177]],\n", + "\n", + " [[183, 189, 197],\n", + " [184, 190, 198],\n", + " [183, 189, 197],\n", + " ...,\n", + " [202, 221, 177],\n", + " [201, 220, 176],\n", + " [201, 220, 176]],\n", + "\n", + " [[184, 190, 198],\n", + " [183, 189, 197],\n", + " [183, 189, 197],\n", + " ...,\n", + " [201, 220, 176],\n", + " [201, 220, 176],\n", + " [201, 220, 176]]], dtype=uint8), 'image': array([[[107, 101, 86],\n", + " [102, 96, 81],\n", + " [ 39, 66, 77],\n", + " ...,\n", + " [154, 161, 167],\n", + " [154, 161, 167],\n", + " [153, 160, 166]],\n", + "\n", + " [[118, 98, 88],\n", + " [122, 102, 92],\n", + " [ 63, 91, 95],\n", + " ...,\n", + " [153, 160, 166],\n", + " [154, 159, 168],\n", + " [155, 160, 169]],\n", + "\n", + " [[121, 90, 79],\n", + " [128, 97, 86],\n", + " [ 70, 94, 92],\n", + " ...,\n", + " [153, 158, 167],\n", + " [147, 151, 163],\n", + " [145, 149, 161]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 69, 60],\n", + " [ 67, 76, 67],\n", + " [ 65, 73, 60],\n", + " ...,\n", + " [ 60, 92, 101],\n", + " [ 59, 91, 100],\n", + " [ 61, 93, 102]],\n", + "\n", + " [[ 56, 64, 51],\n", + " [ 53, 61, 48],\n", + " [ 49, 57, 42],\n", + " ...,\n", + " [ 58, 90, 99],\n", + " [ 56, 88, 99],\n", + " [ 57, 89, 100]],\n", + "\n", + " [[ 53, 58, 46],\n", + " [ 51, 56, 44],\n", + " [ 43, 51, 36],\n", + " ...,\n", + " [ 57, 91, 99],\n", + " [ 57, 91, 101],\n", + " [ 58, 92, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0120497 , -1.400019 , 2.2567096 , -2.5662448 , -1.5984977 ,\n", + " 3.4220662 , 0.4402678 , 0.16121309, -0.02498595, 0.75610036,\n", + " 0.6506091 , 0.05005798, 0.05014284, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00533333, -0.01238095, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01262857, -0.01931429, -0.01217143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 6, 4, 2],\n", + " [ 8, 6, 4],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 5, 5, 3],\n", + " [ 5, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[190, 192, 201],\n", + " [190, 192, 201],\n", + " [189, 191, 200],\n", + " ...,\n", + " [203, 223, 178],\n", + " [204, 224, 180],\n", + " [204, 224, 180]],\n", + "\n", + " [[187, 188, 197],\n", + " [188, 190, 198],\n", + " [188, 190, 198],\n", + " ...,\n", + " [203, 223, 178],\n", + " [201, 220, 176],\n", + " [201, 220, 176]],\n", + "\n", + " [[187, 188, 197],\n", + " [187, 188, 197],\n", + " [188, 190, 198],\n", + " ...,\n", + " [203, 223, 178],\n", + " [202, 221, 177],\n", + " [201, 220, 176]]], dtype=uint8), 'image': array([[[110, 105, 85],\n", + " [109, 104, 84],\n", + " [ 40, 71, 69],\n", + " ...,\n", + " [152, 159, 165],\n", + " [150, 155, 164],\n", + " [150, 155, 164]],\n", + "\n", + " [[121, 104, 86],\n", + " [124, 107, 89],\n", + " [ 55, 87, 78],\n", + " ...,\n", + " [151, 161, 166],\n", + " [150, 155, 164],\n", + " [150, 155, 164]],\n", + "\n", + " [[111, 83, 64],\n", + " [127, 99, 80],\n", + " [ 76, 109, 92],\n", + " ...,\n", + " [147, 161, 167],\n", + " [146, 152, 163],\n", + " [141, 147, 158]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 74, 51],\n", + " [ 70, 80, 57],\n", + " [ 65, 75, 48],\n", + " ...,\n", + " [ 58, 92, 98],\n", + " [ 61, 93, 100],\n", + " [ 61, 93, 100]],\n", + "\n", + " [[ 55, 65, 42],\n", + " [ 55, 65, 42],\n", + " [ 54, 64, 39],\n", + " ...,\n", + " [ 64, 96, 105],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 48, 57, 37],\n", + " [ 44, 53, 33],\n", + " [ 44, 53, 33],\n", + " ...,\n", + " [ 63, 92, 104],\n", + " [ 66, 93, 104],\n", + " [ 64, 91, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.024355 , -1.4538416 , 2.2930508 , -2.5565522 , -1.5979952 ,\n", + " 3.4096293 , 0.42665708, 0.1540214 , -0.0161692 , 0.75591356,\n", + " 0.65040565, 0.05278223, 0.05273771, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0032381, 0. , 0.0047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00514286, -0.00582857, -0.00068571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[191, 195, 203],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [205, 222, 178],\n", + " [205, 222, 178]],\n", + "\n", + " [[191, 195, 203],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [204, 221, 177],\n", + " [204, 221, 177]],\n", + "\n", + " [[189, 193, 202],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [204, 221, 177],\n", + " [204, 221, 177]]], dtype=uint8), 'image': array([[[109, 105, 94],\n", + " [107, 103, 92],\n", + " [ 34, 61, 70],\n", + " ...,\n", + " [153, 161, 162],\n", + " [153, 160, 164],\n", + " [153, 160, 164]],\n", + "\n", + " [[120, 107, 94],\n", + " [120, 107, 94],\n", + " [ 51, 80, 79],\n", + " ...,\n", + " [150, 157, 163],\n", + " [152, 159, 167],\n", + " [150, 157, 165]],\n", + "\n", + " [[111, 85, 74],\n", + " [130, 104, 93],\n", + " [ 75, 105, 99],\n", + " ...,\n", + " [149, 155, 168],\n", + " [146, 154, 169],\n", + " [144, 152, 167]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 75, 60],\n", + " [ 68, 81, 66],\n", + " [ 71, 81, 67],\n", + " ...,\n", + " [ 64, 96, 107],\n", + " [ 60, 92, 101],\n", + " [ 57, 89, 98]],\n", + "\n", + " [[ 56, 67, 48],\n", + " [ 56, 67, 48],\n", + " [ 54, 62, 47],\n", + " ...,\n", + " [ 56, 88, 99],\n", + " [ 54, 85, 99],\n", + " [ 57, 88, 102]],\n", + "\n", + " [[ 50, 59, 39],\n", + " [ 49, 58, 38],\n", + " [ 46, 55, 35],\n", + " ...,\n", + " [ 56, 87, 101],\n", + " [ 57, 88, 104],\n", + " [ 60, 91, 107]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0366597 , -1.50608 , 2.3327332 , -2.5523531 , -1.5953492 ,\n", + " 3.3971448 , 0.41204247, 0.14764796, -0.00976385, 0.75563943,\n", + " 0.65026915, 0.05629408, 0.054681 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0., 0., -0.], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 5, 5, 3],\n", + " [ 3, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 5, 5, 3],\n", + " [ 5, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 13, 10, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[187, 194, 202],\n", + " [187, 194, 202],\n", + " [186, 193, 201],\n", + " ...,\n", + " [211, 225, 184],\n", + " [211, 225, 184],\n", + " [211, 225, 184]],\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [186, 193, 201],\n", + " ...,\n", + " [211, 225, 184],\n", + " [211, 225, 184],\n", + " [211, 225, 184]],\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [186, 193, 201],\n", + " ...,\n", + " [212, 226, 185],\n", + " [211, 225, 184],\n", + " [211, 225, 184]]], dtype=uint8), 'image': array([[[111, 103, 77],\n", + " [114, 106, 80],\n", + " [ 49, 75, 62],\n", + " ...,\n", + " [ 47, 53, 55],\n", + " [ 49, 48, 50],\n", + " [ 51, 50, 52]],\n", + "\n", + " [[118, 100, 78],\n", + " [119, 101, 79],\n", + " [ 61, 89, 74],\n", + " ...,\n", + " [ 49, 52, 51],\n", + " [ 57, 48, 51],\n", + " [ 58, 49, 52]],\n", + "\n", + " [[113, 86, 64],\n", + " [126, 99, 77],\n", + " [ 82, 111, 93],\n", + " ...,\n", + " [ 53, 53, 47],\n", + " [ 64, 49, 50],\n", + " [ 65, 50, 51]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 75, 56],\n", + " [ 70, 76, 57],\n", + " [ 69, 75, 56],\n", + " ...,\n", + " [ 63, 94, 108],\n", + " [ 66, 98, 109],\n", + " [ 66, 98, 109]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " [ 55, 61, 42],\n", + " ...,\n", + " [ 65, 96, 110],\n", + " [ 68, 97, 109],\n", + " [ 67, 96, 108]],\n", + "\n", + " [[ 48, 55, 33],\n", + " [ 50, 57, 35],\n", + " [ 49, 55, 36],\n", + " ...,\n", + " [ 65, 97, 108],\n", + " [ 68, 97, 109],\n", + " [ 67, 96, 108]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0452423 , -1.5262693 , 2.3518927 , -2.553457 , -1.5919966 ,\n", + " 3.3911233 , 0.405479 , 0.1441851 , -0.00875834, 0.75636166,\n", + " 0.6493327 , 0.05772375, 0.05433304, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 1.0, 'world_vector': array([0., 0., 0.], dtype=float32)}, 'is_first': False, 'is_last': True, 'is_terminal': True, 'observation': {'hand_image': array([[[ 4, 4, 0],\n", + " [ 6, 5, 0],\n", + " [ 8, 6, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 193, 202],\n", + " [189, 193, 202],\n", + " [188, 192, 201],\n", + " ...,\n", + " [211, 224, 185],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[188, 192, 201],\n", + " [188, 192, 201],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[187, 191, 200],\n", + " [187, 191, 200],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]]], dtype=uint8), 'image': array([[[108, 101, 82],\n", + " [104, 97, 78],\n", + " [ 46, 72, 72],\n", + " ...,\n", + " [ 12, 19, 13],\n", + " [ 7, 15, 16],\n", + " [ 39, 47, 48]],\n", + "\n", + " [[115, 102, 78],\n", + " [119, 106, 82],\n", + " [ 61, 91, 83],\n", + " ...,\n", + " [ 16, 20, 15],\n", + " [ 23, 26, 29],\n", + " [ 51, 54, 57]],\n", + "\n", + " [[114, 90, 67],\n", + " [124, 100, 77],\n", + " [ 71, 102, 86],\n", + " ...,\n", + " [ 22, 25, 24],\n", + " [ 43, 43, 45],\n", + " [ 48, 48, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 72, 55],\n", + " [ 71, 77, 60],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 67, 96, 106],\n", + " [ 66, 98, 107],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 55, 61, 44],\n", + " ...,\n", + " [ 69, 98, 106],\n", + " [ 69, 98, 106],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 46, 52, 33],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [ 68, 97, 107],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0464277 , -1.5272526 , 2.3538566 , -2.5545247 , -1.5915655 ,\n", + " 3.390477 , 0.40496254, 0.14374796, -0.00899097, 0.75652575,\n", + " 0.64914 , 0.05783392, 0.05423374, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 1.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 1.0, 'world_vector': array([0., 0., 0.], dtype=float32)}, 'is_first': False, 'is_last': True, 'is_terminal': True, 'observation': {'hand_image': array([[[ 4, 4, 0],\n", + " [ 6, 5, 0],\n", + " [ 8, 6, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 193, 202],\n", + " [189, 193, 202],\n", + " [188, 192, 201],\n", + " ...,\n", + " [211, 224, 185],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[188, 192, 201],\n", + " [188, 192, 201],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[187, 191, 200],\n", + " [187, 191, 200],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]]], dtype=uint8), 'image': array([[[108, 101, 82],\n", + " [104, 97, 78],\n", + " [ 46, 72, 72],\n", + " ...,\n", + " [ 12, 19, 13],\n", + " [ 7, 15, 16],\n", + " [ 39, 47, 48]],\n", + "\n", + " [[115, 102, 78],\n", + " [119, 106, 82],\n", + " [ 61, 91, 83],\n", + " ...,\n", + " [ 16, 20, 15],\n", + " [ 23, 26, 29],\n", + " [ 51, 54, 57]],\n", + "\n", + " [[114, 90, 67],\n", + " [124, 100, 77],\n", + " [ 71, 102, 86],\n", + " ...,\n", + " [ 22, 25, 24],\n", + " [ 43, 43, 45],\n", + " [ 48, 48, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 72, 55],\n", + " [ 71, 77, 60],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 67, 96, 106],\n", + " [ 66, 98, 107],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 55, 61, 44],\n", + " ...,\n", + " [ 69, 98, 106],\n", + " [ 69, 98, 106],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 46, 52, 33],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [ 68, 97, 107],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0464277 , -1.5272526 , 2.3538566 , -2.5545247 , -1.5915655 ,\n", + " 3.390477 , 0.40496254, 0.14374796, -0.00899097, 0.75652575,\n", + " 0.64914 , 0.05783392, 0.05423374, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00038095, 0. , 0.00057143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00228571, 0.00685714, 0.00354286], dtype=float32)}, 'is_first': True, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 47, 33, 26],\n", + " [ 49, 35, 29],\n", + " [ 48, 34, 27],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 19, 12, 11],\n", + " [ 22, 16, 15]],\n", + "\n", + " [[ 48, 34, 27],\n", + " [ 48, 34, 27],\n", + " [ 47, 33, 26],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " [[ 48, 34, 27],\n", + " [ 48, 34, 27],\n", + " [ 47, 33, 26],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[207, 233, 169],\n", + " [207, 233, 169],\n", + " [207, 233, 169],\n", + " ...,\n", + " [158, 176, 198],\n", + " [159, 177, 199],\n", + " [159, 177, 199]],\n", + "\n", + " [[207, 233, 169],\n", + " [207, 233, 169],\n", + " [208, 234, 170],\n", + " ...,\n", + " [159, 177, 199],\n", + " [158, 176, 198],\n", + " [158, 176, 198]],\n", + "\n", + " [[207, 233, 169],\n", + " [208, 234, 170],\n", + " [208, 234, 170],\n", + " ...,\n", + " [158, 176, 198],\n", + " [157, 175, 197],\n", + " [159, 177, 199]]], dtype=uint8), 'image': array([[[112, 105, 76],\n", + " [114, 107, 78],\n", + " [ 53, 77, 65],\n", + " ...,\n", + " [ 98, 92, 94],\n", + " [ 99, 91, 90],\n", + " [ 99, 91, 90]],\n", + "\n", + " [[124, 105, 78],\n", + " [122, 103, 76],\n", + " [ 68, 93, 76],\n", + " ...,\n", + " [ 97, 92, 90],\n", + " [ 97, 92, 88],\n", + " [ 97, 92, 88]],\n", + "\n", + " [[119, 88, 61],\n", + " [129, 98, 71],\n", + " [ 79, 107, 82],\n", + " ...,\n", + " [ 97, 89, 86],\n", + " [ 99, 92, 86],\n", + " [ 96, 89, 83]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 74, 58],\n", + " [ 74, 77, 61],\n", + " [ 70, 75, 61],\n", + " ...,\n", + " [ 70, 95, 102],\n", + " [ 72, 97, 104],\n", + " [ 75, 100, 107]],\n", + "\n", + " [[ 61, 64, 46],\n", + " [ 60, 63, 45],\n", + " [ 58, 61, 43],\n", + " ...,\n", + " [ 72, 97, 102],\n", + " [ 71, 96, 101],\n", + " [ 73, 98, 103]],\n", + "\n", + " [[ 56, 58, 40],\n", + " [ 53, 55, 37],\n", + " [ 52, 55, 37],\n", + " ...,\n", + " [ 68, 96, 100],\n", + " [ 70, 95, 100],\n", + " [ 71, 96, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2969947 , -1.6309904 , 2.4249759 , -2.406821 , -1.5576328 ,\n", + " 2.7911022 , 0.36736205, 0.05681828, -0.01433784, 0.6336503 ,\n", + " 0.77331245, 0.01563 , 0.01519585, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00114286, 0. , 0.01580952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00954286, 0.01148571, 0.00508571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 49, 33, 26],\n", + " [ 51, 35, 29],\n", + " [ 52, 36, 30],\n", + " ...,\n", + " [ 18, 10, 12],\n", + " [ 20, 13, 14],\n", + " [ 20, 13, 14]],\n", + "\n", + " [[ 49, 33, 26],\n", + " [ 49, 33, 26],\n", + " [ 49, 33, 26],\n", + " ...,\n", + " [ 20, 13, 14],\n", + " [ 21, 14, 15],\n", + " [ 21, 14, 15]],\n", + "\n", + " [[ 47, 33, 26],\n", + " [ 47, 33, 26],\n", + " [ 49, 33, 26],\n", + " ...,\n", + " [ 20, 13, 14],\n", + " [ 21, 14, 15],\n", + " [ 21, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[209, 234, 173],\n", + " [209, 234, 173],\n", + " [208, 233, 172],\n", + " ...,\n", + " [158, 176, 198],\n", + " [159, 177, 199],\n", + " [159, 177, 199]],\n", + "\n", + " [[208, 233, 172],\n", + " [208, 233, 172],\n", + " [210, 235, 174],\n", + " ...,\n", + " [161, 178, 200],\n", + " [161, 178, 200],\n", + " [161, 178, 200]],\n", + "\n", + " [[208, 233, 172],\n", + " [209, 234, 173],\n", + " [211, 236, 175],\n", + " ...,\n", + " [161, 178, 200],\n", + " [161, 178, 200],\n", + " [161, 178, 200]]], dtype=uint8), 'image': array([[[116, 108, 82],\n", + " [111, 103, 77],\n", + " [ 49, 74, 67],\n", + " ...,\n", + " [ 97, 90, 82],\n", + " [ 99, 92, 84],\n", + " [ 99, 92, 84]],\n", + "\n", + " [[127, 110, 83],\n", + " [117, 100, 73],\n", + " [ 66, 94, 79],\n", + " ...,\n", + " [ 95, 88, 80],\n", + " [ 99, 92, 84],\n", + " [ 96, 89, 81]],\n", + "\n", + " [[120, 92, 64],\n", + " [127, 99, 71],\n", + " [ 81, 108, 86],\n", + " ...,\n", + " [ 95, 88, 78],\n", + " [ 92, 85, 75],\n", + " [ 89, 82, 72]],\n", + "\n", + " ...,\n", + "\n", + " [[ 74, 77, 61],\n", + " [ 79, 82, 66],\n", + " [ 75, 78, 60],\n", + " ...,\n", + " [ 72, 98, 98],\n", + " [ 72, 98, 98],\n", + " [ 72, 98, 98]],\n", + "\n", + " [[ 65, 69, 48],\n", + " [ 67, 71, 50],\n", + " [ 61, 65, 44],\n", + " ...,\n", + " [ 72, 98, 98],\n", + " [ 73, 97, 95],\n", + " [ 73, 97, 95]],\n", + "\n", + " [[ 59, 62, 41],\n", + " [ 58, 61, 40],\n", + " [ 51, 55, 34],\n", + " ...,\n", + " [ 70, 96, 96],\n", + " [ 72, 96, 96],\n", + " [ 73, 97, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2966354 , -1.6166404 , 2.418114 , -2.4144275 , -1.5576924 ,\n", + " 2.7917013 , 0.3713127 , 0.05632875, -0.01605012, 0.63372785,\n", + " 0.7732474 , 0.01564922, 0.01525001, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00095238, 0.01561905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01108571, 0.01057143, 0.00765714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 38, 29, 22],\n", + " [ 42, 33, 25],\n", + " [ 42, 33, 25],\n", + " ...,\n", + " [ 21, 15, 14],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " [[ 37, 28, 20],\n", + " [ 38, 29, 22],\n", + " [ 39, 30, 23],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 21, 15, 14],\n", + " [ 21, 15, 14]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 35, 28, 22],\n", + " [ 38, 29, 22],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 21, 15, 14],\n", + " [ 21, 15, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[199, 224, 170],\n", + " [198, 223, 169],\n", + " [196, 224, 167],\n", + " ...,\n", + " [159, 177, 199],\n", + " [159, 177, 199],\n", + " [161, 178, 200]],\n", + "\n", + " [[201, 226, 172],\n", + " [198, 223, 169],\n", + " [196, 224, 167],\n", + " ...,\n", + " [161, 178, 200],\n", + " [159, 177, 199],\n", + " [161, 178, 200]],\n", + "\n", + " [[201, 226, 172],\n", + " [201, 226, 172],\n", + " [197, 225, 168],\n", + " ...,\n", + " [159, 177, 199],\n", + " [158, 176, 198],\n", + " [159, 177, 199]]], dtype=uint8), 'image': array([[[119, 106, 66],\n", + " [116, 103, 63],\n", + " [ 52, 71, 59],\n", + " ...,\n", + " [ 84, 75, 68],\n", + " [ 83, 74, 67],\n", + " [ 80, 71, 64]],\n", + "\n", + " [[128, 108, 68],\n", + " [128, 108, 68],\n", + " [ 64, 89, 68],\n", + " ...,\n", + " [ 81, 71, 59],\n", + " [ 80, 71, 62],\n", + " [ 79, 70, 61]],\n", + "\n", + " [[120, 93, 58],\n", + " [130, 103, 68],\n", + " [ 80, 106, 80],\n", + " ...,\n", + " [ 78, 68, 54],\n", + " [ 76, 68, 54],\n", + " [ 78, 70, 56]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 75, 54],\n", + " [ 70, 73, 52],\n", + " [ 74, 74, 54],\n", + " ...,\n", + " [ 78, 101, 106],\n", + " [ 77, 100, 105],\n", + " [ 74, 97, 102]],\n", + "\n", + " [[ 63, 66, 43],\n", + " [ 63, 66, 43],\n", + " [ 61, 64, 41],\n", + " ...,\n", + " [ 72, 97, 102],\n", + " [ 72, 97, 102],\n", + " [ 71, 96, 101]],\n", + "\n", + " [[ 55, 55, 31],\n", + " [ 55, 55, 31],\n", + " [ 52, 55, 32],\n", + " ...,\n", + " [ 66, 95, 96],\n", + " [ 70, 95, 100],\n", + " [ 69, 94, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3029208 , -1.591381 , 2.4043546 , -2.4257529 , -1.5580639 ,\n", + " 2.7932708 , 0.37898293, 0.05273136, -0.01882746, 0.63674 ,\n", + " 0.7707726 , 0.0154203 , 0.01529887, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0192381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01171429, 0.00902857, 0.00897143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 16, 12, 8],\n", + " [ 17, 14, 10],\n", + " [ 15, 11, 7],\n", + " ...,\n", + " [ 16, 12, 10],\n", + " [ 18, 14, 12],\n", + " [ 19, 15, 14]],\n", + "\n", + " [[ 20, 14, 11],\n", + " [ 18, 12, 8],\n", + " [ 16, 12, 8],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14]],\n", + "\n", + " [[ 20, 14, 11],\n", + " [ 18, 12, 8],\n", + " [ 18, 12, 8],\n", + " ...,\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[211, 232, 185],\n", + " [210, 230, 184],\n", + " [210, 230, 184],\n", + " ...,\n", + " [159, 174, 197],\n", + " [160, 175, 198],\n", + " [160, 175, 198]],\n", + "\n", + " [[212, 233, 186],\n", + " [211, 232, 185],\n", + " [211, 232, 185],\n", + " ...,\n", + " [161, 177, 199],\n", + " [161, 177, 199],\n", + " [161, 177, 199]],\n", + "\n", + " [[212, 233, 186],\n", + " [212, 233, 186],\n", + " [212, 233, 186],\n", + " ...,\n", + " [161, 177, 199],\n", + " [160, 175, 198],\n", + " [161, 177, 199]]], dtype=uint8), 'image': array([[[111, 105, 81],\n", + " [108, 102, 78],\n", + " [ 49, 76, 71],\n", + " ...,\n", + " [ 69, 69, 47],\n", + " [ 70, 68, 49],\n", + " [ 70, 68, 49]],\n", + "\n", + " [[121, 105, 82],\n", + " [123, 107, 84],\n", + " [ 65, 92, 83],\n", + " ...,\n", + " [ 72, 71, 54],\n", + " [ 72, 67, 49],\n", + " [ 78, 73, 55]],\n", + "\n", + " [[117, 90, 68],\n", + " [129, 102, 80],\n", + " [ 78, 107, 89],\n", + " ...,\n", + " [ 75, 74, 59],\n", + " [ 76, 74, 55],\n", + " [ 81, 79, 60]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 75, 52],\n", + " [ 83, 86, 63],\n", + " [ 78, 81, 58],\n", + " ...,\n", + " [ 70, 100, 105],\n", + " [ 67, 99, 106],\n", + " [ 66, 98, 105]],\n", + "\n", + " [[ 60, 63, 40],\n", + " [ 56, 59, 36],\n", + " [ 57, 61, 40],\n", + " ...,\n", + " [ 68, 93, 100],\n", + " [ 68, 96, 102],\n", + " [ 68, 96, 102]],\n", + "\n", + " [[ 51, 54, 33],\n", + " [ 51, 54, 33],\n", + " [ 51, 54, 36],\n", + " ...,\n", + " [ 69, 94, 99],\n", + " [ 71, 97, 99],\n", + " [ 71, 97, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.314199 , -1.5623087 , 2.39123 , -2.4421766 , -1.557824 ,\n", + " 2.7923725 , 0.3873519 , 0.0471875 , -0.0229582 , 0.64069533,\n", + " 0.7674803 , 0.01556755, 0.01552759, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01411429, 0.00897143, 0.01274286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 20, 17, 13],\n", + " [ 18, 15, 11],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 17, 13, 11],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " [[ 19, 16, 12],\n", + " [ 18, 15, 11],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 17, 13, 11],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " [[ 21, 16, 10],\n", + " [ 20, 15, 9],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 230, 185],\n", + " [210, 230, 185],\n", + " [209, 228, 184],\n", + " ...,\n", + " [156, 172, 190],\n", + " [156, 172, 190],\n", + " [158, 175, 192]],\n", + "\n", + " [[210, 230, 185],\n", + " [210, 230, 185],\n", + " [211, 231, 187],\n", + " ...,\n", + " [158, 175, 192],\n", + " [158, 175, 192],\n", + " [158, 175, 192]],\n", + "\n", + " [[211, 231, 187],\n", + " [211, 231, 187],\n", + " [211, 231, 187],\n", + " ...,\n", + " [158, 175, 192],\n", + " [158, 175, 192],\n", + " [157, 174, 191]]], dtype=uint8), 'image': array([[[110, 105, 72],\n", + " [108, 103, 70],\n", + " [ 48, 77, 74],\n", + " ...,\n", + " [ 17, 20, 21],\n", + " [ 19, 26, 20],\n", + " [ 36, 43, 37]],\n", + "\n", + " [[118, 103, 73],\n", + " [121, 106, 76],\n", + " [ 55, 87, 76],\n", + " ...,\n", + " [ 13, 19, 17],\n", + " [ 36, 44, 31],\n", + " [ 47, 55, 42]],\n", + "\n", + " [[116, 89, 63],\n", + " [126, 99, 73],\n", + " [ 74, 106, 87],\n", + " ...,\n", + " [ 25, 34, 27],\n", + " [ 56, 65, 45],\n", + " [ 61, 70, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 53],\n", + " [ 75, 78, 60],\n", + " [ 73, 75, 64],\n", + " ...,\n", + " [ 67, 97, 100],\n", + " [ 68, 99, 99],\n", + " [ 67, 98, 98]],\n", + "\n", + " [[ 61, 65, 44],\n", + " [ 64, 68, 47],\n", + " [ 59, 61, 43],\n", + " ...,\n", + " [ 66, 96, 101],\n", + " [ 66, 96, 99],\n", + " [ 66, 96, 99]],\n", + "\n", + " [[ 60, 59, 42],\n", + " [ 57, 56, 39],\n", + " [ 49, 51, 33],\n", + " ...,\n", + " [ 65, 95, 100],\n", + " [ 61, 94, 98],\n", + " [ 61, 94, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3272011 , -1.5332474 , 2.3811233 , -2.4609396 , -1.5572494 ,\n", + " 2.7913055 , 0.3946254 , 0.04107245, -0.02822383, 0.6452304 ,\n", + " 0.7636719 , 0.01568247, 0.01539803, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.01790476], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01222857, 0.00857143, 0.01382857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 26, 18, 13],\n", + " [ 26, 18, 13],\n", + " [ 26, 18, 14],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 18, 13, 14],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 24, 17, 12],\n", + " [ 23, 16, 10],\n", + " [ 23, 16, 10],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 24, 17, 12],\n", + " [ 23, 16, 10],\n", + " [ 23, 16, 10],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[206, 228, 182],\n", + " [206, 228, 182],\n", + " [205, 227, 180],\n", + " ...,\n", + " [153, 170, 188],\n", + " [154, 171, 189],\n", + " [154, 171, 189]],\n", + "\n", + " [[207, 230, 183],\n", + " [207, 230, 183],\n", + " [207, 230, 183],\n", + " ...,\n", + " [153, 170, 188],\n", + " [153, 170, 188],\n", + " [153, 170, 188]],\n", + "\n", + " [[206, 228, 182],\n", + " [207, 230, 183],\n", + " [207, 230, 183],\n", + " ...,\n", + " [153, 170, 188],\n", + " [154, 171, 189],\n", + " [153, 170, 188]]], dtype=uint8), 'image': array([[[108, 102, 78],\n", + " [108, 102, 78],\n", + " [ 55, 82, 75],\n", + " ...,\n", + " [ 71, 79, 66],\n", + " [ 69, 77, 64],\n", + " [ 70, 78, 65]],\n", + "\n", + " [[111, 94, 76],\n", + " [123, 106, 88],\n", + " [ 66, 94, 81],\n", + " ...,\n", + " [ 69, 77, 66],\n", + " [ 67, 80, 67],\n", + " [ 67, 80, 67]],\n", + "\n", + " [[113, 85, 68],\n", + " [130, 102, 85],\n", + " [ 80, 108, 95],\n", + " ...,\n", + " [ 70, 78, 67],\n", + " [ 68, 83, 69],\n", + " [ 68, 83, 69]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 73, 45],\n", + " [ 72, 77, 49],\n", + " [ 69, 76, 50],\n", + " ...,\n", + " [ 63, 93, 98],\n", + " [ 68, 96, 102],\n", + " [ 68, 96, 102]],\n", + "\n", + " [[ 60, 65, 35],\n", + " [ 61, 66, 36],\n", + " [ 58, 63, 33],\n", + " ...,\n", + " [ 62, 92, 97],\n", + " [ 64, 89, 96],\n", + " [ 63, 88, 95]],\n", + "\n", + " [[ 57, 58, 31],\n", + " [ 56, 57, 30],\n", + " [ 53, 57, 27],\n", + " ...,\n", + " [ 59, 89, 94],\n", + " [ 62, 87, 94],\n", + " [ 59, 84, 91]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3431368 , -1.4991382 , 2.3722975 , -2.4858325 , -1.5566143 ,\n", + " 2.7856972 , 0.40199557, 0.03360351, -0.03550917, 0.6491278 ,\n", + " 0.76036435, 0.01573002, 0.01522294, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00019048, 0. , 0.01238095], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01257143, 0.01074286, 0.01348571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 3, 4, 2],\n", + " [ 6, 7, 4],\n", + " [ 9, 7, 5],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 17, 12, 13],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 9, 7, 5],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 8, 6, 4],\n", + " [ 9, 7, 5],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[203, 227, 182],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [158, 174, 187],\n", + " [161, 174, 188],\n", + " [161, 174, 188]],\n", + "\n", + " [[201, 226, 181],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [160, 176, 189],\n", + " [162, 176, 189],\n", + " [161, 174, 188]],\n", + "\n", + " [[203, 227, 182],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [161, 177, 190],\n", + " [162, 176, 189],\n", + " [161, 174, 188]]], dtype=uint8), 'image': array([[[115, 109, 85],\n", + " [112, 106, 82],\n", + " [ 56, 83, 78],\n", + " ...,\n", + " [ 80, 93, 88],\n", + " [ 77, 92, 87],\n", + " [ 78, 93, 88]],\n", + "\n", + " [[124, 108, 85],\n", + " [126, 110, 87],\n", + " [ 54, 85, 71],\n", + " ...,\n", + " [ 77, 92, 87],\n", + " [ 77, 92, 89],\n", + " [ 74, 89, 86]],\n", + "\n", + " [[116, 89, 67],\n", + " [137, 110, 88],\n", + " [ 69, 101, 82],\n", + " ...,\n", + " [ 74, 92, 86],\n", + " [ 72, 90, 84],\n", + " [ 69, 87, 81]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 72, 54],\n", + " [ 73, 75, 57],\n", + " [ 69, 70, 57],\n", + " ...,\n", + " [ 82, 89, 83],\n", + " [ 82, 87, 77],\n", + " [ 81, 86, 76]],\n", + "\n", + " [[ 60, 66, 49],\n", + " [ 62, 68, 51],\n", + " [ 59, 65, 48],\n", + " ...,\n", + " [ 82, 86, 79],\n", + " [ 84, 86, 77],\n", + " [ 84, 86, 77]],\n", + "\n", + " [[ 53, 58, 46],\n", + " [ 56, 61, 49],\n", + " [ 49, 55, 38],\n", + " ...,\n", + " [ 86, 87, 81],\n", + " [ 90, 88, 80],\n", + " [ 90, 88, 80]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3573966 , -1.4627742 , 2.364023 , -2.5137966 , -1.5562791 ,\n", + " 2.7826176 , 0.40912914, 0.02653525, -0.04385389, 0.6533296 ,\n", + " 0.756757 , 0.0157517 , 0.01520285, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00133333, -0.01009524, 0.01047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01257143, 0.01045714, 0.01451429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 34, 26, 21],\n", + " [ 33, 25, 20],\n", + " [ 29, 24, 20],\n", + " ...,\n", + " [ 19, 15, 14],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " [ 31, 24, 18],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " ...,\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [205, 226, 182],\n", + " ...,\n", + " [154, 171, 189],\n", + " [156, 172, 190],\n", + " [156, 172, 190]],\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [206, 228, 183],\n", + " ...,\n", + " [154, 171, 189],\n", + " [154, 171, 189],\n", + " [156, 172, 190]],\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [206, 228, 183],\n", + " ...,\n", + " [156, 172, 190],\n", + " [156, 172, 190],\n", + " [154, 171, 189]]], dtype=uint8), 'image': array([[[112, 105, 84],\n", + " [109, 102, 81],\n", + " [ 38, 66, 70],\n", + " ...,\n", + " [ 43, 54, 35],\n", + " [ 42, 52, 40],\n", + " [ 41, 51, 39]],\n", + "\n", + " [[122, 104, 82],\n", + " [123, 105, 83],\n", + " [ 61, 93, 86],\n", + " ...,\n", + " [ 43, 54, 33],\n", + " [ 40, 50, 36],\n", + " [ 40, 50, 36]],\n", + "\n", + " [[118, 91, 69],\n", + " [127, 100, 78],\n", + " [ 75, 106, 90],\n", + " ...,\n", + " [ 42, 53, 32],\n", + " [ 41, 49, 36],\n", + " [ 41, 49, 36]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 75, 52],\n", + " [ 69, 79, 56],\n", + " [ 64, 73, 55],\n", + " ...,\n", + " [ 92, 93, 80],\n", + " [ 97, 98, 87],\n", + " [ 95, 96, 85]],\n", + "\n", + " [[ 55, 66, 45],\n", + " [ 55, 66, 45],\n", + " [ 54, 65, 46],\n", + " ...,\n", + " [ 97, 94, 78],\n", + " [ 99, 97, 85],\n", + " [ 99, 97, 85]],\n", + "\n", + " [[ 54, 64, 48],\n", + " [ 48, 58, 42],\n", + " [ 47, 55, 40],\n", + " ...,\n", + " [ 99, 96, 80],\n", + " [ 99, 98, 83],\n", + " [101, 100, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3698246 , -1.4250783 , 2.3521204 , -2.5393493 , -1.5560521 ,\n", + " 2.7788906 , 0.41707364, 0.0197808 , -0.05207061, 0.6565796 ,\n", + " 0.75394094, 0.01570738, 0.0151506 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00933333, 0.008 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01325714, 0.01108571, 0.0148 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 33, 26, 18],\n", + " [ 33, 26, 18],\n", + " [ 33, 26, 18],\n", + " ...,\n", + " [ 21, 17, 16],\n", + " [ 21, 17, 17],\n", + " [ 20, 16, 16]],\n", + "\n", + " [[ 30, 23, 17],\n", + " [ 30, 23, 17],\n", + " [ 29, 22, 16],\n", + " ...,\n", + " [ 20, 16, 15],\n", + " [ 21, 17, 17],\n", + " [ 21, 17, 17]],\n", + "\n", + " [[ 27, 22, 16],\n", + " [ 26, 21, 15],\n", + " [ 26, 21, 15],\n", + " ...,\n", + " [ 20, 16, 16],\n", + " [ 21, 17, 17],\n", + " [ 21, 17, 17]],\n", + "\n", + " ...,\n", + "\n", + " [[205, 229, 184],\n", + " [205, 229, 184],\n", + " [204, 228, 183],\n", + " ...,\n", + " [164, 175, 189],\n", + " [165, 176, 190],\n", + " [165, 176, 190]],\n", + "\n", + " [[205, 229, 184],\n", + " [205, 229, 184],\n", + " [205, 229, 184],\n", + " ...,\n", + " [165, 176, 190],\n", + " [165, 176, 190],\n", + " [165, 176, 190]],\n", + "\n", + " [[205, 229, 184],\n", + " [206, 230, 185],\n", + " [206, 230, 185],\n", + " ...,\n", + " [164, 175, 189],\n", + " [165, 176, 190],\n", + " [164, 175, 189]]], dtype=uint8), 'image': array([[[110, 107, 75],\n", + " [111, 108, 76],\n", + " [ 41, 73, 64],\n", + " ...,\n", + " [ 76, 50, 50],\n", + " [ 80, 51, 41],\n", + " [ 84, 55, 45]],\n", + "\n", + " [[121, 106, 78],\n", + " [126, 111, 83],\n", + " [ 59, 92, 77],\n", + " ...,\n", + " [ 80, 54, 52],\n", + " [ 84, 55, 43],\n", + " [ 84, 55, 43]],\n", + "\n", + " [[116, 89, 65],\n", + " [130, 103, 79],\n", + " [ 73, 107, 87],\n", + " ...,\n", + " [ 84, 57, 51],\n", + " [ 86, 56, 49],\n", + " [ 87, 57, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 74, 73, 58],\n", + " [ 76, 75, 60],\n", + " [ 72, 71, 56],\n", + " ...,\n", + " [104, 103, 88],\n", + " [104, 103, 88],\n", + " [105, 104, 89]],\n", + "\n", + " [[ 63, 65, 47],\n", + " [ 65, 67, 49],\n", + " [ 62, 64, 48],\n", + " ...,\n", + " [105, 104, 87],\n", + " [106, 108, 92],\n", + " [109, 111, 95]],\n", + "\n", + " [[ 56, 58, 40],\n", + " [ 54, 56, 38],\n", + " [ 49, 51, 33],\n", + " ...,\n", + " [105, 108, 90],\n", + " [103, 108, 94],\n", + " [103, 108, 94]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3830068 , -1.3866261 , 2.3424206 , -2.5729144 , -1.5544709 ,\n", + " 2.7728264 , 0.42537263, 0.01276265, -0.0608653 , 0.65915596,\n", + " 0.7516113 , 0.01741757, 0.01704437, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.0047619 , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01417143, 0.01262857, 0.01457143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[145, 165, 180],\n", + " [147, 167, 182],\n", + " [147, 167, 182],\n", + " ...,\n", + " [ 21, 18, 19],\n", + " [ 21, 17, 17],\n", + " [ 23, 18, 19]],\n", + "\n", + " [[147, 167, 182],\n", + " [147, 167, 182],\n", + " [147, 167, 182],\n", + " ...,\n", + " [ 19, 17, 17],\n", + " [ 23, 18, 19],\n", + " [ 23, 18, 19]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 21, 17, 17],\n", + " [ 23, 18, 19],\n", + " [ 23, 18, 19]],\n", + "\n", + " ...,\n", + "\n", + " [[208, 230, 185],\n", + " [208, 230, 185],\n", + " [207, 229, 184],\n", + " ...,\n", + " [158, 173, 189],\n", + " [161, 174, 190],\n", + " [161, 174, 190]],\n", + "\n", + " [[208, 230, 185],\n", + " [208, 230, 185],\n", + " [208, 230, 185],\n", + " ...,\n", + " [158, 173, 189],\n", + " [160, 172, 189],\n", + " [160, 172, 189]],\n", + "\n", + " [[207, 229, 184],\n", + " [208, 230, 185],\n", + " [209, 231, 187],\n", + " ...,\n", + " [158, 173, 189],\n", + " [160, 172, 189],\n", + " [160, 172, 189]]], dtype=uint8), 'image': array([[[111, 104, 73],\n", + " [111, 104, 73],\n", + " [ 48, 75, 68],\n", + " ...,\n", + " [ 79, 86, 90],\n", + " [ 81, 88, 94],\n", + " [ 82, 89, 95]],\n", + "\n", + " [[120, 101, 72],\n", + " [118, 99, 70],\n", + " [ 66, 94, 79],\n", + " ...,\n", + " [ 78, 83, 88],\n", + " [ 85, 87, 93],\n", + " [ 86, 88, 94]],\n", + "\n", + " [[118, 86, 62],\n", + " [129, 97, 73],\n", + " [ 79, 108, 86],\n", + " ...,\n", + " [ 85, 87, 95],\n", + " [ 85, 87, 93],\n", + " [ 87, 89, 95]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 52],\n", + " [ 72, 79, 57],\n", + " [ 71, 77, 58],\n", + " ...,\n", + " [113, 111, 99],\n", + " [113, 111, 101],\n", + " [111, 109, 99]],\n", + "\n", + " [[ 59, 66, 44],\n", + " [ 58, 65, 43],\n", + " [ 56, 62, 43],\n", + " ...,\n", + " [114, 112, 100],\n", + " [113, 111, 99],\n", + " [114, 112, 100]],\n", + "\n", + " [[ 52, 55, 39],\n", + " [ 51, 54, 38],\n", + " [ 50, 53, 37],\n", + " ...,\n", + " [114, 112, 100],\n", + " [114, 112, 100],\n", + " [115, 113, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.396764 , -1.3478969 , 2.3317156 , -2.606684 , -1.5517172 ,\n", + " 2.7645218 , 0.4339046 , 0.00549856, -0.06976949, 0.6610677 ,\n", + " 0.74982053, 0.01980009, 0.01914801, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00438095, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01577143, 0.01262857, 0.01222857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[156, 168, 180],\n", + " [155, 167, 178],\n", + " [153, 167, 178],\n", + " ...,\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21]],\n", + "\n", + " [[155, 167, 178],\n", + " [155, 167, 178],\n", + " [151, 166, 177],\n", + " ...,\n", + " [ 20, 20, 20],\n", + " [ 20, 20, 20],\n", + " [ 20, 20, 20]],\n", + "\n", + " [[153, 167, 178],\n", + " [151, 166, 177],\n", + " [151, 166, 177],\n", + " ...,\n", + " [ 20, 20, 20],\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21]],\n", + "\n", + " ...,\n", + "\n", + " [[204, 229, 182],\n", + " [204, 229, 182],\n", + " [203, 228, 180],\n", + " ...,\n", + " [148, 168, 190],\n", + " [148, 168, 190],\n", + " [148, 168, 190]],\n", + "\n", + " [[204, 229, 182],\n", + " [203, 228, 180],\n", + " [203, 228, 180],\n", + " ...,\n", + " [150, 170, 192],\n", + " [149, 169, 191],\n", + " [147, 167, 189]],\n", + "\n", + " [[204, 229, 182],\n", + " [204, 229, 182],\n", + " [203, 228, 180],\n", + " ...,\n", + " [150, 170, 192],\n", + " [150, 170, 192],\n", + " [150, 170, 192]]], dtype=uint8), 'image': array([[[111, 103, 89],\n", + " [107, 99, 85],\n", + " [ 44, 74, 79],\n", + " ...,\n", + " [106, 102, 105],\n", + " [ 99, 102, 103],\n", + " [ 99, 102, 103]],\n", + "\n", + " [[123, 107, 84],\n", + " [122, 106, 83],\n", + " [ 61, 93, 86],\n", + " ...,\n", + " [104, 104, 100],\n", + " [102, 102, 102],\n", + " [101, 101, 101]],\n", + "\n", + " [[124, 95, 72],\n", + " [130, 101, 78],\n", + " [ 73, 104, 88],\n", + " ...,\n", + " [105, 102, 97],\n", + " [102, 102, 96],\n", + " [102, 102, 96]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 52],\n", + " [ 74, 77, 59],\n", + " [ 69, 73, 52],\n", + " ...,\n", + " [117, 119, 103],\n", + " [119, 118, 101],\n", + " [113, 112, 95]],\n", + "\n", + " [[ 59, 62, 44],\n", + " [ 60, 63, 45],\n", + " [ 60, 63, 45],\n", + " ...,\n", + " [115, 117, 99],\n", + " [118, 120, 104],\n", + " [119, 121, 105]],\n", + "\n", + " [[ 51, 54, 36],\n", + " [ 53, 56, 38],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [117, 119, 103],\n", + " [119, 121, 105],\n", + " [119, 121, 105]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4100306e+00, -1.3073558e+00, 2.3148079e+00, -2.6330740e+00,\n", + " -1.5499452e+00, 2.7541919e+00, 4.4335869e-01, -2.3262841e-03,\n", + " -7.8644946e-02, 6.6207409e-01, 7.4887121e-01, 2.1015564e-02,\n", + " 2.0203222e-02, 0.0000000e+00, 0.0000000e+00], dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00152381, -0.00171429, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01742857, 0.01297143, 0.01085714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[147, 166, 183],\n", + " [147, 166, 183],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[148, 167, 184],\n", + " [148, 167, 184],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[148, 167, 184],\n", + " [148, 167, 184],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 228, 182],\n", + " [210, 228, 182],\n", + " [209, 227, 179],\n", + " ...,\n", + " [156, 172, 192],\n", + " [156, 172, 192],\n", + " [156, 172, 192]],\n", + "\n", + " [[211, 229, 183],\n", + " [211, 229, 183],\n", + " [210, 228, 180],\n", + " ...,\n", + " [158, 174, 194],\n", + " [157, 173, 193],\n", + " [157, 173, 193]],\n", + "\n", + " [[211, 229, 183],\n", + " [211, 229, 183],\n", + " [211, 230, 181],\n", + " ...,\n", + " [156, 172, 192],\n", + " [156, 172, 192],\n", + " [154, 170, 191]]], dtype=uint8), 'image': array([[[110, 111, 82],\n", + " [104, 105, 76],\n", + " [ 35, 70, 73],\n", + " ...,\n", + " [111, 113, 102],\n", + " [115, 116, 110],\n", + " [115, 116, 110]],\n", + "\n", + " [[124, 112, 81],\n", + " [119, 107, 76],\n", + " [ 49, 84, 76],\n", + " ...,\n", + " [111, 113, 102],\n", + " [114, 115, 109],\n", + " [114, 115, 109]],\n", + "\n", + " [[121, 98, 66],\n", + " [123, 100, 68],\n", + " [ 74, 108, 88],\n", + " ...,\n", + " [110, 112, 103],\n", + " [115, 116, 110],\n", + " [116, 117, 111]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 48],\n", + " [ 71, 78, 52],\n", + " [ 65, 74, 54],\n", + " ...,\n", + " [122, 123, 112],\n", + " [121, 123, 112],\n", + " [121, 123, 112]],\n", + "\n", + " [[ 60, 68, 39],\n", + " [ 60, 68, 39],\n", + " [ 52, 64, 40],\n", + " ...,\n", + " [120, 121, 108],\n", + " [124, 125, 112],\n", + " [124, 125, 112]],\n", + "\n", + " [[ 55, 61, 42],\n", + " [ 51, 57, 38],\n", + " [ 44, 53, 35],\n", + " ...,\n", + " [122, 123, 110],\n", + " [122, 123, 110],\n", + " [122, 123, 110]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4241116 , -1.2713832 , 2.294727 , -2.6514275 , -1.5482687 ,\n", + " 2.7406254 , 0.4530185 , -0.01090289, -0.08589576, 0.66217947,\n", + " 0.74872 , 0.02204888, 0.02122683, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00380952, -0.0007619 , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01782857, 0.01325714, 0.01085714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[149, 166, 182],\n", + " [149, 166, 182],\n", + " [151, 166, 182],\n", + " ...,\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22],\n", + " [ 25, 23, 23]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 24, 23, 21],\n", + " [ 24, 22, 22],\n", + " [ 25, 23, 23]],\n", + "\n", + " ...,\n", + "\n", + " [[212, 228, 180],\n", + " [212, 228, 180],\n", + " [211, 226, 180],\n", + " ...,\n", + " [159, 173, 191],\n", + " [159, 173, 191],\n", + " [159, 173, 191]],\n", + "\n", + " [[213, 229, 181],\n", + " [212, 228, 180],\n", + " [213, 228, 183],\n", + " ...,\n", + " [158, 172, 190],\n", + " [158, 172, 190],\n", + " [158, 172, 190]],\n", + "\n", + " [[213, 229, 181],\n", + " [213, 229, 181],\n", + " [213, 228, 183],\n", + " ...,\n", + " [156, 171, 189],\n", + " [158, 172, 190],\n", + " [158, 172, 190]]], dtype=uint8), 'image': array([[[105, 107, 73],\n", + " [107, 109, 75],\n", + " [ 37, 72, 64],\n", + " ...,\n", + " [183, 184, 175],\n", + " [171, 167, 170],\n", + " [167, 163, 166]],\n", + "\n", + " [[116, 105, 71],\n", + " [120, 109, 75],\n", + " [ 53, 86, 71],\n", + " ...,\n", + " [179, 176, 173],\n", + " [170, 166, 169],\n", + " [163, 159, 162]],\n", + "\n", + " [[124, 101, 71],\n", + " [124, 101, 71],\n", + " [ 75, 107, 88],\n", + " ...,\n", + " [179, 173, 175],\n", + " [167, 166, 168],\n", + " [160, 159, 161]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 73, 54],\n", + " [ 72, 78, 59],\n", + " [ 70, 76, 59],\n", + " ...,\n", + " [126, 127, 114],\n", + " [126, 127, 116],\n", + " [123, 124, 113]],\n", + "\n", + " [[ 58, 64, 45],\n", + " [ 57, 63, 44],\n", + " [ 53, 62, 42],\n", + " ...,\n", + " [127, 126, 111],\n", + " [127, 125, 113],\n", + " [128, 126, 114]],\n", + "\n", + " [[ 54, 57, 39],\n", + " [ 51, 54, 36],\n", + " [ 44, 53, 33],\n", + " ...,\n", + " [129, 128, 113],\n", + " [127, 128, 115],\n", + " [127, 128, 115]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4396536 , -1.2382988 , 2.2719407 , -2.66238 , -1.5463403 ,\n", + " 2.7252848 , 0.46257424, -0.020392 , -0.09226454, 0.66218996,\n", + " 0.74868333, 0.02275231, 0.02144971, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00285714, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01782857, 0.01342857, 0.01022857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[137, 159, 181],\n", + " [137, 159, 181],\n", + " [138, 160, 182],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 20, 18],\n", + " [ 28, 24, 23]],\n", + "\n", + " [[137, 159, 181],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[137, 159, 181],\n", + " [137, 159, 181],\n", + " [137, 159, 181],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22]],\n", + "\n", + " ...,\n", + "\n", + " [[211, 227, 179],\n", + " [211, 227, 179],\n", + " [211, 227, 179],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 168, 190],\n", + " [150, 168, 190]],\n", + "\n", + " [[211, 227, 179],\n", + " [211, 227, 179],\n", + " [211, 227, 179],\n", + " ...,\n", + " [151, 169, 191],\n", + " [149, 166, 189],\n", + " [149, 166, 189]],\n", + "\n", + " [[210, 226, 178],\n", + " [210, 226, 178],\n", + " [211, 227, 179],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 168, 190],\n", + " [148, 165, 187]]], dtype=uint8), 'image': array([[[111, 106, 75],\n", + " [107, 102, 71],\n", + " [ 53, 78, 69],\n", + " ...,\n", + " [162, 162, 162],\n", + " [167, 167, 169],\n", + " [173, 173, 175]],\n", + "\n", + " [[119, 104, 74],\n", + " [121, 106, 76],\n", + " [ 65, 92, 74],\n", + " ...,\n", + " [161, 161, 161],\n", + " [167, 167, 167],\n", + " [175, 175, 175]],\n", + "\n", + " [[111, 88, 58],\n", + " [125, 102, 72],\n", + " [ 79, 106, 84],\n", + " ...,\n", + " [159, 162, 161],\n", + " [168, 168, 168],\n", + " [174, 174, 174]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 73, 44],\n", + " [ 70, 78, 49],\n", + " [ 69, 76, 52],\n", + " ...,\n", + " [131, 132, 121],\n", + " [134, 132, 122],\n", + " [133, 131, 121]],\n", + "\n", + " [[ 60, 68, 39],\n", + " [ 58, 66, 37],\n", + " [ 56, 66, 41],\n", + " ...,\n", + " [133, 131, 119],\n", + " [136, 133, 119],\n", + " [139, 136, 122]],\n", + "\n", + " [[ 55, 59, 34],\n", + " [ 50, 54, 29],\n", + " [ 46, 56, 33],\n", + " ...,\n", + " [137, 134, 118],\n", + " [141, 135, 120],\n", + " [139, 133, 118]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4565248 , -1.2037634 , 2.2444725 , -2.6683176 , -1.5432638 ,\n", + " 2.7085657 , 0.47299433, -0.03085847, -0.0988073 , 0.6621888 ,\n", + " 0.74867994, 0.02358894, 0.0206851 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00380952, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01788571, 0.01325714, 0.01051429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[143, 159, 179],\n", + " [145, 161, 181],\n", + " [144, 160, 180],\n", + " ...,\n", + " [ 27, 23, 22],\n", + " [ 27, 23, 22],\n", + " [ 27, 23, 22]],\n", + "\n", + " [[143, 159, 179],\n", + " [144, 160, 180],\n", + " [143, 159, 179],\n", + " ...,\n", + " [ 24, 20, 18],\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19]],\n", + "\n", + " [[141, 159, 179],\n", + " [141, 159, 179],\n", + " [141, 159, 179],\n", + " ...,\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 227, 176],\n", + " [210, 227, 176],\n", + " [208, 225, 175],\n", + " ...,\n", + " [147, 164, 186],\n", + " [150, 165, 187],\n", + " [150, 165, 187]],\n", + "\n", + " [[208, 225, 175],\n", + " [208, 225, 175],\n", + " [208, 225, 175],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 165, 187],\n", + " [150, 165, 187]],\n", + "\n", + " [[207, 224, 174],\n", + " [208, 225, 175],\n", + " [208, 225, 175],\n", + " ...,\n", + " [149, 166, 189],\n", + " [149, 164, 186],\n", + " [150, 165, 187]]], dtype=uint8), 'image': array([[[108, 104, 68],\n", + " [109, 105, 69],\n", + " [ 45, 71, 56],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " [[120, 103, 66],\n", + " [125, 108, 71],\n", + " [ 66, 93, 71],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " [[119, 89, 57],\n", + " [128, 98, 66],\n", + " [ 80, 108, 81],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 73, 50],\n", + " [ 76, 80, 57],\n", + " [ 73, 76, 58],\n", + " ...,\n", + " [137, 133, 124],\n", + " [141, 134, 124],\n", + " [137, 130, 120]],\n", + "\n", + " [[ 59, 62, 37],\n", + " [ 60, 63, 38],\n", + " [ 60, 63, 38],\n", + " ...,\n", + " [136, 134, 124],\n", + " [136, 134, 124],\n", + " [138, 136, 126]],\n", + "\n", + " [[ 53, 56, 33],\n", + " [ 48, 51, 28],\n", + " [ 48, 51, 26],\n", + " ...,\n", + " [136, 137, 126],\n", + " [139, 137, 129],\n", + " [140, 138, 130]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.471241 , -1.1722492 , 2.2166338 , -2.6705124 , -1.5408925 ,\n", + " 2.6938715 , 0.4829263 , -0.04071204, -0.10468176, 0.6621572 ,\n", + " 0.74871343, 0.02410153, 0.0198809 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0047619, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01811429, 0.01285714, 0.01 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[136, 158, 179],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 21, 19, 17],\n", + " [ 21, 19, 17],\n", + " [ 23, 21, 19]],\n", + "\n", + " [[137, 159, 181],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18],\n", + " [ 23, 21, 19]],\n", + "\n", + " [[138, 161, 180],\n", + " [138, 161, 180],\n", + " [137, 160, 179],\n", + " ...,\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[204, 221, 170],\n", + " [204, 221, 170],\n", + " [203, 220, 169],\n", + " ...,\n", + " [141, 158, 181],\n", + " [142, 159, 182],\n", + " [142, 159, 182]],\n", + "\n", + " [[204, 221, 170],\n", + " [203, 220, 169],\n", + " [203, 220, 169],\n", + " ...,\n", + " [142, 159, 182],\n", + " [142, 159, 182],\n", + " [141, 158, 181]],\n", + "\n", + " [[204, 221, 170],\n", + " [203, 220, 169],\n", + " [204, 221, 170],\n", + " ...,\n", + " [141, 158, 181],\n", + " [142, 159, 182],\n", + " [141, 158, 181]]], dtype=uint8), 'image': array([[[107, 101, 63],\n", + " [109, 103, 65],\n", + " [ 47, 75, 62],\n", + " ...,\n", + " [155, 155, 157],\n", + " [155, 155, 151],\n", + " [156, 156, 152]],\n", + "\n", + " [[118, 102, 68],\n", + " [122, 106, 72],\n", + " [ 57, 86, 68],\n", + " ...,\n", + " [157, 157, 159],\n", + " [156, 156, 152],\n", + " [156, 156, 152]],\n", + "\n", + " [[123, 94, 71],\n", + " [125, 96, 73],\n", + " [ 66, 98, 77],\n", + " ...,\n", + " [158, 157, 159],\n", + " [156, 156, 152],\n", + " [156, 156, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 70, 55],\n", + " [ 77, 76, 61],\n", + " [ 76, 78, 60],\n", + " ...,\n", + " [140, 141, 130],\n", + " [140, 141, 128],\n", + " [134, 135, 122]],\n", + "\n", + " [[ 63, 62, 45],\n", + " [ 65, 64, 47],\n", + " [ 58, 57, 42],\n", + " ...,\n", + " [143, 141, 133],\n", + " [140, 141, 128],\n", + " [137, 138, 125]],\n", + "\n", + " [[ 56, 53, 39],\n", + " [ 57, 54, 40],\n", + " [ 53, 51, 39],\n", + " ...,\n", + " [142, 138, 129],\n", + " [142, 140, 128],\n", + " [143, 141, 129]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4855387 , -1.1409982 , 2.1876225 , -2.6713645 , -1.5381626 ,\n", + " 2.679764 , 0.49294764, -0.05065339, -0.11059836, 0.66218203,\n", + " 0.74868876, 0.02483457, 0.0190603 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00533333, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01862857, 0.01217143, 0.00902857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 23, 19, 17]],\n", + "\n", + " [[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " [[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[199, 216, 166],\n", + " [199, 216, 166],\n", + " [198, 215, 164],\n", + " ...,\n", + " [135, 150, 172],\n", + " [136, 151, 174],\n", + " [136, 151, 174]],\n", + "\n", + " [[198, 215, 164],\n", + " [198, 215, 164],\n", + " [198, 215, 164],\n", + " ...,\n", + " [136, 151, 174],\n", + " [136, 151, 174],\n", + " [136, 151, 174]],\n", + "\n", + " [[198, 215, 164],\n", + " [198, 215, 164],\n", + " [198, 215, 164],\n", + " ...,\n", + " [136, 151, 174],\n", + " [136, 151, 174],\n", + " [136, 151, 174]]], dtype=uint8), 'image': array([[[113, 102, 79],\n", + " [112, 101, 78],\n", + " [ 54, 74, 69],\n", + " ...,\n", + " [224, 220, 209],\n", + " [224, 220, 211],\n", + " [225, 221, 212]],\n", + "\n", + " [[120, 101, 76],\n", + " [118, 99, 74],\n", + " [ 61, 85, 71],\n", + " ...,\n", + " [224, 220, 211],\n", + " [226, 219, 211],\n", + " [228, 221, 213]],\n", + "\n", + " [[116, 84, 62],\n", + " [128, 96, 74],\n", + " [ 76, 101, 82],\n", + " ...,\n", + " [224, 220, 211],\n", + " [226, 222, 213],\n", + " [227, 223, 214]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 73, 50],\n", + " [ 73, 76, 53],\n", + " [ 69, 72, 49],\n", + " ...,\n", + " [140, 140, 134],\n", + " [139, 139, 133],\n", + " [140, 140, 134]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 58, 62, 37],\n", + " [ 59, 63, 38],\n", + " ...,\n", + " [142, 140, 132],\n", + " [141, 138, 133],\n", + " [145, 142, 137]],\n", + "\n", + " [[ 52, 55, 32],\n", + " [ 49, 52, 29],\n", + " [ 48, 52, 31],\n", + " ...,\n", + " [145, 143, 133],\n", + " [142, 137, 133],\n", + " [146, 141, 137]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.500686 , -1.1089195 , 2.1561332 , -2.6699607 , -1.5347141 ,\n", + " 2.6646018 , 0.5029847 , -0.06102105, -0.11659307, 0.6621161 ,\n", + " 0.7487434 , 0.02574205, 0.0179702 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00761905, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01902857, 0.0112 , 0.00971429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[118, 147, 169],\n", + " [118, 147, 169],\n", + " [119, 148, 170],\n", + " ...,\n", + " [ 18, 17, 15],\n", + " [ 20, 16, 15],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[119, 145, 168],\n", + " [118, 144, 167],\n", + " [117, 145, 168],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 20, 16, 15],\n", + " [ 23, 19, 17]],\n", + "\n", + " [[118, 144, 167],\n", + " [118, 144, 167],\n", + " [117, 145, 168],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[177, 172, 159],\n", + " [178, 173, 160],\n", + " [179, 177, 157],\n", + " ...,\n", + " [138, 151, 174],\n", + " [138, 151, 174],\n", + " [138, 151, 174]],\n", + "\n", + " [[176, 171, 158],\n", + " [177, 172, 159],\n", + " [178, 176, 156],\n", + " ...,\n", + " [139, 152, 175],\n", + " [138, 151, 174],\n", + " [139, 152, 175]],\n", + "\n", + " [[173, 169, 156],\n", + " [175, 170, 157],\n", + " [177, 175, 154],\n", + " ...,\n", + " [140, 153, 176],\n", + " [140, 153, 176],\n", + " [141, 154, 177]]], dtype=uint8), 'image': array([[[107, 106, 78],\n", + " [101, 100, 72],\n", + " [ 36, 74, 57],\n", + " ...,\n", + " [207, 203, 192],\n", + " [198, 190, 187],\n", + " [186, 178, 175]],\n", + "\n", + " [[120, 108, 81],\n", + " [110, 98, 71],\n", + " [ 58, 95, 72],\n", + " ...,\n", + " [205, 201, 192],\n", + " [198, 192, 194],\n", + " [186, 180, 182]],\n", + "\n", + " [[108, 84, 57],\n", + " [122, 98, 71],\n", + " [ 72, 106, 82],\n", + " ...,\n", + " [207, 199, 196],\n", + " [197, 191, 195],\n", + " [186, 180, 184]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 53],\n", + " [ 72, 75, 57],\n", + " [ 70, 72, 56],\n", + " ...,\n", + " [146, 142, 133],\n", + " [144, 140, 131],\n", + " [142, 138, 129]],\n", + "\n", + " [[ 60, 64, 43],\n", + " [ 60, 64, 43],\n", + " [ 59, 63, 42],\n", + " ...,\n", + " [148, 141, 133],\n", + " [150, 143, 135],\n", + " [148, 141, 133]],\n", + "\n", + " [[ 54, 57, 36],\n", + " [ 54, 57, 36],\n", + " [ 48, 51, 33],\n", + " ...,\n", + " [149, 142, 134],\n", + " [148, 141, 133],\n", + " [149, 142, 134]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5155468 , -1.0804976 , 2.1259022 , -2.6659186 , -1.5311936 ,\n", + " 2.6497624 , 0.51242495, -0.07151319, -0.12196171, 0.66212773,\n", + " 0.74872553, 0.02683827, 0.01663197, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00590476, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01891429, 0.01205714, 0.01451429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 150, 173],\n", + " [120, 149, 172],\n", + " [121, 150, 173],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 16, 14, 14],\n", + " [ 17, 15, 15]],\n", + "\n", + " [[119, 148, 170],\n", + " [119, 148, 170],\n", + " [118, 147, 169],\n", + " ...,\n", + " [ 15, 12, 13],\n", + " [ 16, 14, 14],\n", + " [ 16, 14, 14]],\n", + "\n", + " [[121, 147, 170],\n", + " [121, 147, 170],\n", + " [120, 146, 169],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 160, 168],\n", + " [154, 160, 168],\n", + " [154, 159, 170],\n", + " ...,\n", + " [152, 157, 175],\n", + " [152, 157, 175],\n", + " [152, 157, 175]],\n", + "\n", + " [[154, 160, 168],\n", + " [153, 159, 167],\n", + " [154, 159, 170],\n", + " ...,\n", + " [152, 157, 175],\n", + " [152, 157, 175],\n", + " [152, 157, 175]],\n", + "\n", + " [[154, 160, 168],\n", + " [154, 160, 168],\n", + " [153, 158, 169],\n", + " ...,\n", + " [151, 156, 173],\n", + " [151, 156, 173],\n", + " [151, 156, 173]]], dtype=uint8), 'image': array([[[111, 100, 77],\n", + " [113, 102, 79],\n", + " [ 51, 71, 68],\n", + " ...,\n", + " [137, 132, 128],\n", + " [136, 136, 130],\n", + " [138, 138, 132]],\n", + "\n", + " [[120, 101, 76],\n", + " [124, 105, 80],\n", + " [ 62, 86, 74],\n", + " ...,\n", + " [139, 136, 131],\n", + " [138, 135, 130],\n", + " [140, 137, 132]],\n", + "\n", + " [[120, 86, 62],\n", + " [130, 96, 72],\n", + " [ 80, 102, 84],\n", + " ...,\n", + " [138, 135, 130],\n", + " [140, 137, 132],\n", + " [141, 138, 133]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 72, 49],\n", + " [ 74, 77, 54],\n", + " [ 73, 76, 55],\n", + " ...,\n", + " [148, 144, 137],\n", + " [148, 144, 135],\n", + " [143, 139, 130]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 60, 64, 39],\n", + " [ 59, 62, 39],\n", + " ...,\n", + " [148, 144, 135],\n", + " [149, 145, 136],\n", + " [147, 143, 134]],\n", + "\n", + " [[ 53, 56, 31],\n", + " [ 52, 55, 30],\n", + " [ 45, 49, 24],\n", + " ...,\n", + " [149, 145, 136],\n", + " [149, 145, 136],\n", + " [149, 145, 136]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5310063 , -1.052627 , 2.096078 , -2.660988 , -1.5265115 ,\n", + " 2.6343114 , 0.5215374 , -0.08259036, -0.12778392, 0.66204727,\n", + " 0.74878216, 0.0282663 , 0.01482305, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00361905, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01805714, 0.01268571, 0.01908571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 145, 168],\n", + " [120, 144, 167],\n", + " [120, 144, 167],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 19, 15, 14],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[120, 144, 167],\n", + " [120, 144, 167],\n", + " [118, 142, 166],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 19, 15, 14],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[120, 144, 167],\n", + " [120, 144, 167],\n", + " [118, 142, 166],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 157, 175],\n", + " [154, 157, 175],\n", + " [154, 158, 173],\n", + " ...,\n", + " [150, 157, 176],\n", + " [150, 157, 176],\n", + " [150, 157, 176]],\n", + "\n", + " [[150, 153, 171],\n", + " [150, 153, 171],\n", + " [151, 155, 171],\n", + " ...,\n", + " [151, 158, 177],\n", + " [151, 158, 177],\n", + " [150, 157, 176]],\n", + "\n", + " [[147, 150, 168],\n", + " [148, 151, 169],\n", + " [149, 153, 168],\n", + " ...,\n", + " [152, 159, 178],\n", + " [151, 158, 177],\n", + " [151, 158, 177]]], dtype=uint8), 'image': array([[[113, 102, 79],\n", + " [107, 96, 73],\n", + " [ 49, 73, 71],\n", + " ...,\n", + " [159, 158, 158],\n", + " [158, 157, 161],\n", + " [158, 157, 161]],\n", + "\n", + " [[118, 99, 74],\n", + " [119, 100, 75],\n", + " [ 61, 86, 77],\n", + " ...,\n", + " [159, 159, 161],\n", + " [157, 157, 161],\n", + " [154, 154, 158]],\n", + "\n", + " [[109, 77, 53],\n", + " [135, 103, 79],\n", + " [ 77, 104, 86],\n", + " ...,\n", + " [158, 157, 164],\n", + " [152, 151, 160],\n", + " [147, 146, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 69, 49],\n", + " [ 72, 72, 52],\n", + " [ 66, 69, 46],\n", + " ...,\n", + " [148, 146, 138],\n", + " [147, 144, 141],\n", + " [146, 143, 140]],\n", + "\n", + " [[ 63, 63, 41],\n", + " [ 62, 62, 40],\n", + " [ 57, 57, 37],\n", + " ...,\n", + " [145, 146, 137],\n", + " [146, 143, 140],\n", + " [146, 143, 140]],\n", + "\n", + " [[ 57, 55, 34],\n", + " [ 56, 54, 33],\n", + " [ 50, 50, 28],\n", + " ...,\n", + " [148, 149, 138],\n", + " [148, 145, 140],\n", + " [149, 146, 141]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5453887 , -1.0169486 , 2.0630584 , -2.6606886 , -1.5223564 ,\n", + " 2.6199625 , 0.531208 , -0.0935498 , -0.13631946, 0.6620134 ,\n", + " 0.748793 , 0.02956677, 0.01315534, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0.0007619 , 0.00152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01308571, 0.01091429, 0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[100, 127, 159],\n", + " [ 98, 126, 158],\n", + " [100, 127, 159],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 12, 12, 12],\n", + " [ 12, 12, 12]],\n", + "\n", + " [[ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " [[100, 127, 159],\n", + " [ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " ...,\n", + "\n", + " [[146, 148, 168],\n", + " [146, 148, 168],\n", + " [142, 147, 167],\n", + " ...,\n", + " [149, 153, 175],\n", + " [149, 153, 175],\n", + " [149, 153, 175]],\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [145, 149, 169],\n", + " ...,\n", + " [149, 153, 175],\n", + " [149, 153, 175],\n", + " [149, 153, 175]],\n", + "\n", + " [[146, 148, 168],\n", + " [147, 149, 169],\n", + " [145, 149, 169],\n", + " ...,\n", + " [152, 156, 178],\n", + " [149, 153, 175],\n", + " [149, 153, 175]]], dtype=uint8), 'image': array([[[105, 98, 77],\n", + " [103, 96, 75],\n", + " [ 48, 72, 74],\n", + " ...,\n", + " [158, 158, 154],\n", + " [157, 152, 148],\n", + " [159, 154, 150]],\n", + "\n", + " [[119, 101, 79],\n", + " [113, 95, 73],\n", + " [ 53, 80, 75],\n", + " ...,\n", + " [156, 155, 155],\n", + " [156, 153, 150],\n", + " [156, 153, 150]],\n", + "\n", + " [[124, 94, 76],\n", + " [123, 93, 75],\n", + " [ 69, 97, 84],\n", + " ...,\n", + " [156, 155, 157],\n", + " [148, 147, 149],\n", + " [148, 147, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 71, 54],\n", + " [ 78, 77, 60],\n", + " [ 76, 75, 58],\n", + " ...,\n", + " [151, 144, 134],\n", + " [147, 145, 135],\n", + " [145, 143, 133]],\n", + "\n", + " [[ 59, 63, 42],\n", + " [ 57, 61, 40],\n", + " [ 55, 58, 37],\n", + " ...,\n", + " [151, 144, 134],\n", + " [148, 144, 135],\n", + " [148, 144, 135]],\n", + "\n", + " [[ 50, 53, 35],\n", + " [ 52, 55, 37],\n", + " [ 48, 51, 33],\n", + " ...,\n", + " [151, 144, 134],\n", + " [149, 145, 136],\n", + " [149, 145, 136]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.556693 , -0.9751294 , 2.0289373 , -2.666302 , -1.5200332 ,\n", + " 2.6086464 , 0.5411583 , -0.10370405, -0.14767449, 0.66197366,\n", + " 0.748819 , 0.03023103, 0.01212799, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0.00095238, 0.01142857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00805714, 0. , 0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[102, 127, 159],\n", + " [100, 126, 158],\n", + " [102, 128, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " [[102, 127, 159],\n", + " [100, 126, 158],\n", + " [100, 127, 157],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14]],\n", + "\n", + " [[104, 128, 158],\n", + " [104, 128, 158],\n", + " [102, 128, 158],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [149, 152, 170],\n", + " [150, 153, 171],\n", + " [150, 153, 171]],\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [149, 152, 170],\n", + " [148, 151, 169],\n", + " [149, 152, 170]],\n", + "\n", + " [[146, 148, 168],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " [148, 151, 169]]], dtype=uint8), 'image': array([[[108, 103, 74],\n", + " [105, 100, 71],\n", + " [ 44, 71, 62],\n", + " ...,\n", + " [158, 154, 155],\n", + " [158, 154, 157],\n", + " [157, 153, 156]],\n", + "\n", + " [[119, 102, 73],\n", + " [123, 106, 77],\n", + " [ 62, 93, 77],\n", + " ...,\n", + " [155, 154, 154],\n", + " [156, 155, 157],\n", + " [154, 153, 155]],\n", + "\n", + " [[120, 91, 66],\n", + " [127, 98, 73],\n", + " [ 71, 100, 80],\n", + " ...,\n", + " [152, 152, 156],\n", + " [148, 147, 151],\n", + " [148, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 68, 49],\n", + " [ 67, 73, 54],\n", + " [ 64, 71, 49],\n", + " ...,\n", + " [150, 146, 137],\n", + " [147, 143, 134],\n", + " [142, 138, 129]],\n", + "\n", + " [[ 56, 63, 41],\n", + " [ 56, 63, 41],\n", + " [ 57, 64, 42],\n", + " ...,\n", + " [148, 144, 135],\n", + " [148, 144, 135],\n", + " [148, 144, 135]],\n", + "\n", + " [[ 50, 56, 39],\n", + " [ 50, 56, 39],\n", + " [ 46, 52, 33],\n", + " ...,\n", + " [148, 146, 136],\n", + " [147, 145, 135],\n", + " [147, 145, 135]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5630758 , -0.9333814 , 1.9979777 , -2.6762102 , -1.5196503 ,\n", + " 2.6031442 , 0.5501979 , -0.11132966, -0.16009057, 0.66227996,\n", + " 0.7485525 , 0.03024558, 0.01181715, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00704762, 0. , 0.02038095], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01777143, 0. , 0.00914286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[112, 130, 150],\n", + " [ 97, 115, 135],\n", + " [ 83, 101, 109],\n", + " ...,\n", + " [ 17, 15, 15],\n", + " [ 18, 16, 16],\n", + " [ 18, 16, 16]],\n", + "\n", + " [[112, 129, 151],\n", + " [102, 120, 142],\n", + " [ 89, 106, 117],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15]],\n", + "\n", + " [[113, 133, 155],\n", + " [105, 125, 147],\n", + " [ 97, 113, 126],\n", + " ...,\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 148, 168],\n", + " [144, 148, 168],\n", + " [144, 148, 168],\n", + " ...,\n", + " [139, 149, 172],\n", + " [139, 149, 172],\n", + " [139, 149, 172]],\n", + "\n", + " [[139, 144, 163],\n", + " [140, 145, 165],\n", + " [139, 144, 163],\n", + " ...,\n", + " [140, 150, 174],\n", + " [139, 149, 172],\n", + " [140, 150, 174]],\n", + "\n", + " [[139, 144, 163],\n", + " [139, 144, 163],\n", + " [139, 144, 163],\n", + " ...,\n", + " [140, 150, 174],\n", + " [140, 150, 174],\n", + " [139, 149, 172]]], dtype=uint8), 'image': array([[[107, 100, 71],\n", + " [109, 102, 73],\n", + " [ 46, 69, 65],\n", + " ...,\n", + " [152, 155, 158],\n", + " [153, 156, 157],\n", + " [152, 155, 156]],\n", + "\n", + " [[114, 95, 68],\n", + " [122, 103, 76],\n", + " [ 52, 78, 65],\n", + " ...,\n", + " [150, 153, 156],\n", + " [152, 155, 156],\n", + " [151, 154, 155]],\n", + "\n", + " [[118, 86, 64],\n", + " [127, 95, 73],\n", + " [ 76, 101, 82],\n", + " ...,\n", + " [150, 152, 158],\n", + " [149, 152, 155],\n", + " [148, 151, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 73, 76, 53],\n", + " [ 75, 78, 55],\n", + " [ 68, 72, 47],\n", + " ...,\n", + " [150, 148, 140],\n", + " [148, 146, 138],\n", + " [141, 139, 131]],\n", + "\n", + " [[ 58, 61, 38],\n", + " [ 59, 62, 39],\n", + " [ 55, 58, 35],\n", + " ...,\n", + " [150, 146, 139],\n", + " [150, 146, 137],\n", + " [147, 143, 134]],\n", + "\n", + " [[ 53, 53, 33],\n", + " [ 51, 51, 31],\n", + " [ 49, 52, 31],\n", + " ...,\n", + " [148, 144, 137],\n", + " [148, 144, 137],\n", + " [150, 146, 139]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.569471 , -0.8993924 , 1.9853354 , -2.6972144 , -1.5198654 ,\n", + " 2.60294 , 0.5531312 , -0.11650646, -0.1728949 , 0.6645783 ,\n", + " 0.7465192 , 0.03010343, 0.01176925, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00095238, -0.00552381, 0.01314286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01902857, -0.00474286, 0.00297143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 150, 168],\n", + " [147, 150, 168],\n", + " [147, 150, 168],\n", + " ...,\n", + " [135, 146, 169],\n", + " [136, 147, 170],\n", + " [136, 147, 170]],\n", + "\n", + " [[148, 151, 169],\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " ...,\n", + " [133, 143, 167],\n", + " [134, 144, 168],\n", + " [134, 144, 168]],\n", + "\n", + " [[149, 152, 170],\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " ...,\n", + " [133, 143, 167],\n", + " [134, 144, 168],\n", + " [134, 144, 168]]], dtype=uint8), 'image': array([[[109, 101, 77],\n", + " [109, 101, 77],\n", + " [ 42, 67, 62],\n", + " ...,\n", + " [155, 155, 151],\n", + " [156, 151, 149],\n", + " [157, 152, 150]],\n", + "\n", + " [[122, 103, 78],\n", + " [124, 105, 80],\n", + " [ 62, 86, 72],\n", + " ...,\n", + " [152, 152, 152],\n", + " [152, 151, 151],\n", + " [153, 152, 152]],\n", + "\n", + " [[126, 93, 76],\n", + " [127, 94, 77],\n", + " [ 78, 105, 85],\n", + " ...,\n", + " [150, 153, 154],\n", + " [151, 151, 153],\n", + " [146, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 70, 48],\n", + " [ 67, 74, 52],\n", + " [ 66, 73, 51],\n", + " ...,\n", + " [148, 146, 136],\n", + " [146, 143, 138],\n", + " [143, 140, 135]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 57, 63, 44],\n", + " [ 60, 63, 45],\n", + " ...,\n", + " [149, 144, 140],\n", + " [149, 144, 140],\n", + " [147, 142, 138]],\n", + "\n", + " [[ 51, 52, 41],\n", + " [ 52, 53, 42],\n", + " [ 47, 50, 34],\n", + " ...,\n", + " [152, 145, 139],\n", + " [149, 144, 140],\n", + " [148, 143, 139]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5539145 , -0.88672715, 1.9930334 , -2.7198975 , -1.5237097 ,\n", + " 2.630835 , 0.5523699 , -0.10781911, -0.17971404, 0.6691864 ,\n", + " 0.7424202 , 0.02894053, 0.0128143 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00266667, -0.00590476, 0.0112381 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01988572, -0.00342857, 0.0024 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 150, 168],\n", + " [147, 150, 168],\n", + " [147, 150, 168],\n", + " ...,\n", + " [134, 144, 168],\n", + " [134, 144, 168],\n", + " [134, 144, 168]],\n", + "\n", + " [[148, 151, 169],\n", + " [148, 151, 169],\n", + " [147, 150, 168],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [132, 142, 165]],\n", + "\n", + " [[148, 151, 169],\n", + " [147, 150, 168],\n", + " [146, 149, 166],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [132, 142, 165]]], dtype=uint8), 'image': array([[[104, 101, 71],\n", + " [101, 98, 68],\n", + " [ 43, 70, 63],\n", + " ...,\n", + " [157, 157, 161],\n", + " [155, 154, 156],\n", + " [154, 153, 155]],\n", + "\n", + " [[116, 101, 73],\n", + " [119, 104, 76],\n", + " [ 63, 91, 76],\n", + " ...,\n", + " [154, 154, 158],\n", + " [154, 153, 157],\n", + " [152, 151, 155]],\n", + "\n", + " [[110, 86, 59],\n", + " [125, 101, 74],\n", + " [ 75, 102, 80],\n", + " ...,\n", + " [152, 155, 158],\n", + " [146, 146, 150],\n", + " [147, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 69, 46],\n", + " [ 69, 72, 49],\n", + " [ 65, 68, 52],\n", + " ...,\n", + " [146, 145, 130],\n", + " [147, 143, 134],\n", + " [146, 142, 133]],\n", + "\n", + " [[ 62, 65, 40],\n", + " [ 61, 64, 39],\n", + " [ 55, 57, 41],\n", + " ...,\n", + " [146, 144, 132],\n", + " [146, 143, 138],\n", + " [147, 144, 139]],\n", + "\n", + " [[ 55, 55, 35],\n", + " [ 54, 54, 34],\n", + " [ 46, 48, 30],\n", + " ...,\n", + " [146, 144, 136],\n", + " [143, 143, 137],\n", + " [143, 143, 137]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5357125 , -0.88817865, 2.0125475 , -2.7420037 , -1.524392 ,\n", + " 2.6583688 , 0.5487908 , -0.09566072, -0.18201101, 0.6725909 ,\n", + " 0.7392859 , 0.02962112, 0.01415604, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00285714, -0.00609524, 0.01180952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01977143, -0.00291429, 0.00234286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 13, 11],\n", + " [ 15, 13, 11],\n", + " [ 14, 12, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 14, 12, 10],\n", + " [ 14, 12, 10],\n", + " [ 12, 11, 9],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14]],\n", + "\n", + " [[ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 10, 11, 9],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [139, 146, 167],\n", + " [138, 146, 170],\n", + " [138, 146, 170]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [138, 144, 166],\n", + " [136, 144, 168],\n", + " [135, 143, 167]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [138, 144, 166],\n", + " [135, 143, 167],\n", + " [135, 143, 167]]], dtype=uint8), 'image': array([[[109, 97, 68],\n", + " [112, 100, 71],\n", + " [ 49, 69, 64],\n", + " ...,\n", + " [153, 152, 152],\n", + " [156, 155, 155],\n", + " [155, 154, 154]],\n", + "\n", + " [[119, 97, 71],\n", + " [124, 102, 76],\n", + " [ 59, 83, 69],\n", + " ...,\n", + " [151, 151, 151],\n", + " [150, 150, 150],\n", + " [150, 150, 150]],\n", + "\n", + " [[117, 83, 59],\n", + " [128, 94, 70],\n", + " [ 74, 99, 80],\n", + " ...,\n", + " [150, 150, 154],\n", + " [145, 145, 147],\n", + " [144, 144, 146]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 71, 43],\n", + " [ 72, 77, 49],\n", + " [ 67, 71, 48],\n", + " ...,\n", + " [141, 142, 133],\n", + " [140, 140, 134],\n", + " [139, 139, 133]],\n", + "\n", + " [[ 54, 59, 31],\n", + " [ 55, 60, 32],\n", + " [ 52, 59, 33],\n", + " ...,\n", + " [143, 144, 133],\n", + " [143, 144, 135],\n", + " [143, 144, 135]],\n", + "\n", + " [[ 53, 57, 32],\n", + " [ 51, 55, 30],\n", + " [ 45, 52, 28],\n", + " ...,\n", + " [143, 144, 135],\n", + " [143, 144, 135],\n", + " [143, 144, 135]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5170557 , -0.8910335 , 2.0314274 , -2.7631152 , -1.5247868 ,\n", + " 2.6843548 , 0.545448 , -0.08352495, -0.18336658, 0.67522 ,\n", + " 0.7368057 , 0.03068829, 0.01592391, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00419048, -0.0072381 , 0.0127619 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01982857, -0.00034286, 0.00194286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 15, 15],\n", + " [ 14, 14, 14],\n", + " [ 13, 13, 13],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 19, 18, 16],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 12, 12, 12],\n", + " [ 12, 12, 12],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 10, 12, 12],\n", + " [ 10, 12, 12],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 157, 170],\n", + " [153, 157, 170],\n", + " [151, 155, 171],\n", + " ...,\n", + " [147, 149, 169],\n", + " [148, 149, 172],\n", + " [148, 149, 172]],\n", + "\n", + " [[154, 158, 171],\n", + " [154, 158, 171],\n", + " [154, 158, 173],\n", + " ...,\n", + " [148, 150, 170],\n", + " [148, 149, 172],\n", + " [149, 151, 173]],\n", + "\n", + " [[153, 157, 170],\n", + " [154, 158, 171],\n", + " [154, 158, 173],\n", + " ...,\n", + " [148, 150, 170],\n", + " [147, 148, 171],\n", + " [148, 149, 172]]], dtype=uint8), 'image': array([[[103, 96, 79],\n", + " [105, 98, 81],\n", + " [ 48, 72, 72],\n", + " ...,\n", + " [153, 154, 148],\n", + " [152, 152, 148],\n", + " [152, 152, 148]],\n", + "\n", + " [[112, 95, 77],\n", + " [112, 95, 77],\n", + " [ 58, 83, 76],\n", + " ...,\n", + " [152, 153, 149],\n", + " [151, 151, 151],\n", + " [149, 149, 149]],\n", + "\n", + " [[112, 82, 64],\n", + " [121, 91, 73],\n", + " [ 75, 101, 86],\n", + " ...,\n", + " [148, 151, 152],\n", + " [147, 147, 151],\n", + " [145, 145, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 69, 46],\n", + " [ 68, 71, 48],\n", + " [ 65, 68, 43],\n", + " ...,\n", + " [139, 137, 129],\n", + " [142, 140, 130],\n", + " [139, 137, 127]],\n", + "\n", + " [[ 58, 61, 38],\n", + " [ 54, 57, 34],\n", + " [ 49, 52, 27],\n", + " ...,\n", + " [141, 139, 131],\n", + " [142, 140, 130],\n", + " [142, 140, 130]],\n", + "\n", + " [[ 51, 51, 31],\n", + " [ 51, 51, 31],\n", + " [ 45, 49, 21],\n", + " ...,\n", + " [140, 138, 130],\n", + " [142, 140, 130],\n", + " [143, 141, 131]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4967704 , -0.89257985, 2.048069 , -2.7829902 , -1.5259126 ,\n", + " 2.7121851 , 0.5425027 , -0.0710614 , -0.1848708 , 0.67793345,\n", + " 0.734238 , 0.03148263, 0.01759424, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00457143, -0.00704762, 0.01142857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, -5.7142857e-05, 2.0571428e-03], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 17, 16],\n", + " [ 11, 16, 15],\n", + " [ 10, 12, 12],\n", + " ...,\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15],\n", + " [ 15, 16, 14]],\n", + "\n", + " [[ 11, 13, 13],\n", + " [ 11, 13, 13],\n", + " [ 12, 12, 12],\n", + " ...,\n", + " [ 15, 16, 14],\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15]],\n", + "\n", + " [[ 11, 13, 13],\n", + " [ 11, 13, 13],\n", + " [ 12, 12, 12],\n", + " ...,\n", + " [ 15, 16, 14],\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [149, 151, 172],\n", + " [148, 153, 173],\n", + " [148, 153, 173]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [150, 153, 173],\n", + " [147, 152, 172],\n", + " [147, 152, 172]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [151, 154, 174],\n", + " [147, 152, 172],\n", + " [147, 152, 172]]], dtype=uint8), 'image': array([[[103, 99, 61],\n", + " [104, 100, 62],\n", + " [ 49, 76, 58],\n", + " ...,\n", + " [153, 154, 150],\n", + " [152, 152, 148],\n", + " [152, 152, 148]],\n", + "\n", + " [[114, 101, 63],\n", + " [117, 104, 66],\n", + " [ 52, 80, 55],\n", + " ...,\n", + " [150, 153, 152],\n", + " [151, 151, 151],\n", + " [150, 150, 150]],\n", + "\n", + " [[113, 87, 56],\n", + " [123, 97, 66],\n", + " [ 75, 104, 74],\n", + " ...,\n", + " [146, 152, 154],\n", + " [147, 147, 149],\n", + " [145, 145, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 45],\n", + " [ 70, 73, 48],\n", + " [ 65, 69, 46],\n", + " ...,\n", + " [138, 136, 124],\n", + " [134, 136, 120],\n", + " [133, 135, 119]],\n", + "\n", + " [[ 58, 61, 36],\n", + " [ 56, 59, 34],\n", + " [ 54, 57, 34],\n", + " ...,\n", + " [138, 137, 122],\n", + " [138, 137, 122],\n", + " [138, 137, 122]],\n", + "\n", + " [[ 54, 57, 32],\n", + " [ 51, 54, 29],\n", + " [ 47, 51, 30],\n", + " ...,\n", + " [139, 138, 123],\n", + " [138, 137, 122],\n", + " [138, 137, 122]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4744623 , -0.8928798 , 2.0601006 , -2.8009937 , -1.5276612 ,\n", + " 2.7427585 , 0.54119325, -0.05844606, -0.18594792, 0.6808814 ,\n", + " 0.7314025 , 0.03243729, 0.01996658, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00514286, -0.00666667, 0.01314286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01982857, 0. , 0.00194286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 14, 14],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12],\n", + " [ 17, 16, 14]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 8, 11, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[143, 150, 169],\n", + " [143, 150, 169],\n", + " [143, 150, 169],\n", + " ...,\n", + " [145, 149, 171],\n", + " [145, 149, 169],\n", + " [145, 149, 169]],\n", + "\n", + " [[145, 152, 172],\n", + " [145, 152, 172],\n", + " [145, 152, 172],\n", + " ...,\n", + " [145, 149, 171],\n", + " [146, 151, 170],\n", + " [145, 149, 169]],\n", + "\n", + " [[145, 152, 172],\n", + " [146, 153, 173],\n", + " [146, 153, 173],\n", + " ...,\n", + " [145, 149, 171],\n", + " [146, 151, 170],\n", + " [145, 149, 169]]], dtype=uint8), 'image': array([[[111, 99, 81],\n", + " [106, 94, 76],\n", + " [ 49, 73, 73],\n", + " ...,\n", + " [156, 156, 150],\n", + " [152, 153, 149],\n", + " [152, 153, 149]],\n", + "\n", + " [[121, 101, 81],\n", + " [115, 95, 75],\n", + " [ 55, 82, 75],\n", + " ...,\n", + " [152, 153, 149],\n", + " [152, 152, 152],\n", + " [150, 150, 150]],\n", + "\n", + " [[120, 88, 68],\n", + " [125, 93, 73],\n", + " [ 68, 96, 81],\n", + " ...,\n", + " [149, 152, 151],\n", + " [146, 146, 148],\n", + " [145, 145, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 68, 45],\n", + " [ 66, 70, 47],\n", + " [ 62, 65, 40],\n", + " ...,\n", + " [131, 129, 121],\n", + " [134, 130, 121],\n", + " [132, 128, 119]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 57, 61, 36],\n", + " [ 54, 58, 35],\n", + " ...,\n", + " [133, 129, 120],\n", + " [133, 129, 120],\n", + " [134, 130, 121]],\n", + "\n", + " [[ 55, 59, 36],\n", + " [ 50, 54, 31],\n", + " [ 45, 49, 28],\n", + " ...,\n", + " [134, 130, 121],\n", + " [133, 129, 120],\n", + " [135, 131, 122]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.451041 , -0.89231616, 2.0696073 , -2.817558 , -1.5302237 ,\n", + " 2.7739055 , 0.54045534, -0.0457982 , -0.18693021, 0.6836287 ,\n", + " 0.728724 , 0.03323772, 0.02254771, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00457143, -0.00438095, 0.01333333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01977143, 0. , 0.00228571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 14, 14],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 13, 14, 11],\n", + " [ 13, 14, 11],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 14, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " ...,\n", + "\n", + " [[142, 151, 172],\n", + " [142, 151, 172],\n", + " [141, 149, 171],\n", + " ...,\n", + " [143, 145, 167],\n", + " [144, 146, 168],\n", + " [144, 146, 168]],\n", + "\n", + " [[141, 149, 171],\n", + " [141, 149, 171],\n", + " [141, 149, 171],\n", + " ...,\n", + " [144, 146, 168],\n", + " [142, 144, 166],\n", + " [141, 142, 165]],\n", + "\n", + " [[142, 151, 172],\n", + " [142, 151, 172],\n", + " [142, 151, 172],\n", + " ...,\n", + " [144, 146, 168],\n", + " [143, 145, 167],\n", + " [143, 145, 167]]], dtype=uint8), 'image': array([[[109, 102, 71],\n", + " [111, 104, 73],\n", + " [ 44, 69, 62],\n", + " ...,\n", + " [151, 155, 150],\n", + " [151, 154, 153],\n", + " [149, 152, 151]],\n", + "\n", + " [[117, 95, 69],\n", + " [121, 99, 73],\n", + " [ 66, 90, 76],\n", + " ...,\n", + " [150, 153, 154],\n", + " [147, 150, 151],\n", + " [147, 150, 151]],\n", + "\n", + " [[114, 80, 56],\n", + " [124, 90, 66],\n", + " [ 77, 99, 79],\n", + " ...,\n", + " [148, 151, 152],\n", + " [143, 146, 149],\n", + " [143, 146, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 59, 71, 47],\n", + " [ 62, 74, 50],\n", + " [ 61, 70, 50],\n", + " ...,\n", + " [125, 123, 113],\n", + " [123, 124, 113],\n", + " [120, 121, 110]],\n", + "\n", + " [[ 47, 62, 35],\n", + " [ 48, 63, 36],\n", + " [ 52, 64, 40],\n", + " ...,\n", + " [124, 125, 114],\n", + " [126, 124, 114],\n", + " [125, 123, 113]],\n", + "\n", + " [[ 45, 57, 31],\n", + " [ 43, 55, 29],\n", + " [ 42, 54, 28],\n", + " ...,\n", + " [127, 128, 117],\n", + " [128, 126, 116],\n", + " [127, 125, 115]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4267576 , -0.8909252 , 2.0777245 , -2.832958 , -1.5336002 ,\n", + " 2.8063924 , 0.5396934 , -0.03303649, -0.18807867, 0.6865643 ,\n", + " 0.7258565 , 0.03373102, 0.02497944, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00495238, -0.0047619 , 0.01333333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01977143, 0. , 0.00177143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 15, 12],\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 14, 12, 10]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " [[ 8, 12, 9],\n", + " [ 8, 12, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 155, 172],\n", + " [149, 155, 172],\n", + " [148, 154, 171],\n", + " ...,\n", + " [144, 148, 168],\n", + " [145, 149, 169],\n", + " [145, 149, 169]],\n", + "\n", + " [[149, 155, 172],\n", + " [151, 156, 173],\n", + " [151, 156, 173],\n", + " ...,\n", + " [144, 148, 168],\n", + " [144, 148, 168],\n", + " [145, 149, 169]],\n", + "\n", + " [[149, 155, 172],\n", + " [148, 154, 171],\n", + " [148, 154, 171],\n", + " ...,\n", + " [145, 149, 169],\n", + " [145, 149, 169],\n", + " [145, 149, 169]]], dtype=uint8), 'image': array([[[111, 104, 71],\n", + " [103, 96, 63],\n", + " [ 54, 74, 67],\n", + " ...,\n", + " [154, 153, 153],\n", + " [153, 152, 152],\n", + " [153, 152, 152]],\n", + "\n", + " [[120, 105, 68],\n", + " [111, 96, 59],\n", + " [ 62, 83, 70],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[120, 89, 60],\n", + " [122, 91, 62],\n", + " [ 78, 100, 82],\n", + " ...,\n", + " [152, 152, 156],\n", + " [146, 145, 152],\n", + " [144, 143, 150]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 48],\n", + " [ 69, 76, 50],\n", + " [ 64, 71, 47],\n", + " ...,\n", + " [115, 112, 96],\n", + " [115, 114, 97],\n", + " [114, 113, 96]],\n", + "\n", + " [[ 54, 58, 33],\n", + " [ 57, 61, 36],\n", + " [ 60, 67, 41],\n", + " ...,\n", + " [119, 118, 103],\n", + " [118, 117, 102],\n", + " [117, 116, 101]],\n", + "\n", + " [[ 52, 55, 34],\n", + " [ 48, 51, 30],\n", + " [ 45, 49, 24],\n", + " ...,\n", + " [120, 119, 104],\n", + " [118, 117, 102],\n", + " [118, 117, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4022958 , -0.88859874, 2.0833273 , -2.8456123 , -1.5374321 ,\n", + " 2.8394403 , 0.5391687 , -0.02050408, -0.18934578, 0.6896574 ,\n", + " 0.7228375 , 0.03391662, 0.02698694, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00438095, -0.00533333, 0.012 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.02 , 0. , 0.0008], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 14, 13, 8],\n", + " [ 14, 13, 8],\n", + " [ 14, 13, 8]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[148, 154, 171],\n", + " [148, 154, 171],\n", + " [149, 151, 172],\n", + " ...,\n", + " [140, 147, 168],\n", + " [142, 148, 170],\n", + " [142, 148, 170]],\n", + "\n", + " [[147, 153, 170],\n", + " [146, 151, 169],\n", + " [149, 151, 172],\n", + " ...,\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " [142, 148, 170]],\n", + "\n", + " [[146, 151, 169],\n", + " [147, 153, 170],\n", + " [149, 151, 172],\n", + " ...,\n", + " [142, 148, 170],\n", + " [143, 149, 171],\n", + " [142, 148, 170]]], dtype=uint8), 'image': array([[[103, 98, 69],\n", + " [105, 100, 71],\n", + " [ 44, 71, 66],\n", + " ...,\n", + " [154, 153, 157],\n", + " [156, 152, 157],\n", + " [154, 150, 155]],\n", + "\n", + " [[111, 94, 69],\n", + " [123, 106, 81],\n", + " [ 63, 90, 81],\n", + " ...,\n", + " [152, 152, 156],\n", + " [151, 150, 154],\n", + " [152, 151, 155]],\n", + "\n", + " [[111, 79, 57],\n", + " [128, 96, 74],\n", + " [ 75, 100, 83],\n", + " ...,\n", + " [147, 149, 155],\n", + " [146, 146, 150],\n", + " [144, 144, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 66, 41],\n", + " [ 67, 70, 45],\n", + " [ 68, 69, 40],\n", + " ...,\n", + " [105, 104, 89],\n", + " [108, 107, 90],\n", + " [108, 107, 90]],\n", + "\n", + " [[ 56, 60, 30],\n", + " [ 59, 63, 33],\n", + " [ 60, 65, 28],\n", + " ...,\n", + " [110, 109, 94],\n", + " [111, 110, 95],\n", + " [107, 106, 91]],\n", + "\n", + " [[ 51, 53, 21],\n", + " [ 51, 53, 21],\n", + " [ 49, 52, 13],\n", + " ...,\n", + " [112, 111, 96],\n", + " [113, 112, 97],\n", + " [111, 110, 95]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.377918 , -0.88633186, 2.08747 , -2.8563583 , -1.5414795 ,\n", + " 2.8719716 , 0.5387577 , -0.00821607, -0.19039896, 0.6925935 ,\n", + " 0.7199444 , 0.03404387, 0.02890096, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.0047619 , -0.00609524, 0.01047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , -0.00022857, 0.0004 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 14, 13, 8]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 8, 11, 10],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [151, 155, 172],\n", + " ...,\n", + " [134, 142, 165],\n", + " [135, 143, 167],\n", + " [135, 143, 167]],\n", + "\n", + " [[148, 151, 169],\n", + " [150, 153, 171],\n", + " [150, 153, 171],\n", + " ...,\n", + " [136, 144, 168],\n", + " [136, 144, 168],\n", + " [136, 144, 168]],\n", + "\n", + " [[143, 146, 164],\n", + " [147, 150, 168],\n", + " [146, 149, 166],\n", + " ...,\n", + " [136, 144, 168],\n", + " [136, 144, 168],\n", + " [136, 144, 168]]], dtype=uint8), 'image': array([[[108, 98, 66],\n", + " [108, 98, 66],\n", + " [ 51, 73, 57],\n", + " ...,\n", + " [154, 153, 153],\n", + " [153, 153, 149],\n", + " [152, 152, 148]],\n", + "\n", + " [[113, 97, 65],\n", + " [113, 97, 65],\n", + " [ 62, 88, 64],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[110, 82, 54],\n", + " [121, 93, 65],\n", + " [ 74, 100, 74],\n", + " ...,\n", + " [149, 151, 157],\n", + " [145, 147, 153],\n", + " [145, 147, 153]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 69, 42],\n", + " [ 72, 73, 46],\n", + " [ 71, 71, 49],\n", + " ...,\n", + " [ 95, 88, 78],\n", + " [ 93, 89, 78],\n", + " [ 93, 89, 78]],\n", + "\n", + " [[ 62, 63, 36],\n", + " [ 61, 62, 35],\n", + " [ 58, 58, 36],\n", + " ...,\n", + " [ 94, 91, 77],\n", + " [ 94, 92, 80],\n", + " [ 97, 95, 83]],\n", + "\n", + " [[ 57, 55, 34],\n", + " [ 52, 50, 29],\n", + " [ 48, 48, 28],\n", + " ...,\n", + " [ 98, 95, 81],\n", + " [ 98, 94, 83],\n", + " [100, 96, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3518052 , -0.8849166 , 2.0911217 , -2.8662055 , -1.5458258 ,\n", + " 2.9062974 , 0.53834844, 0.00483379, -0.19096382, 0.6955367 ,\n", + " 0.71699756, 0.0343527 , 0.03103506, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00609524, -0.00457143, 0.00780952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " ...,\n", + " [ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [151, 155, 172],\n", + " [151, 155, 172],\n", + " ...,\n", + " [127, 137, 161],\n", + " [128, 139, 162],\n", + " [128, 139, 162]],\n", + "\n", + " [[149, 152, 170],\n", + " [151, 155, 172],\n", + " [151, 155, 172],\n", + " ...,\n", + " [128, 139, 162],\n", + " [127, 137, 161],\n", + " [127, 137, 161]],\n", + "\n", + " [[149, 152, 170],\n", + " [149, 152, 170],\n", + " [150, 153, 171],\n", + " ...,\n", + " [128, 139, 162],\n", + " [127, 137, 161],\n", + " [127, 137, 161]]], dtype=uint8), 'image': array([[[104, 104, 71],\n", + " [100, 100, 67],\n", + " [ 47, 79, 68],\n", + " ...,\n", + " [158, 152, 154],\n", + " [156, 150, 154],\n", + " [156, 150, 154]],\n", + "\n", + " [[111, 97, 62],\n", + " [114, 100, 65],\n", + " [ 61, 93, 72],\n", + " ...,\n", + " [156, 153, 150],\n", + " [154, 150, 151],\n", + " [153, 149, 150]],\n", + "\n", + " [[105, 79, 48],\n", + " [122, 96, 65],\n", + " [ 77, 105, 78],\n", + " ...,\n", + " [153, 153, 149],\n", + " [148, 147, 147],\n", + " [147, 146, 146]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 47],\n", + " [ 70, 73, 50],\n", + " [ 67, 71, 50],\n", + " ...,\n", + " [ 77, 77, 71],\n", + " [ 75, 75, 71],\n", + " [ 74, 74, 70]],\n", + "\n", + " [[ 56, 59, 34],\n", + " [ 59, 62, 37],\n", + " [ 53, 55, 37],\n", + " ...,\n", + " [ 80, 78, 68],\n", + " [ 78, 75, 70],\n", + " [ 79, 76, 71]],\n", + "\n", + " [[ 52, 50, 31],\n", + " [ 54, 52, 33],\n", + " [ 48, 47, 32],\n", + " ...,\n", + " [ 83, 80, 66],\n", + " [ 83, 79, 72],\n", + " [ 86, 82, 75]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3262916 , -0.88400537, 2.0940309 , -2.8750207 , -1.5502566 ,\n", + " 2.9384649 , 0.5379629 , 0.01745152, -0.19117709, 0.69790804,\n", + " 0.71456426, 0.03479953, 0.03333521, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00666667, -0.00095238, 0.00533333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 14, 13],\n", + " [ 5, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 5, 10, 9],\n", + " [ 5, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 7, 10, 9],\n", + " [ 7, 10, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [153, 156, 173],\n", + " ...,\n", + " [134, 142, 165],\n", + " [134, 142, 165],\n", + " [134, 142, 165]],\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [153, 156, 173],\n", + " ...,\n", + " [131, 139, 163],\n", + " [133, 140, 164],\n", + " [133, 140, 164]],\n", + "\n", + " [[153, 156, 173],\n", + " [154, 157, 175],\n", + " [154, 157, 175],\n", + " ...,\n", + " [133, 140, 164],\n", + " [133, 140, 164],\n", + " [131, 139, 163]]], dtype=uint8), 'image': array([[[106, 97, 60],\n", + " [107, 98, 61],\n", + " [ 44, 68, 56],\n", + " ...,\n", + " [152, 150, 157],\n", + " [152, 152, 156],\n", + " [152, 152, 156]],\n", + "\n", + " [[116, 95, 64],\n", + " [123, 102, 71],\n", + " [ 60, 85, 68],\n", + " ...,\n", + " [152, 151, 158],\n", + " [150, 150, 154],\n", + " [151, 151, 155]],\n", + "\n", + " [[113, 82, 55],\n", + " [127, 96, 69],\n", + " [ 78, 103, 82],\n", + " ...,\n", + " [152, 152, 156],\n", + " [145, 148, 151],\n", + " [141, 144, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 66, 45],\n", + " [ 67, 71, 50],\n", + " [ 61, 71, 46],\n", + " ...,\n", + " [ 44, 73, 85],\n", + " [ 43, 71, 86],\n", + " [ 48, 76, 91]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 53, 60, 36],\n", + " [ 50, 61, 31],\n", + " ...,\n", + " [ 50, 79, 89],\n", + " [ 44, 73, 85],\n", + " [ 45, 74, 86]],\n", + "\n", + " [[ 50, 54, 31],\n", + " [ 50, 54, 31],\n", + " [ 42, 52, 25],\n", + " ...,\n", + " [ 65, 97, 106],\n", + " [ 57, 88, 102],\n", + " [ 52, 83, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.299629 , -0.8831771 , 2.0949645 , -2.8811018 , -1.5560879 ,\n", + " 2.9703782 , 0.5377479 , 0.03023179, -0.1911558 , 0.6998129 ,\n", + " 0.71258867, 0.03465674, 0.03575221, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.004 , 0. , 0.00552381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 10, 8],\n", + " [ 10, 9, 7],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 11, 10, 8],\n", + " [ 11, 10, 8],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[160, 160, 174],\n", + " [160, 160, 174],\n", + " [157, 159, 173],\n", + " ...,\n", + " [137, 143, 165],\n", + " [137, 143, 165],\n", + " [137, 143, 165]],\n", + "\n", + " [[159, 159, 173],\n", + " [160, 160, 174],\n", + " [159, 162, 175],\n", + " ...,\n", + " [137, 143, 165],\n", + " [136, 142, 164],\n", + " [136, 142, 164]],\n", + "\n", + " [[158, 158, 171],\n", + " [158, 158, 171],\n", + " [157, 159, 173],\n", + " ...,\n", + " [136, 142, 164],\n", + " [136, 142, 164],\n", + " [135, 141, 163]]], dtype=uint8), 'image': array([[[111, 101, 71],\n", + " [104, 94, 64],\n", + " [ 49, 69, 66],\n", + " ...,\n", + " [155, 154, 156],\n", + " [153, 152, 152],\n", + " [151, 150, 150]],\n", + "\n", + " [[119, 101, 69],\n", + " [119, 101, 69],\n", + " [ 62, 84, 68],\n", + " ...,\n", + " [152, 155, 156],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[118, 87, 58],\n", + " [126, 95, 66],\n", + " [ 78, 101, 78],\n", + " ...,\n", + " [149, 154, 159],\n", + " [144, 147, 150],\n", + " [143, 146, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 71, 49],\n", + " [ 65, 72, 50],\n", + " [ 63, 71, 42],\n", + " ...,\n", + " [ 64, 89, 94],\n", + " [ 63, 91, 97],\n", + " [ 63, 91, 97]],\n", + "\n", + " [[ 53, 60, 36],\n", + " [ 51, 58, 34],\n", + " [ 49, 56, 30],\n", + " ...,\n", + " [ 59, 87, 93],\n", + " [ 63, 87, 97],\n", + " [ 64, 88, 98]],\n", + "\n", + " [[ 53, 57, 34],\n", + " [ 52, 56, 33],\n", + " [ 42, 49, 25],\n", + " ...,\n", + " [ 55, 84, 92],\n", + " [ 57, 84, 93],\n", + " [ 57, 84, 93]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2725835 , -0.88199025, 2.093504 , -2.8825166 , -1.5627569 ,\n", + " 3.0007691 , 0.53741384, 0.0429778 , -0.19125551, 0.70110106,\n", + " 0.7112887 , 0.03353317, 0.03743315, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00647619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 12],\n", + " [ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 7, 7, 7],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 7, 7, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[155, 156, 172],\n", + " [155, 156, 172],\n", + " [155, 155, 171],\n", + " ...,\n", + " [129, 140, 163],\n", + " [131, 141, 164],\n", + " [131, 141, 164]],\n", + "\n", + " [[152, 154, 169],\n", + " [153, 155, 171],\n", + " [157, 156, 172],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [131, 141, 164]],\n", + "\n", + " [[155, 156, 172],\n", + " [153, 155, 171],\n", + " [155, 155, 171],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [131, 141, 164]]], dtype=uint8), 'image': array([[[107, 96, 71],\n", + " [107, 96, 71],\n", + " [ 51, 74, 66],\n", + " ...,\n", + " [151, 154, 155],\n", + " [153, 152, 154],\n", + " [153, 152, 154]],\n", + "\n", + " [[112, 95, 68],\n", + " [118, 101, 74],\n", + " [ 58, 83, 66],\n", + " ...,\n", + " [150, 153, 152],\n", + " [151, 150, 152],\n", + " [151, 150, 152]],\n", + "\n", + " [[118, 90, 62],\n", + " [130, 102, 74],\n", + " [ 69, 94, 73],\n", + " ...,\n", + " [150, 153, 154],\n", + " [146, 146, 150],\n", + " [147, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 70, 48],\n", + " [ 66, 73, 51],\n", + " [ 62, 69, 45],\n", + " ...,\n", + " [ 66, 94, 100],\n", + " [ 64, 92, 98],\n", + " [ 62, 90, 96]],\n", + "\n", + " [[ 55, 62, 38],\n", + " [ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " ...,\n", + " [ 68, 93, 100],\n", + " [ 64, 89, 96],\n", + " [ 64, 89, 96]],\n", + "\n", + " [[ 47, 51, 30],\n", + " [ 51, 55, 34],\n", + " [ 42, 49, 27],\n", + " ...,\n", + " [ 65, 90, 97],\n", + " [ 63, 88, 95],\n", + " [ 63, 88, 95]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2453115 , -0.88051444, 2.0899243 , -2.8813412 , -1.5684208 ,\n", + " 3.031338 , 0.53737307, 0.05604258, -0.19126037, 0.70236003,\n", + " 0.7100228 , 0.03269348, 0.03858953, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00666667], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 14, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 14, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [155, 155, 173],\n", + " ...,\n", + " [134, 142, 165],\n", + " [134, 142, 164],\n", + " [134, 142, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [135, 143, 167],\n", + " [134, 142, 164],\n", + " [134, 142, 164]],\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [156, 156, 175],\n", + " ...,\n", + " [135, 143, 167],\n", + " [134, 142, 164],\n", + " [134, 142, 164]]], dtype=uint8), 'image': array([[[109, 98, 73],\n", + " [108, 97, 72],\n", + " [ 55, 75, 68],\n", + " ...,\n", + " [154, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[114, 94, 72],\n", + " [121, 101, 79],\n", + " [ 62, 84, 68],\n", + " ...,\n", + " [153, 152, 154],\n", + " [151, 151, 153],\n", + " [150, 150, 152]],\n", + "\n", + " [[109, 77, 55],\n", + " [126, 94, 72],\n", + " [ 78, 101, 78],\n", + " ...,\n", + " [151, 151, 155],\n", + " [147, 147, 149],\n", + " [146, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 71, 46],\n", + " [ 69, 73, 48],\n", + " [ 67, 71, 48],\n", + " ...,\n", + " [ 63, 96, 98],\n", + " [ 62, 92, 95],\n", + " [ 61, 91, 94]],\n", + "\n", + " [[ 61, 65, 37],\n", + " [ 62, 66, 38],\n", + " [ 57, 61, 40],\n", + " ...,\n", + " [ 61, 94, 96],\n", + " [ 62, 92, 95],\n", + " [ 60, 90, 93]],\n", + "\n", + " [[ 53, 56, 31],\n", + " [ 53, 56, 31],\n", + " [ 46, 50, 29],\n", + " ...,\n", + " [ 61, 94, 96],\n", + " [ 62, 92, 95],\n", + " [ 61, 91, 94]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.219752 , -0.8787039 , 2.0854707 , -2.8789427 , -1.5713781 ,\n", + " 3.0608265 , 0.53732604, 0.06888904, -0.19135544, 0.70380074,\n", + " 0.7085868 , 0.03269861, 0.03873029, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00552381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 7, 14, 13],\n", + " [ 3, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 3, 10, 9],\n", + " [ 2, 9, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 5, 10, 9],\n", + " [ 4, 9, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 151, 172],\n", + " [149, 151, 172],\n", + " [148, 150, 170],\n", + " ...,\n", + " [126, 139, 162],\n", + " [127, 140, 163],\n", + " [127, 140, 163]],\n", + "\n", + " [[151, 154, 174],\n", + " [153, 155, 175],\n", + " [151, 154, 174],\n", + " ...,\n", + " [127, 140, 163],\n", + " [126, 139, 162],\n", + " [127, 140, 163]],\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [153, 155, 175],\n", + " ...,\n", + " [129, 141, 164],\n", + " [127, 140, 163],\n", + " [127, 140, 163]]], dtype=uint8), 'image': array([[[104, 99, 66],\n", + " [102, 97, 64],\n", + " [ 47, 71, 57],\n", + " ...,\n", + " [155, 154, 154],\n", + " [153, 152, 152],\n", + " [152, 151, 151]],\n", + "\n", + " [[114, 99, 71],\n", + " [118, 103, 75],\n", + " [ 57, 84, 66],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [147, 147, 149]],\n", + "\n", + " [[116, 89, 65],\n", + " [131, 104, 80],\n", + " [ 68, 95, 73],\n", + " ...,\n", + " [153, 152, 159],\n", + " [147, 147, 151],\n", + " [146, 146, 150]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 48],\n", + " [ 72, 75, 52],\n", + " [ 75, 76, 49],\n", + " ...,\n", + " [ 66, 91, 98],\n", + " [ 65, 90, 97],\n", + " [ 64, 89, 96]],\n", + "\n", + " [[ 57, 61, 36],\n", + " [ 55, 59, 34],\n", + " [ 51, 54, 29],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 90, 96],\n", + " [ 62, 90, 96]],\n", + "\n", + " [[ 55, 58, 33],\n", + " [ 51, 54, 29],\n", + " [ 48, 51, 26],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 62, 92, 95],\n", + " [ 63, 93, 96]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.194349 , -0.8765333 , 2.0798793 , -2.8753808 , -1.5737132 ,\n", + " 3.09029 , 0.5372993 , 0.08182718, -0.19148241, 0.7052792 ,\n", + " 0.7071188 , 0.03283391, 0.03855143, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 14, 13],\n", + " [ 5, 10, 9],\n", + " [ 7, 10, 9],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 7, 10, 9],\n", + " [ 6, 9, 8],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 7, 11, 8],\n", + " [ 6, 9, 7],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [155, 155, 173],\n", + " ...,\n", + " [129, 140, 163],\n", + " [131, 141, 164],\n", + " [131, 141, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [129, 140, 163],\n", + " [129, 140, 163],\n", + " [131, 141, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [129, 140, 163],\n", + " [129, 140, 163],\n", + " [129, 140, 163]]], dtype=uint8), 'image': array([[[106, 98, 72],\n", + " [103, 95, 69],\n", + " [ 52, 74, 73],\n", + " ...,\n", + " [155, 154, 154],\n", + " [153, 152, 154],\n", + " [154, 153, 155]],\n", + "\n", + " [[115, 97, 65],\n", + " [113, 95, 63],\n", + " [ 64, 88, 74],\n", + " ...,\n", + " [153, 153, 155],\n", + " [154, 153, 157],\n", + " [151, 150, 154]],\n", + "\n", + " [[111, 84, 51],\n", + " [120, 93, 60],\n", + " [ 83, 106, 83],\n", + " ...,\n", + " [152, 152, 156],\n", + " [150, 149, 156],\n", + " [146, 145, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 69, 52],\n", + " [ 69, 75, 58],\n", + " [ 65, 71, 52],\n", + " ...,\n", + " [ 71, 96, 101],\n", + " [ 78, 100, 103],\n", + " [ 75, 97, 100]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 57, 63, 44],\n", + " [ 56, 63, 41],\n", + " ...,\n", + " [ 67, 95, 101],\n", + " [ 75, 100, 105],\n", + " [ 75, 100, 105]],\n", + "\n", + " [[ 47, 50, 34],\n", + " [ 50, 53, 37],\n", + " [ 45, 52, 30],\n", + " ...,\n", + " [ 64, 91, 102],\n", + " [ 69, 96, 105],\n", + " [ 71, 98, 107]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1694727 , -0.87412244, 2.0735097 , -2.8713624 , -1.5762395 ,\n", + " 3.1185782 , 0.53730077, 0.09447004, -0.19158448, 0.7065261 ,\n", + " 0.7058712 , 0.03291051, 0.03851758, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.004], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 10, 10],\n", + " [ 8, 8, 8],\n", + " [ 7, 8, 5],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [158, 157, 173],\n", + " ...,\n", + " [122, 136, 161],\n", + " [123, 137, 162],\n", + " [123, 137, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [159, 158, 174],\n", + " ...,\n", + " [123, 137, 162],\n", + " [123, 137, 162],\n", + " [123, 137, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [158, 157, 173],\n", + " ...,\n", + " [123, 137, 162],\n", + " [124, 138, 163],\n", + " [124, 138, 163]]], dtype=uint8), 'image': array([[[106, 103, 71],\n", + " [104, 101, 69],\n", + " [ 49, 72, 66],\n", + " ...,\n", + " [155, 156, 150],\n", + " [155, 156, 147],\n", + " [153, 154, 145]],\n", + "\n", + " [[117, 102, 74],\n", + " [119, 104, 76],\n", + " [ 59, 85, 70],\n", + " ...,\n", + " [151, 155, 148],\n", + " [153, 155, 146],\n", + " [152, 154, 145]],\n", + "\n", + " [[110, 82, 54],\n", + " [128, 100, 72],\n", + " [ 74, 99, 78],\n", + " ...,\n", + " [149, 153, 148],\n", + " [145, 149, 142],\n", + " [142, 146, 139]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 67, 49],\n", + " [ 67, 70, 52],\n", + " [ 66, 69, 51],\n", + " ...,\n", + " [ 61, 91, 96],\n", + " [ 60, 93, 97],\n", + " [ 58, 91, 95]],\n", + "\n", + " [[ 57, 61, 40],\n", + " [ 58, 62, 41],\n", + " [ 49, 55, 36],\n", + " ...,\n", + " [ 59, 92, 96],\n", + " [ 60, 93, 97],\n", + " [ 59, 92, 96]],\n", + "\n", + " [[ 52, 56, 35],\n", + " [ 50, 54, 33],\n", + " [ 44, 50, 31],\n", + " ...,\n", + " [ 59, 88, 96],\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1447768 , -0.8713878 , 2.066051 , -2.8665173 , -1.5785502 ,\n", + " 3.1462178 , 0.53738934, 0.10712075, -0.19166769, 0.70760185,\n", + " 0.7047916 , 0.03304661, 0.03842265, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 15, 12],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [156, 156, 175],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [157, 158, 176],\n", + " ...,\n", + " [113, 129, 156],\n", + " [112, 128, 155],\n", + " [113, 129, 156]],\n", + "\n", + " [[157, 158, 176],\n", + " [156, 156, 175],\n", + " [157, 158, 176],\n", + " ...,\n", + " [115, 131, 158],\n", + " [113, 129, 156],\n", + " [114, 130, 157]]], dtype=uint8), 'image': array([[[112, 100, 73],\n", + " [112, 100, 73],\n", + " [ 53, 71, 69],\n", + " ...,\n", + " [150, 154, 149],\n", + " [149, 153, 146],\n", + " [150, 154, 147]],\n", + "\n", + " [[120, 101, 72],\n", + " [117, 98, 69],\n", + " [ 67, 88, 75],\n", + " ...,\n", + " [149, 153, 148],\n", + " [148, 152, 147],\n", + " [148, 152, 147]],\n", + "\n", + " [[110, 82, 54],\n", + " [121, 93, 65],\n", + " [ 77, 99, 81],\n", + " ...,\n", + " [149, 152, 151],\n", + " [144, 147, 146],\n", + " [143, 146, 145]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 72, 48],\n", + " [ 64, 76, 52],\n", + " [ 63, 73, 50],\n", + " ...,\n", + " [ 60, 90, 95],\n", + " [ 58, 83, 90],\n", + " [ 58, 83, 90]],\n", + "\n", + " [[ 53, 63, 38],\n", + " [ 51, 61, 36],\n", + " [ 52, 59, 37],\n", + " ...,\n", + " [ 60, 90, 95],\n", + " [ 58, 83, 90],\n", + " [ 56, 81, 88]],\n", + "\n", + " [[ 48, 52, 29],\n", + " [ 50, 54, 31],\n", + " [ 46, 53, 31],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 87, 92],\n", + " [ 55, 80, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1198535 , -0.8682216 , 2.057539 , -2.860892 , -1.5811008 ,\n", + " 3.1736052 , 0.53747594, 0.11990292, -0.19179024, 0.70851207,\n", + " 0.7038783 , 0.03306015, 0.03838015, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 154, 174],\n", + " [149, 154, 174],\n", + " [150, 153, 171],\n", + " ...,\n", + " [112, 132, 161],\n", + " [112, 132, 161],\n", + " [112, 132, 161]],\n", + "\n", + " [[151, 155, 175],\n", + " [151, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [110, 130, 159],\n", + " [109, 129, 157],\n", + " [111, 131, 160]],\n", + "\n", + " [[152, 156, 176],\n", + " [152, 156, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [112, 132, 161],\n", + " [113, 133, 162],\n", + " [115, 135, 163]]], dtype=uint8), 'image': array([[[104, 100, 62],\n", + " [ 97, 93, 55],\n", + " [ 39, 68, 67],\n", + " ...,\n", + " [152, 155, 154],\n", + " [151, 154, 153],\n", + " [150, 153, 152]],\n", + "\n", + " [[116, 101, 62],\n", + " [111, 96, 57],\n", + " [ 54, 84, 74],\n", + " ...,\n", + " [152, 155, 156],\n", + " [149, 152, 153],\n", + " [147, 150, 151]],\n", + "\n", + " [[118, 87, 60],\n", + " [122, 91, 64],\n", + " [ 73, 102, 84],\n", + " ...,\n", + " [150, 152, 160],\n", + " [144, 146, 154],\n", + " [144, 146, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 69, 46],\n", + " [ 67, 71, 48],\n", + " [ 64, 68, 47],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " [ 56, 62, 43],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 98]],\n", + "\n", + " [[ 43, 52, 32],\n", + " [ 43, 52, 32],\n", + " [ 42, 50, 35],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 61, 93, 100],\n", + " [ 58, 90, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0955417 , -0.8648394 , 2.0481286 , -2.8546073 , -1.5833639 ,\n", + " 3.2002604 , 0.53765064, 0.1325151 , -0.191861 , 0.7093723 ,\n", + " 0.70301116, 0.03317269, 0.03828627, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00514286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00011429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 8, 16, 12],\n", + " [ 4, 12, 9],\n", + " [ 8, 12, 9],\n", + " ...,\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " [[ 5, 11, 8],\n", + " [ 5, 11, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 9, 11, 6],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " [[ 7, 11, 8],\n", + " [ 7, 11, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 9, 11, 6],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[155, 158, 176],\n", + " [155, 158, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [112, 127, 156],\n", + " [112, 127, 156],\n", + " [113, 128, 157]],\n", + "\n", + " [[154, 157, 175],\n", + " [155, 158, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [114, 129, 159],\n", + " [114, 129, 159],\n", + " [113, 128, 157]],\n", + "\n", + " [[155, 158, 176],\n", + " [156, 159, 177],\n", + " [155, 158, 176],\n", + " ...,\n", + " [117, 133, 162],\n", + " [119, 134, 163],\n", + " [117, 133, 162]]], dtype=uint8), 'image': array([[[108, 97, 74],\n", + " [108, 97, 74],\n", + " [ 43, 66, 60],\n", + " ...,\n", + " [151, 151, 151],\n", + " [156, 153, 150],\n", + " [157, 154, 151]],\n", + "\n", + " [[111, 91, 69],\n", + " [121, 101, 79],\n", + " [ 65, 89, 75],\n", + " ...,\n", + " [152, 151, 155],\n", + " [156, 152, 153],\n", + " [155, 151, 152]],\n", + "\n", + " [[106, 77, 56],\n", + " [125, 96, 75],\n", + " [ 81, 106, 87],\n", + " ...,\n", + " [152, 151, 160],\n", + " [151, 149, 156],\n", + " [147, 145, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 63, 50],\n", + " [ 67, 69, 56],\n", + " [ 63, 69, 50],\n", + " ...,\n", + " [ 66, 96, 99],\n", + " [ 66, 96, 99],\n", + " [ 65, 95, 98]],\n", + "\n", + " [[ 57, 60, 44],\n", + " [ 59, 62, 46],\n", + " [ 59, 62, 44],\n", + " ...,\n", + " [ 65, 95, 98],\n", + " [ 61, 91, 94],\n", + " [ 62, 92, 95]],\n", + "\n", + " [[ 51, 54, 38],\n", + " [ 47, 50, 34],\n", + " [ 46, 49, 31],\n", + " ...,\n", + " [ 65, 95, 98],\n", + " [ 60, 90, 95],\n", + " [ 62, 92, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0713017 , -0.8611215 , 2.037857 , -2.8476746 , -1.585627 ,\n", + " 3.2267466 , 0.5377978 , 0.14518696, -0.19194975, 0.7101977 ,\n", + " 0.7021796 , 0.03325085, 0.03817584, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00514286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00022857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 12],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[151, 154, 174],\n", + " [153, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [120, 136, 163]],\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [154, 157, 175],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [119, 135, 162]]], dtype=uint8), 'image': array([[[104, 103, 75],\n", + " [ 94, 93, 65],\n", + " [ 41, 71, 63],\n", + " ...,\n", + " [148, 153, 158],\n", + " [147, 152, 157],\n", + " [148, 153, 158]],\n", + "\n", + " [[116, 106, 78],\n", + " [106, 96, 68],\n", + " [ 58, 91, 74],\n", + " ...,\n", + " [146, 152, 154],\n", + " [147, 152, 157],\n", + " [146, 151, 156]],\n", + "\n", + " [[114, 90, 67],\n", + " [114, 90, 67],\n", + " [ 69, 101, 80],\n", + " ...,\n", + " [147, 152, 157],\n", + " [143, 148, 153],\n", + " [141, 146, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 71, 46],\n", + " [ 67, 77, 52],\n", + " [ 66, 73, 51],\n", + " ...,\n", + " [ 67, 96, 104],\n", + " [ 63, 95, 102],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 53, 63, 38],\n", + " [ 55, 65, 40],\n", + " [ 53, 60, 36],\n", + " ...,\n", + " [ 66, 96, 101],\n", + " [ 64, 96, 103],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 46, 56, 31],\n", + " [ 47, 57, 32],\n", + " [ 43, 50, 26],\n", + " ...,\n", + " [ 66, 94, 100],\n", + " [ 63, 92, 102],\n", + " [ 65, 94, 104]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.046966 , -0.8571275 , 2.026507 , -2.8399389 , -1.5878781 ,\n", + " 3.254046 , 0.5379952 , 0.15803353, -0.19198048, 0.71127206,\n", + " 0.7010918 , 0.03336547, 0.03806647, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, 0.0000000e+00, -5.7142857e-05], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 16, 14],\n", + " [ 8, 9, 7],\n", + " [ 7, 8, 5],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 9, 8, 4],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [153, 155, 175],\n", + " ...,\n", + " [121, 135, 160],\n", + " [121, 134, 162],\n", + " [121, 134, 162]],\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [154, 156, 176],\n", + " ...,\n", + " [116, 130, 155],\n", + " [115, 128, 156],\n", + " [117, 131, 158]],\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [154, 156, 176],\n", + " ...,\n", + " [114, 128, 153],\n", + " [114, 127, 155],\n", + " [116, 130, 157]]], dtype=uint8), 'image': array([[[104, 102, 83],\n", + " [ 95, 93, 74],\n", + " [ 44, 68, 78],\n", + " ...,\n", + " [151, 154, 155],\n", + " [150, 153, 154],\n", + " [150, 153, 154]],\n", + "\n", + " [[110, 96, 77],\n", + " [114, 100, 81],\n", + " [ 54, 80, 80],\n", + " ...,\n", + " [147, 153, 155],\n", + " [147, 153, 155],\n", + " [148, 154, 156]],\n", + "\n", + " [[109, 81, 64],\n", + " [130, 102, 85],\n", + " [ 73, 100, 91],\n", + " ...,\n", + " [146, 151, 158],\n", + " [145, 151, 153],\n", + " [140, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 68, 45],\n", + " [ 66, 70, 47],\n", + " [ 64, 68, 47],\n", + " ...,\n", + " [ 61, 94, 98],\n", + " [ 63, 96, 100],\n", + " [ 63, 96, 100]],\n", + "\n", + " [[ 57, 61, 36],\n", + " [ 58, 62, 37],\n", + " [ 55, 62, 36],\n", + " ...,\n", + " [ 64, 97, 101],\n", + " [ 64, 97, 101],\n", + " [ 61, 94, 98]],\n", + "\n", + " [[ 52, 57, 29],\n", + " [ 50, 55, 27],\n", + " [ 46, 51, 23],\n", + " ...,\n", + " [ 65, 98, 102],\n", + " [ 64, 97, 101],\n", + " [ 61, 94, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.02317 , -0.8530019 , 2.0143075 , -2.8314588 , -1.5901531 ,\n", + " 3.280961 , 0.5382555 , 0.17071289, -0.1919237 , 0.7123984 ,\n", + " 0.69994664, 0.03346113, 0.0379937 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.004], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [120, 136, 163],\n", + " [121, 137, 164],\n", + " [121, 137, 164]],\n", + "\n", + " [[150, 153, 173],\n", + " [151, 154, 174],\n", + " [151, 155, 172],\n", + " ...,\n", + " [121, 137, 164],\n", + " [122, 138, 165],\n", + " [123, 139, 166]],\n", + "\n", + " [[150, 153, 173],\n", + " [151, 154, 174],\n", + " [153, 156, 173],\n", + " ...,\n", + " [123, 139, 166],\n", + " [124, 140, 167],\n", + " [124, 140, 167]]], dtype=uint8), 'image': array([[[101, 98, 84],\n", + " [106, 103, 89],\n", + " [ 37, 68, 68],\n", + " ...,\n", + " [148, 156, 153],\n", + " [147, 155, 152],\n", + " [147, 155, 152]],\n", + "\n", + " [[111, 98, 85],\n", + " [118, 105, 92],\n", + " [ 46, 80, 75],\n", + " ...,\n", + " [148, 156, 155],\n", + " [147, 153, 151],\n", + " [148, 154, 152]],\n", + "\n", + " [[105, 80, 71],\n", + " [122, 97, 88],\n", + " [ 61, 93, 84],\n", + " ...,\n", + " [148, 153, 158],\n", + " [143, 149, 151],\n", + " [141, 147, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 57, 68, 47],\n", + " [ 63, 74, 53],\n", + " [ 59, 75, 52],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 59, 87, 93],\n", + " [ 56, 84, 90]],\n", + "\n", + " [[ 50, 62, 38],\n", + " [ 49, 61, 37],\n", + " [ 48, 60, 36],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 59, 87, 93],\n", + " [ 57, 85, 91]],\n", + "\n", + " [[ 47, 57, 34],\n", + " [ 47, 57, 34],\n", + " [ 41, 53, 29],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 56, 86, 89],\n", + " [ 55, 85, 88]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9994097 , -0.84833604, 2.0012221 , -2.822284 , -1.5922841 ,\n", + " 3.3077435 , 0.5384106 , 0.18353929, -0.19198695, 0.7134904 ,\n", + " 0.6988444 , 0.03347312, 0.0377824 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[152, 156, 178],\n", + " [152, 156, 178],\n", + " [154, 156, 176],\n", + " ...,\n", + " [116, 132, 159],\n", + " [117, 133, 160],\n", + " [117, 133, 160]],\n", + "\n", + " [[152, 156, 178],\n", + " [153, 157, 179],\n", + " [155, 157, 177],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[152, 156, 178],\n", + " [153, 157, 179],\n", + " [155, 157, 177],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]]], dtype=uint8), 'image': array([[[102, 95, 74],\n", + " [107, 100, 79],\n", + " [ 37, 68, 66],\n", + " ...,\n", + " [147, 152, 159],\n", + " [152, 151, 160],\n", + " [151, 150, 159]],\n", + "\n", + " [[109, 93, 72],\n", + " [121, 105, 84],\n", + " [ 52, 84, 75],\n", + " ...,\n", + " [147, 152, 159],\n", + " [150, 152, 160],\n", + " [145, 147, 155]],\n", + "\n", + " [[110, 82, 63],\n", + " [124, 96, 77],\n", + " [ 66, 99, 84],\n", + " ...,\n", + " [147, 152, 161],\n", + " [147, 149, 157],\n", + " [144, 146, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 57, 65, 50],\n", + " [ 62, 70, 55],\n", + " [ 64, 69, 57],\n", + " ...,\n", + " [ 86, 118, 125],\n", + " [ 80, 114, 120],\n", + " [ 67, 101, 107]],\n", + "\n", + " [[ 56, 62, 45],\n", + " [ 57, 63, 46],\n", + " [ 55, 57, 44],\n", + " ...,\n", + " [ 67, 99, 108],\n", + " [ 77, 111, 119],\n", + " [ 73, 107, 115]],\n", + "\n", + " [[ 52, 55, 37],\n", + " [ 51, 54, 36],\n", + " [ 45, 47, 34],\n", + " ...,\n", + " [ 60, 92, 101],\n", + " [ 59, 93, 101],\n", + " [ 70, 104, 112]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9757934 , -0.84310704, 1.9867959 , -2.8126876 , -1.5945233 ,\n", + " 3.334057 , 0.53880805, 0.19645281, -0.19202068, 0.71446204,\n", + " 0.69784665, 0.03360009, 0.03775069, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 9, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[151, 155, 175],\n", + " [151, 155, 175],\n", + " [149, 154, 174],\n", + " ...,\n", + " [125, 138, 161],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[148, 153, 173],\n", + " [148, 153, 173],\n", + " [148, 153, 173],\n", + " ...,\n", + " [126, 139, 162],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[148, 153, 173],\n", + " [147, 152, 172],\n", + " [149, 154, 174],\n", + " ...,\n", + " [127, 140, 163],\n", + " [127, 140, 163],\n", + " [127, 140, 163]]], dtype=uint8), 'image': array([[[107, 95, 75],\n", + " [108, 96, 76],\n", + " [ 49, 68, 72],\n", + " ...,\n", + " [155, 157, 163],\n", + " [153, 152, 159],\n", + " [152, 151, 158]],\n", + "\n", + " [[115, 95, 73],\n", + " [122, 102, 80],\n", + " [ 62, 85, 79],\n", + " ...,\n", + " [152, 154, 162],\n", + " [151, 153, 159],\n", + " [149, 151, 157]],\n", + "\n", + " [[118, 85, 68],\n", + " [128, 95, 78],\n", + " [ 78, 102, 90],\n", + " ...,\n", + " [148, 153, 162],\n", + " [148, 150, 158],\n", + " [145, 147, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 65, 51],\n", + " [ 67, 72, 58],\n", + " [ 64, 69, 55],\n", + " ...,\n", + " [ 83, 111, 126],\n", + " [ 89, 121, 132],\n", + " [ 82, 114, 125]],\n", + "\n", + " [[ 55, 61, 42],\n", + " [ 53, 59, 40],\n", + " [ 54, 57, 41],\n", + " ...,\n", + " [ 62, 90, 105],\n", + " [ 73, 102, 112],\n", + " [ 82, 111, 121]],\n", + "\n", + " [[ 48, 50, 32],\n", + " [ 52, 54, 36],\n", + " [ 47, 51, 30],\n", + " ...,\n", + " [ 58, 90, 101],\n", + " [ 60, 89, 99],\n", + " [ 58, 87, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9526203 , -0.8375662 , 1.9717474 , -2.8025048 , -1.5965704 ,\n", + " 3.3599634 , 0.539095 , 0.20931895, -0.1921031 , 0.7154446 ,\n", + " 0.6968444 , 0.03366781, 0.03759455, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, 0.0000000e+00, -5.7142857e-05], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 14, 10],\n", + " [ 10, 12, 7],\n", + " [ 8, 10, 5],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 7],\n", + " [ 11, 10, 8]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[145, 151, 173],\n", + " [145, 151, 173],\n", + " [144, 150, 172],\n", + " ...,\n", + " [125, 140, 163],\n", + " [125, 140, 163],\n", + " [125, 140, 163]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [125, 140, 163],\n", + " [124, 139, 162],\n", + " [123, 138, 161]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [121, 136, 158],\n", + " [121, 136, 158],\n", + " [121, 136, 158]]], dtype=uint8), 'image': array([[[103, 96, 77],\n", + " [102, 95, 76],\n", + " [ 50, 74, 74],\n", + " ...,\n", + " [150, 155, 160],\n", + " [150, 155, 160],\n", + " [150, 155, 160]],\n", + "\n", + " [[115, 97, 77],\n", + " [112, 94, 74],\n", + " [ 57, 85, 72],\n", + " ...,\n", + " [152, 154, 162],\n", + " [147, 152, 157],\n", + " [147, 152, 157]],\n", + "\n", + " [[120, 91, 68],\n", + " [123, 94, 71],\n", + " [ 73, 102, 82],\n", + " ...,\n", + " [148, 152, 164],\n", + " [143, 148, 155],\n", + " [142, 147, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 68, 51],\n", + " [ 66, 72, 55],\n", + " [ 62, 67, 53],\n", + " ...,\n", + " [ 87, 121, 129],\n", + " [ 74, 108, 116],\n", + " [ 66, 100, 108]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 57, 64, 40],\n", + " [ 53, 58, 44],\n", + " ...,\n", + " [ 85, 119, 127],\n", + " [ 87, 121, 131],\n", + " [ 78, 112, 122]],\n", + "\n", + " [[ 55, 62, 40],\n", + " [ 48, 55, 33],\n", + " [ 45, 51, 34],\n", + " ...,\n", + " [ 64, 98, 104],\n", + " [ 75, 107, 116],\n", + " [ 82, 114, 123]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9290407 , -0.83140117, 1.9549513 , -2.7911823 , -1.5989178 ,\n", + " 3.386706 , 0.53953487, 0.22216113, -0.19215716, 0.7165386 ,\n", + " 0.6957205 , 0.0336923 , 0.03755271, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00057143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 14, 11],\n", + " [ 8, 12, 9],\n", + " [ 8, 10, 5],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " [[ 8, 12, 9],\n", + " [ 8, 12, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[152, 156, 176],\n", + " [152, 156, 176],\n", + " [151, 155, 175],\n", + " ...,\n", + " [126, 139, 162],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[149, 154, 174],\n", + " [149, 154, 174],\n", + " [151, 155, 175],\n", + " ...,\n", + " [123, 135, 158],\n", + " [123, 135, 158],\n", + " [122, 134, 157]],\n", + "\n", + " [[148, 153, 173],\n", + " [148, 153, 173],\n", + " [149, 154, 174],\n", + " ...,\n", + " [120, 133, 156],\n", + " [119, 132, 155],\n", + " [120, 133, 156]]], dtype=uint8), 'image': array([[[107, 95, 77],\n", + " [108, 96, 78],\n", + " [ 49, 73, 73],\n", + " ...,\n", + " [153, 156, 157],\n", + " [152, 155, 156],\n", + " [150, 153, 154]],\n", + "\n", + " [[115, 94, 77],\n", + " [120, 99, 82],\n", + " [ 57, 84, 77],\n", + " ...,\n", + " [150, 158, 159],\n", + " [150, 156, 158],\n", + " [147, 153, 155]],\n", + "\n", + " [[112, 82, 64],\n", + " [125, 95, 77],\n", + " [ 75, 104, 86],\n", + " ...,\n", + " [147, 154, 158],\n", + " [142, 149, 153],\n", + " [140, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 72, 60],\n", + " [ 67, 77, 65],\n", + " [ 62, 71, 62],\n", + " ...,\n", + " [ 64, 98, 108],\n", + " [ 66, 95, 105],\n", + " [ 66, 95, 105]],\n", + "\n", + " [[ 56, 64, 49],\n", + " [ 55, 63, 48],\n", + " [ 54, 61, 53],\n", + " ...,\n", + " [ 72, 106, 114],\n", + " [ 67, 99, 106],\n", + " [ 66, 98, 105]],\n", + "\n", + " [[ 50, 56, 39],\n", + " [ 48, 54, 37],\n", + " [ 39, 47, 34],\n", + " ...,\n", + " [ 89, 121, 128],\n", + " [ 87, 119, 126],\n", + " [ 74, 106, 113]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9072087 , -0.82534486, 1.9385734 , -2.780183 , -1.6008447 ,\n", + " 3.4115925 , 0.5398998 , 0.23507659, -0.19219233, 0.7176641 ,\n", + " 0.6945623 , 0.03376771, 0.03743221, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00361905, 0. , 0.00742857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01845714, -0.00131429, -0.01542857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 10],\n", + " [ 10, 11, 9],\n", + " [ 10, 12, 7],\n", + " ...,\n", + " [ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 5, 5, 3],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [143, 149, 171],\n", + " ...,\n", + " [114, 127, 155],\n", + " [114, 127, 155],\n", + " [114, 127, 155]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [117, 131, 158],\n", + " [117, 131, 158],\n", + " [117, 131, 158]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [140, 147, 168],\n", + " ...,\n", + " [121, 134, 162],\n", + " [121, 134, 162],\n", + " [121, 134, 162]]], dtype=uint8), 'image': array([[[109, 101, 77],\n", + " [110, 102, 78],\n", + " [ 47, 70, 66],\n", + " ...,\n", + " [150, 157, 165],\n", + " [149, 156, 162],\n", + " [148, 155, 161]],\n", + "\n", + " [[117, 101, 80],\n", + " [114, 98, 77],\n", + " [ 61, 86, 77],\n", + " ...,\n", + " [146, 156, 163],\n", + " [146, 156, 161],\n", + " [146, 156, 161]],\n", + "\n", + " [[108, 83, 65],\n", + " [118, 93, 75],\n", + " [ 77, 105, 90],\n", + " ...,\n", + " [146, 155, 165],\n", + " [142, 152, 157],\n", + " [143, 153, 158]],\n", + "\n", + " ...,\n", + "\n", + " [[ 55, 70, 43],\n", + " [ 59, 74, 47],\n", + " [ 58, 73, 46],\n", + " ...,\n", + " [ 61, 91, 96],\n", + " [ 63, 95, 102],\n", + " [ 60, 92, 99]],\n", + "\n", + " [[ 50, 62, 38],\n", + " [ 52, 64, 40],\n", + " [ 51, 62, 32],\n", + " ...,\n", + " [ 58, 91, 93],\n", + " [ 59, 93, 101],\n", + " [ 58, 92, 100]],\n", + "\n", + " [[ 48, 52, 31],\n", + " [ 50, 54, 33],\n", + " [ 45, 52, 26],\n", + " ...,\n", + " [ 67, 100, 102],\n", + " [ 60, 94, 102],\n", + " [ 60, 94, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8851256 , -0.8193839 , 1.9207363 , -2.767865 , -1.60294 ,\n", + " 3.4364538 , 0.5404305 , 0.24793711, -0.1918753 , 0.71865124,\n", + " 0.69353217, 0.03392792, 0.03744877, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.02457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.0012 , -0.00131429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 153, 175],\n", + " [149, 153, 175],\n", + " [151, 154, 174],\n", + " ...,\n", + " [119, 134, 159],\n", + " [123, 135, 160],\n", + " [123, 135, 160]],\n", + "\n", + " [[145, 149, 171],\n", + " [145, 149, 171],\n", + " [147, 149, 169],\n", + " ...,\n", + " [121, 135, 160],\n", + " [123, 135, 160],\n", + " [123, 135, 160]],\n", + "\n", + " [[146, 150, 172],\n", + " [145, 149, 171],\n", + " [147, 149, 169],\n", + " ...,\n", + " [119, 134, 159],\n", + " [123, 135, 160],\n", + " [123, 135, 160]]], dtype=uint8), 'image': array([[[110, 103, 82],\n", + " [104, 97, 76],\n", + " [ 46, 72, 70],\n", + " ...,\n", + " [149, 156, 162],\n", + " [149, 156, 162],\n", + " [150, 157, 163]],\n", + "\n", + " [[117, 102, 76],\n", + " [114, 99, 73],\n", + " [ 64, 92, 79],\n", + " ...,\n", + " [147, 154, 160],\n", + " [148, 155, 161],\n", + " [147, 154, 160]],\n", + "\n", + " [[114, 85, 60],\n", + " [124, 95, 70],\n", + " [ 75, 102, 84],\n", + " ...,\n", + " [145, 152, 158],\n", + " [143, 148, 157],\n", + " [143, 148, 157]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 71, 48],\n", + " [ 63, 73, 50],\n", + " [ 61, 69, 54],\n", + " ...,\n", + " [ 63, 93, 98],\n", + " [ 66, 96, 101],\n", + " [ 65, 95, 100]],\n", + "\n", + " [[ 55, 65, 38],\n", + " [ 54, 64, 37],\n", + " [ 54, 64, 41],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 98],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 47, 57, 32],\n", + " [ 47, 57, 32],\n", + " [ 43, 53, 28],\n", + " ...,\n", + " [ 65, 94, 104],\n", + " [ 62, 91, 101],\n", + " [ 61, 90, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8629944 , -0.83263665, 1.9082379 , -2.740589 , -1.6067833 ,\n", + " 3.4631937 , 0.53934747, 0.260093 , -0.18352933, 0.72030914,\n", + " 0.69181347, 0.03324209, 0.03799796, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00190476, 0.00457143, 0.03009524], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00245714, -0.00251429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 7, 10, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 169, 180],\n", + " [168, 169, 180],\n", + " [167, 168, 179],\n", + " ...,\n", + " [120, 136, 163],\n", + " [120, 137, 161],\n", + " [120, 137, 161]],\n", + "\n", + " [[166, 167, 178],\n", + " [166, 167, 178],\n", + " [167, 168, 179],\n", + " ...,\n", + " [121, 137, 164],\n", + " [123, 140, 165],\n", + " [122, 139, 163]],\n", + "\n", + " [[165, 165, 177],\n", + " [165, 165, 177],\n", + " [165, 165, 177],\n", + " ...,\n", + " [123, 139, 166],\n", + " [124, 141, 166],\n", + " [126, 142, 167]]], dtype=uint8), 'image': array([[[104, 100, 75],\n", + " [104, 100, 75],\n", + " [ 50, 75, 70],\n", + " ...,\n", + " [151, 156, 163],\n", + " [153, 158, 165],\n", + " [151, 156, 163]],\n", + "\n", + " [[114, 101, 77],\n", + " [110, 97, 73],\n", + " [ 61, 89, 76],\n", + " ...,\n", + " [153, 160, 166],\n", + " [154, 159, 166],\n", + " [148, 153, 160]],\n", + "\n", + " [[119, 92, 70],\n", + " [123, 96, 74],\n", + " [ 74, 106, 85],\n", + " ...,\n", + " [146, 158, 162],\n", + " [145, 152, 158],\n", + " [142, 149, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 55],\n", + " [ 71, 74, 58],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 61, 89, 104],\n", + " [ 63, 91, 106],\n", + " [ 65, 93, 108]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 50, 56, 39],\n", + " ...,\n", + " [ 63, 90, 103],\n", + " [ 63, 90, 103],\n", + " [ 63, 90, 103]],\n", + "\n", + " [[ 50, 56, 37],\n", + " [ 46, 52, 33],\n", + " [ 43, 49, 32],\n", + " ...,\n", + " [ 61, 88, 101],\n", + " [ 62, 89, 102],\n", + " [ 61, 88, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8570101 , -0.86013824, 1.9111109 , -2.715855 , -1.6078013 ,\n", + " 3.482787 , 0.5375394 , 0.26292184, -0.17109968, 0.72502565,\n", + " 0.68685734, 0.0334212 , 0.03804962, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00266667, 0.00457143, 0.02761905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00937143, -0.00274286, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 99, 125, 148],\n", + " [ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[100, 124, 147],\n", + " [100, 124, 147],\n", + " [100, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[163, 165, 178],\n", + " [163, 165, 178],\n", + " [163, 165, 178],\n", + " ...,\n", + " [124, 138, 163],\n", + " [125, 139, 166],\n", + " [125, 139, 166]],\n", + "\n", + " [[163, 165, 178],\n", + " [164, 166, 180],\n", + " [165, 167, 181],\n", + " ...,\n", + " [125, 140, 165],\n", + " [123, 137, 164],\n", + " [123, 137, 164]],\n", + "\n", + " [[163, 165, 178],\n", + " [163, 165, 178],\n", + " [164, 166, 180],\n", + " ...,\n", + " [125, 140, 165],\n", + " [125, 139, 166],\n", + " [125, 139, 166]]], dtype=uint8), 'image': array([[[107, 103, 94],\n", + " [102, 98, 89],\n", + " [ 39, 69, 74],\n", + " ...,\n", + " [142, 149, 155],\n", + " [141, 148, 154],\n", + " [141, 148, 154]],\n", + "\n", + " [[120, 105, 95],\n", + " [109, 94, 84],\n", + " [ 42, 76, 71],\n", + " ...,\n", + " [142, 149, 153],\n", + " [139, 146, 150],\n", + " [140, 147, 151]],\n", + "\n", + " [[116, 92, 78],\n", + " [120, 96, 82],\n", + " [ 57, 92, 82],\n", + " ...,\n", + " [144, 151, 155],\n", + " [141, 146, 151],\n", + " [141, 146, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 70, 49],\n", + " [ 71, 75, 54],\n", + " [ 67, 71, 50],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 100],\n", + " [ 62, 91, 101]],\n", + "\n", + " [[ 58, 62, 41],\n", + " [ 57, 61, 40],\n", + " [ 53, 59, 40],\n", + " ...,\n", + " [ 62, 96, 104],\n", + " [ 58, 90, 101],\n", + " [ 57, 89, 100]],\n", + "\n", + " [[ 52, 55, 37],\n", + " [ 50, 53, 35],\n", + " [ 45, 51, 34],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 57, 91, 101],\n", + " [ 59, 93, 103]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8583746 , -0.8919328 , 1.9205203 , -2.6912768 , -1.6058615 ,\n", + " 3.5005364 , 0.5345636 , 0.26166272, -0.15828562, 0.73158187,\n", + " 0.67996943, 0.03315829, 0.03647007, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0032381 , 0.008 , 0.02761905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.0132 , -0.00371429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[106, 130, 153],\n", + " [106, 130, 153],\n", + " [106, 130, 153],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 4, 3, 1],\n", + " [ 4, 3, 1]],\n", + "\n", + " [[106, 130, 153],\n", + " [106, 130, 153],\n", + " [106, 130, 153],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[108, 129, 153],\n", + " [108, 129, 153],\n", + " [108, 129, 153],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [167, 164, 178],\n", + " ...,\n", + " [128, 143, 165],\n", + " [129, 143, 168],\n", + " [129, 143, 168]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [167, 164, 178],\n", + " ...,\n", + " [128, 143, 165],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [166, 163, 177],\n", + " ...,\n", + " [126, 142, 164],\n", + " [126, 141, 166],\n", + " [128, 142, 167]]], dtype=uint8), 'image': array([[[104, 101, 87],\n", + " [ 95, 92, 78],\n", + " [ 33, 69, 75],\n", + " ...,\n", + " [212, 215, 214],\n", + " [207, 207, 207],\n", + " [204, 204, 204]],\n", + "\n", + " [[109, 97, 79],\n", + " [115, 103, 85],\n", + " [ 50, 86, 84],\n", + " ...,\n", + " [212, 215, 214],\n", + " [208, 207, 207],\n", + " [205, 204, 204]],\n", + "\n", + " [[109, 85, 71],\n", + " [122, 98, 84],\n", + " [ 73, 104, 100],\n", + " ...,\n", + " [209, 212, 211],\n", + " [208, 208, 208],\n", + " [206, 206, 206]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 70, 55],\n", + " [ 67, 75, 60],\n", + " [ 66, 72, 55],\n", + " ...,\n", + " [ 61, 90, 102],\n", + " [ 62, 90, 105],\n", + " [ 62, 90, 105]],\n", + "\n", + " [[ 59, 65, 46],\n", + " [ 58, 64, 45],\n", + " [ 51, 57, 40],\n", + " ...,\n", + " [ 62, 91, 101],\n", + " [ 62, 91, 103],\n", + " [ 59, 88, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 49, 55, 36],\n", + " [ 43, 49, 30],\n", + " ...,\n", + " [ 62, 91, 101],\n", + " [ 60, 89, 101],\n", + " [ 60, 89, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.866813 , -0.92632943, 1.9351735 , -2.669913 , -1.602569 ,\n", + " 3.5102549 , 0.5314049 , 0.25643107, -0.14546628, 0.73773456,\n", + " 0.67338413, 0.0331663 , 0.03466382, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.01561905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01651429, -0.0068 , -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[116, 138, 160],\n", + " [115, 137, 158],\n", + " [115, 137, 158],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[115, 137, 158],\n", + " [113, 136, 157],\n", + " [113, 136, 157],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[111, 135, 159],\n", + " [111, 135, 159],\n", + " [111, 135, 159],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 168, 182],\n", + " [168, 168, 182],\n", + " [167, 167, 181],\n", + " ...,\n", + " [127, 144, 168],\n", + " [129, 143, 168],\n", + " [129, 143, 168]],\n", + "\n", + " [[168, 168, 182],\n", + " [167, 167, 181],\n", + " [169, 169, 183],\n", + " ...,\n", + " [128, 145, 169],\n", + " [130, 144, 169],\n", + " [130, 144, 169]],\n", + "\n", + " [[168, 168, 182],\n", + " [168, 168, 182],\n", + " [167, 167, 181],\n", + " ...,\n", + " [129, 146, 170],\n", + " [131, 145, 170],\n", + " [130, 144, 169]]], dtype=uint8), 'image': array([[[102, 100, 77],\n", + " [104, 102, 79],\n", + " [ 38, 72, 69],\n", + " ...,\n", + " [181, 181, 183],\n", + " [182, 182, 182],\n", + " [185, 185, 185]],\n", + "\n", + " [[116, 100, 79],\n", + " [122, 106, 85],\n", + " [ 48, 83, 73],\n", + " ...,\n", + " [180, 180, 184],\n", + " [179, 182, 185],\n", + " [184, 187, 190]],\n", + "\n", + " [[118, 88, 72],\n", + " [133, 103, 87],\n", + " [ 74, 107, 92],\n", + " ...,\n", + " [176, 176, 180],\n", + " [179, 181, 187],\n", + " [185, 187, 193]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 77, 51],\n", + " [ 71, 78, 52],\n", + " [ 68, 70, 52],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 91, 99],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 58, 64, 45],\n", + " [ 58, 61, 45],\n", + " ...,\n", + " [ 64, 92, 98],\n", + " [ 62, 91, 99],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 57, 59, 46],\n", + " [ 52, 54, 41],\n", + " [ 48, 50, 37],\n", + " ...,\n", + " [ 63, 92, 100],\n", + " [ 63, 90, 99],\n", + " [ 63, 90, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8797753 , -0.962189 , 1.9533105 , -2.6487052 , -1.5976962 ,\n", + " 3.5144436 , 0.52728325, 0.24874139, -0.1332232 , 0.7435012 ,\n", + " 0.66718084, 0.03253057, 0.03189772, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00361905, 0.00933333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.0172 , -0.00937143, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[129, 148, 166],\n", + " [129, 148, 166],\n", + " [129, 148, 166],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[129, 148, 166],\n", + " [129, 148, 166],\n", + " [128, 147, 165],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[129, 148, 166],\n", + " [129, 148, 166],\n", + " [128, 147, 165],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 165, 180],\n", + " [168, 165, 180],\n", + " [168, 165, 180],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [168, 165, 180],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [169, 167, 181],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [126, 141, 166]]], dtype=uint8), 'image': array([[[105, 98, 79],\n", + " [108, 101, 82],\n", + " [ 42, 71, 68],\n", + " ...,\n", + " [176, 175, 177],\n", + " [170, 168, 175],\n", + " [168, 166, 173]],\n", + "\n", + " [[119, 100, 83],\n", + " [123, 104, 87],\n", + " [ 54, 84, 76],\n", + " ...,\n", + " [175, 174, 176],\n", + " [173, 172, 176],\n", + " [169, 168, 172]],\n", + "\n", + " [[123, 92, 81],\n", + " [123, 92, 81],\n", + " [ 73, 104, 90],\n", + " ...,\n", + " [175, 174, 174],\n", + " [175, 171, 174],\n", + " [172, 168, 171]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 78, 56],\n", + " [ 71, 78, 56],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 65, 97, 104],\n", + " [ 66, 96, 99],\n", + " [ 62, 92, 95]],\n", + "\n", + " [[ 55, 59, 36],\n", + " [ 59, 63, 40],\n", + " [ 59, 62, 44],\n", + " ...,\n", + " [ 64, 96, 103],\n", + " [ 63, 93, 98],\n", + " [ 62, 92, 97]],\n", + "\n", + " [[ 52, 55, 34],\n", + " [ 52, 55, 34],\n", + " [ 46, 50, 29],\n", + " ...,\n", + " [ 64, 96, 105],\n", + " [ 63, 92, 100],\n", + " [ 61, 90, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8943539 , -1.002066 , 1.9794574 , -2.6350296 , -1.5959238 ,\n", + " 3.5113556 , 0.52145064, 0.23904608, -0.12084077, 0.7473314 ,\n", + " 0.6629031 , 0.0326565 , 0.0314473 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00057143, -0.00780952, 0.0072381 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01051429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[117, 136, 156],\n", + " [119, 137, 157],\n", + " [119, 137, 157],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[119, 137, 157],\n", + " [119, 137, 157],\n", + " [119, 137, 157],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[120, 138, 158],\n", + " [120, 138, 158],\n", + " [120, 138, 158],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [173, 169, 181],\n", + " ...,\n", + " [140, 146, 170],\n", + " [140, 146, 170],\n", + " [140, 146, 170]],\n", + "\n", + " [[175, 171, 183],\n", + " [176, 172, 184],\n", + " [175, 171, 183],\n", + " ...,\n", + " [139, 145, 169],\n", + " [138, 144, 168],\n", + " [139, 145, 169]],\n", + "\n", + " [[176, 172, 184],\n", + " [176, 172, 184],\n", + " [175, 171, 183],\n", + " ...,\n", + " [138, 144, 168],\n", + " [139, 145, 169],\n", + " [139, 145, 169]]], dtype=uint8), 'image': array([[[108, 105, 71],\n", + " [106, 103, 69],\n", + " [ 41, 73, 62],\n", + " ...,\n", + " [129, 132, 133],\n", + " [129, 135, 135],\n", + " [127, 133, 133]],\n", + "\n", + " [[123, 108, 78],\n", + " [121, 106, 76],\n", + " [ 56, 89, 72],\n", + " ...,\n", + " [130, 130, 132],\n", + " [128, 134, 136],\n", + " [129, 135, 137]],\n", + "\n", + " [[112, 86, 57],\n", + " [124, 98, 69],\n", + " [ 69, 101, 80],\n", + " ...,\n", + " [128, 130, 136],\n", + " [129, 135, 137],\n", + " [133, 139, 141]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 71, 49],\n", + " [ 69, 76, 54],\n", + " [ 67, 73, 56],\n", + " ...,\n", + " [ 60, 94, 102],\n", + " [ 59, 93, 103],\n", + " [ 59, 93, 103]],\n", + "\n", + " [[ 57, 64, 40],\n", + " [ 53, 60, 36],\n", + " [ 55, 58, 40],\n", + " ...,\n", + " [ 58, 93, 106],\n", + " [ 56, 91, 104],\n", + " [ 57, 92, 105]],\n", + "\n", + " [[ 53, 56, 35],\n", + " [ 51, 54, 33],\n", + " [ 47, 50, 32],\n", + " ...,\n", + " [ 58, 93, 106],\n", + " [ 57, 92, 107],\n", + " [ 56, 91, 106]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9088843 , -1.0461868 , 2.012751 , -2.6265728 , -1.5953134 ,\n", + " 3.5029898 , 0.5132325 , 0.2287732 , -0.10842803, 0.74936545,\n", + " 0.66052914, 0.03346506, 0.03213668, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00019048, -0.00876191, 0.00685714], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01068571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[119, 134, 152],\n", + " [119, 134, 152],\n", + " [119, 134, 152],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[118, 132, 151],\n", + " [118, 132, 151],\n", + " [118, 132, 151],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[118, 132, 151],\n", + " [118, 132, 151],\n", + " [118, 132, 151],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [137, 144, 170],\n", + " [138, 145, 172],\n", + " [138, 145, 172]],\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [140, 147, 173],\n", + " [138, 145, 172],\n", + " [138, 145, 172]],\n", + "\n", + " [[173, 169, 181],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [141, 148, 174],\n", + " [138, 145, 172],\n", + " [138, 145, 172]]], dtype=uint8), 'image': array([[[109, 104, 73],\n", + " [109, 104, 73],\n", + " [ 49, 76, 67],\n", + " ...,\n", + " [103, 104, 95],\n", + " [106, 103, 98],\n", + " [107, 104, 99]],\n", + "\n", + " [[119, 102, 73],\n", + " [116, 99, 70],\n", + " [ 57, 85, 70],\n", + " ...,\n", + " [104, 105, 96],\n", + " [105, 106, 97],\n", + " [105, 106, 97]],\n", + "\n", + " [[126, 99, 73],\n", + " [120, 93, 67],\n", + " [ 77, 106, 86],\n", + " ...,\n", + " [105, 106, 97],\n", + " [106, 107, 98],\n", + " [108, 109, 100]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 72, 51],\n", + " [ 74, 78, 57],\n", + " [ 73, 75, 62],\n", + " ...,\n", + " [ 63, 97, 107],\n", + " [ 61, 93, 102],\n", + " [ 60, 92, 101]],\n", + "\n", + " [[ 58, 65, 43],\n", + " [ 58, 65, 43],\n", + " [ 56, 61, 49],\n", + " ...,\n", + " [ 63, 97, 107],\n", + " [ 63, 95, 106],\n", + " [ 60, 92, 103]],\n", + "\n", + " [[ 54, 57, 41],\n", + " [ 52, 55, 39],\n", + " [ 45, 50, 36],\n", + " ...,\n", + " [ 60, 94, 104],\n", + " [ 64, 96, 107],\n", + " [ 60, 92, 103]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9227448 , -1.090619 , 2.0471349 , -2.6207187 , -1.5956005 ,\n", + " 3.4939177 , 0.5043737 , 0.21883176, -0.09658621, 0.750923 ,\n", + " 0.65860933, 0.03484912, 0.03367477, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00971429, 0.00666667], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01091429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 140, 158],\n", + " [119, 138, 155],\n", + " [121, 140, 158],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[119, 138, 155],\n", + " [119, 138, 155],\n", + " [119, 138, 155],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[119, 138, 155],\n", + " [119, 138, 155],\n", + " [119, 138, 155],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[166, 165, 181],\n", + " [166, 165, 181],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [136, 146, 172],\n", + " [138, 147, 173]],\n", + "\n", + " [[165, 164, 180],\n", + " [165, 164, 180],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [138, 147, 173],\n", + " [136, 146, 172]],\n", + "\n", + " [[165, 164, 180],\n", + " [165, 164, 180],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [136, 146, 172],\n", + " [138, 147, 173]]], dtype=uint8), 'image': array([[[110, 103, 70],\n", + " [107, 100, 67],\n", + " [ 42, 72, 66],\n", + " ...,\n", + " [ 85, 82, 77],\n", + " [ 86, 83, 78],\n", + " [ 87, 84, 79]],\n", + "\n", + " [[124, 106, 74],\n", + " [117, 99, 67],\n", + " [ 57, 88, 74],\n", + " ...,\n", + " [ 85, 82, 77],\n", + " [ 87, 83, 76],\n", + " [ 90, 86, 79]],\n", + "\n", + " [[117, 90, 57],\n", + " [125, 98, 65],\n", + " [ 70, 102, 81],\n", + " ...,\n", + " [ 84, 81, 76],\n", + " [ 85, 83, 75],\n", + " [ 87, 85, 77]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 55],\n", + " [ 72, 75, 59],\n", + " [ 69, 72, 54],\n", + " ...,\n", + " [ 59, 93, 103],\n", + " [ 60, 94, 102],\n", + " [ 59, 93, 101]],\n", + "\n", + " [[ 60, 62, 49],\n", + " [ 60, 62, 49],\n", + " [ 58, 61, 45],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 63, 92, 102],\n", + " [ 63, 92, 102]],\n", + "\n", + " [[ 56, 57, 46],\n", + " [ 52, 53, 42],\n", + " [ 47, 49, 36],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 61, 90, 98],\n", + " [ 62, 91, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9372277 , -1.138132 , 2.0828366 , -2.614289 , -1.5960315 ,\n", + " 3.4834929 , 0.4947406 , 0.20868209, -0.08435462, 0.7522201 ,\n", + " 0.656926 , 0.03664842, 0.03563798, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01047619, 0.00609524], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.0112 , -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[126, 144, 159],\n", + " [126, 144, 159],\n", + " [125, 143, 158],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[128, 145, 161],\n", + " [126, 144, 159],\n", + " [125, 143, 158],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3]],\n", + "\n", + " [[130, 145, 161],\n", + " [129, 144, 159],\n", + " [126, 144, 159],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 175, 189],\n", + " [175, 175, 189],\n", + " [174, 174, 188],\n", + " ...,\n", + " [145, 151, 166],\n", + " [145, 150, 168],\n", + " [145, 150, 168]],\n", + "\n", + " [[174, 174, 188],\n", + " [174, 174, 188],\n", + " [174, 174, 188],\n", + " ...,\n", + " [146, 152, 167],\n", + " [145, 150, 168],\n", + " [145, 150, 168]],\n", + "\n", + " [[173, 173, 187],\n", + " [174, 174, 188],\n", + " [174, 174, 188],\n", + " ...,\n", + " [146, 152, 167],\n", + " [146, 151, 169],\n", + " [145, 150, 168]]], dtype=uint8), 'image': array([[[101, 102, 89],\n", + " [ 97, 98, 85],\n", + " [ 48, 83, 84],\n", + " ...,\n", + " [ 90, 87, 82],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " [[112, 103, 84],\n", + " [114, 105, 86],\n", + " [ 41, 79, 73],\n", + " ...,\n", + " [ 84, 84, 78],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " [[114, 91, 72],\n", + " [134, 111, 92],\n", + " [ 60, 97, 84],\n", + " ...,\n", + " [ 89, 86, 81],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 70, 51],\n", + " [ 69, 75, 56],\n", + " [ 63, 72, 54],\n", + " ...,\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98],\n", + " [ 60, 89, 97]],\n", + "\n", + " [[ 56, 65, 47],\n", + " [ 55, 64, 46],\n", + " [ 54, 65, 46],\n", + " ...,\n", + " [ 63, 95, 104],\n", + " [ 59, 88, 96],\n", + " [ 60, 89, 97]],\n", + "\n", + " [[ 52, 58, 41],\n", + " [ 50, 56, 39],\n", + " [ 49, 58, 40],\n", + " ...,\n", + " [ 58, 90, 99],\n", + " [ 56, 88, 95],\n", + " [ 57, 89, 96]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9522376 , -1.1864475 , 2.1171021 , -2.6065881 , -1.5964864 ,\n", + " 3.4725657 , 0.48520797, 0.19863266, -0.07217506, 0.75350314,\n", + " 0.65520203, 0.03879603, 0.03792322, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01314286, 0.00419048], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01691429, -0.01308571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[142, 154, 166],\n", + " [142, 154, 166],\n", + " [142, 154, 166],\n", + " ...,\n", + " [ 9, 5, 3],\n", + " [ 8, 4, 0],\n", + " [ 10, 7, 3]],\n", + "\n", + " [[142, 154, 166],\n", + " [142, 154, 166],\n", + " [141, 153, 164],\n", + " ...,\n", + " [ 9, 5, 3],\n", + " [ 10, 7, 3],\n", + " [ 10, 7, 3]],\n", + "\n", + " [[141, 153, 164],\n", + " [141, 153, 164],\n", + " [141, 153, 164],\n", + " ...,\n", + " [ 9, 5, 1],\n", + " [ 10, 7, 3],\n", + " [ 10, 7, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[177, 180, 191],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [196, 212, 171],\n", + " [197, 213, 172],\n", + " [197, 213, 172]],\n", + "\n", + " [[177, 180, 191],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [198, 214, 173],\n", + " [198, 214, 173],\n", + " [197, 213, 172]],\n", + "\n", + " [[176, 179, 190],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [198, 214, 173],\n", + " [198, 214, 173],\n", + " [197, 213, 172]]], dtype=uint8), 'image': array([[[104, 98, 85],\n", + " [103, 97, 84],\n", + " [ 45, 72, 81],\n", + " ...,\n", + " [138, 136, 143],\n", + " [126, 119, 130],\n", + " [113, 106, 117]],\n", + "\n", + " [[114, 99, 87],\n", + " [114, 99, 87],\n", + " [ 52, 83, 83],\n", + " ...,\n", + " [132, 131, 142],\n", + " [125, 120, 130],\n", + " [113, 108, 118]],\n", + "\n", + " [[123, 97, 84],\n", + " [122, 96, 83],\n", + " [ 70, 99, 96],\n", + " ...,\n", + " [133, 134, 149],\n", + " [116, 113, 129],\n", + " [102, 99, 115]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 54],\n", + " [ 71, 74, 58],\n", + " [ 67, 73, 56],\n", + " ...,\n", + " [ 56, 90, 100],\n", + " [ 54, 83, 95],\n", + " [ 61, 90, 102]],\n", + "\n", + " [[ 58, 61, 45],\n", + " [ 59, 62, 46],\n", + " [ 60, 62, 49],\n", + " ...,\n", + " [ 58, 90, 101],\n", + " [ 58, 87, 99],\n", + " [ 54, 83, 95]],\n", + "\n", + " [[ 52, 54, 43],\n", + " [ 50, 52, 41],\n", + " [ 45, 47, 38],\n", + " ...,\n", + " [ 61, 93, 104],\n", + " [ 60, 92, 103],\n", + " [ 54, 86, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9681327 , -1.236956 , 2.1511526 , -2.5970871 , -1.5968579 ,\n", + " 3.4606204 , 0.4752485 , 0.18844554, -0.05988858, 0.7547231 ,\n", + " 0.65350336, 0.0411324 , 0.04042946, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01333333, 0.00342857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01662857, -0.01388571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 63, 60, 69],\n", + " [ 66, 63, 73],\n", + " [ 70, 69, 79],\n", + " ...,\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " [[ 81, 84, 95],\n", + " [ 86, 89, 100],\n", + " [ 87, 93, 103],\n", + " ...,\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4]],\n", + "\n", + " [[101, 110, 124],\n", + " [103, 112, 126],\n", + " [108, 117, 129],\n", + " ...,\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 188, 197],\n", + " [189, 188, 197],\n", + " [189, 188, 197],\n", + " ...,\n", + " [195, 211, 170],\n", + " [194, 212, 171],\n", + " [194, 212, 171]],\n", + "\n", + " [[188, 187, 196],\n", + " [188, 187, 196],\n", + " [188, 187, 196],\n", + " ...,\n", + " [197, 213, 172],\n", + " [196, 215, 173],\n", + " [194, 212, 171]],\n", + "\n", + " [[187, 186, 195],\n", + " [188, 187, 196],\n", + " [188, 187, 196],\n", + " ...,\n", + " [197, 213, 172],\n", + " [195, 214, 172],\n", + " [194, 212, 171]]], dtype=uint8), 'image': array([[[ 98, 101, 80],\n", + " [ 95, 98, 77],\n", + " [ 36, 74, 72],\n", + " ...,\n", + " [155, 157, 165],\n", + " [156, 158, 166],\n", + " [156, 158, 166]],\n", + "\n", + " [[107, 95, 77],\n", + " [114, 102, 84],\n", + " [ 53, 90, 81],\n", + " ...,\n", + " [151, 156, 165],\n", + " [151, 156, 165],\n", + " [151, 156, 165]],\n", + "\n", + " [[118, 92, 79],\n", + " [123, 97, 84],\n", + " [ 67, 102, 92],\n", + " ...,\n", + " [151, 155, 167],\n", + " [148, 152, 164],\n", + " [147, 151, 163]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 68, 53],\n", + " [ 74, 73, 58],\n", + " [ 70, 72, 56],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 63, 95, 102],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 65, 65, 43],\n", + " [ 63, 63, 41],\n", + " [ 58, 58, 38],\n", + " ...,\n", + " [ 60, 92, 99],\n", + " [ 64, 96, 103],\n", + " [ 63, 95, 102]],\n", + "\n", + " [[ 56, 56, 32],\n", + " [ 56, 56, 32],\n", + " [ 50, 50, 28],\n", + " ...,\n", + " [ 65, 94, 102],\n", + " [ 63, 95, 104],\n", + " [ 61, 93, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.98276 , -1.2880267 , 2.1854315 , -2.5885217 , -1.5973486 ,\n", + " 3.4484115 , 0.46462205, 0.17914946, -0.04822387, 0.75541085,\n", + " 0.65233696, 0.04393689, 0.04336553, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01409524, 0.00152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01582857, -0.01577143, -0.0192 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7],\n", + " [ 11, 7, 5]],\n", + "\n", + " [[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 11, 7, 5],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " [[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 11, 7, 5],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 191, 200],\n", + " [189, 191, 200],\n", + " [188, 190, 198],\n", + " ...,\n", + " [201, 218, 174],\n", + " [201, 218, 174],\n", + " [201, 218, 174]],\n", + "\n", + " [[188, 190, 198],\n", + " [188, 190, 198],\n", + " [187, 188, 197],\n", + " ...,\n", + " [199, 216, 173],\n", + " [201, 218, 174],\n", + " [201, 218, 174]],\n", + "\n", + " [[186, 187, 196],\n", + " [187, 188, 197],\n", + " [187, 188, 197],\n", + " ...,\n", + " [201, 218, 174],\n", + " [199, 216, 173],\n", + " [199, 216, 173]]], dtype=uint8), 'image': array([[[106, 103, 89],\n", + " [102, 99, 85],\n", + " [ 43, 73, 78],\n", + " ...,\n", + " [152, 164, 170],\n", + " [150, 160, 165],\n", + " [151, 161, 166]],\n", + "\n", + " [[115, 103, 87],\n", + " [116, 104, 88],\n", + " [ 55, 86, 82],\n", + " ...,\n", + " [152, 162, 169],\n", + " [150, 157, 163],\n", + " [150, 157, 163]],\n", + "\n", + " [[113, 87, 72],\n", + " [122, 96, 81],\n", + " [ 70, 101, 87],\n", + " ...,\n", + " [150, 160, 167],\n", + " [146, 153, 161],\n", + " [147, 154, 162]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 54],\n", + " [ 72, 75, 59],\n", + " [ 64, 72, 59],\n", + " ...,\n", + " [ 61, 93, 104],\n", + " [ 64, 96, 105],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 61, 64, 48],\n", + " [ 60, 63, 47],\n", + " [ 57, 62, 52],\n", + " ...,\n", + " [ 59, 93, 103],\n", + " [ 64, 96, 105],\n", + " [ 60, 92, 101]],\n", + "\n", + " [[ 54, 55, 44],\n", + " [ 52, 53, 42],\n", + " [ 48, 53, 41],\n", + " ...,\n", + " [ 60, 94, 104],\n", + " [ 60, 92, 101],\n", + " [ 61, 93, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9976504 , -1.3427528 , 2.2203817 , -2.5778692 , -1.5979112 ,\n", + " 3.4355202 , 0.45309877, 0.16995318, -0.03630768, 0.7559353 ,\n", + " 0.6512862 , 0.04700414, 0.04667673, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00114286, -0.01142857, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01417143, -0.01828571, -0.01594286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 11, 8, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 7, 4],\n", + " [ 13, 7, 4]],\n", + "\n", + " [[ 10, 7, 3],\n", + " [ 10, 7, 3],\n", + " [ 9, 5, 1],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 9, 5, 1],\n", + " [ 9, 5, 1],\n", + " [ 9, 5, 1],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [184, 190, 198],\n", + " ...,\n", + " [201, 220, 176],\n", + " [202, 221, 177],\n", + " [202, 221, 177]],\n", + "\n", + " [[183, 189, 197],\n", + " [184, 190, 198],\n", + " [183, 189, 197],\n", + " ...,\n", + " [202, 221, 177],\n", + " [201, 220, 176],\n", + " [201, 220, 176]],\n", + "\n", + " [[184, 190, 198],\n", + " [183, 189, 197],\n", + " [183, 189, 197],\n", + " ...,\n", + " [201, 220, 176],\n", + " [201, 220, 176],\n", + " [201, 220, 176]]], dtype=uint8), 'image': array([[[107, 101, 86],\n", + " [102, 96, 81],\n", + " [ 39, 66, 77],\n", + " ...,\n", + " [154, 161, 167],\n", + " [154, 161, 167],\n", + " [153, 160, 166]],\n", + "\n", + " [[118, 98, 88],\n", + " [122, 102, 92],\n", + " [ 63, 91, 95],\n", + " ...,\n", + " [153, 160, 166],\n", + " [154, 159, 168],\n", + " [155, 160, 169]],\n", + "\n", + " [[121, 90, 79],\n", + " [128, 97, 86],\n", + " [ 70, 94, 92],\n", + " ...,\n", + " [153, 158, 167],\n", + " [147, 151, 163],\n", + " [145, 149, 161]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 69, 60],\n", + " [ 67, 76, 67],\n", + " [ 65, 73, 60],\n", + " ...,\n", + " [ 60, 92, 101],\n", + " [ 59, 91, 100],\n", + " [ 61, 93, 102]],\n", + "\n", + " [[ 56, 64, 51],\n", + " [ 53, 61, 48],\n", + " [ 49, 57, 42],\n", + " ...,\n", + " [ 58, 90, 99],\n", + " [ 56, 88, 99],\n", + " [ 57, 89, 100]],\n", + "\n", + " [[ 53, 58, 46],\n", + " [ 51, 56, 44],\n", + " [ 43, 51, 36],\n", + " ...,\n", + " [ 57, 91, 99],\n", + " [ 57, 91, 101],\n", + " [ 58, 92, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0120497 , -1.400019 , 2.2567096 , -2.5662448 , -1.5984977 ,\n", + " 3.4220662 , 0.4402678 , 0.16121309, -0.02498595, 0.75610036,\n", + " 0.6506091 , 0.05005798, 0.05014284, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00533333, -0.01238095, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01262857, -0.01931429, -0.01217143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 6, 4, 2],\n", + " [ 8, 6, 4],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 5, 5, 3],\n", + " [ 5, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[190, 192, 201],\n", + " [190, 192, 201],\n", + " [189, 191, 200],\n", + " ...,\n", + " [203, 223, 178],\n", + " [204, 224, 180],\n", + " [204, 224, 180]],\n", + "\n", + " [[187, 188, 197],\n", + " [188, 190, 198],\n", + " [188, 190, 198],\n", + " ...,\n", + " [203, 223, 178],\n", + " [201, 220, 176],\n", + " [201, 220, 176]],\n", + "\n", + " [[187, 188, 197],\n", + " [187, 188, 197],\n", + " [188, 190, 198],\n", + " ...,\n", + " [203, 223, 178],\n", + " [202, 221, 177],\n", + " [201, 220, 176]]], dtype=uint8), 'image': array([[[110, 105, 85],\n", + " [109, 104, 84],\n", + " [ 40, 71, 69],\n", + " ...,\n", + " [152, 159, 165],\n", + " [150, 155, 164],\n", + " [150, 155, 164]],\n", + "\n", + " [[121, 104, 86],\n", + " [124, 107, 89],\n", + " [ 55, 87, 78],\n", + " ...,\n", + " [151, 161, 166],\n", + " [150, 155, 164],\n", + " [150, 155, 164]],\n", + "\n", + " [[111, 83, 64],\n", + " [127, 99, 80],\n", + " [ 76, 109, 92],\n", + " ...,\n", + " [147, 161, 167],\n", + " [146, 152, 163],\n", + " [141, 147, 158]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 74, 51],\n", + " [ 70, 80, 57],\n", + " [ 65, 75, 48],\n", + " ...,\n", + " [ 58, 92, 98],\n", + " [ 61, 93, 100],\n", + " [ 61, 93, 100]],\n", + "\n", + " [[ 55, 65, 42],\n", + " [ 55, 65, 42],\n", + " [ 54, 64, 39],\n", + " ...,\n", + " [ 64, 96, 105],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 48, 57, 37],\n", + " [ 44, 53, 33],\n", + " [ 44, 53, 33],\n", + " ...,\n", + " [ 63, 92, 104],\n", + " [ 66, 93, 104],\n", + " [ 64, 91, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.024355 , -1.4538416 , 2.2930508 , -2.5565522 , -1.5979952 ,\n", + " 3.4096293 , 0.42665708, 0.1540214 , -0.0161692 , 0.75591356,\n", + " 0.65040565, 0.05278223, 0.05273771, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0032381, 0. , 0.0047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00514286, -0.00582857, -0.00068571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[191, 195, 203],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [205, 222, 178],\n", + " [205, 222, 178]],\n", + "\n", + " [[191, 195, 203],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [204, 221, 177],\n", + " [204, 221, 177]],\n", + "\n", + " [[189, 193, 202],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [204, 221, 177],\n", + " [204, 221, 177]]], dtype=uint8), 'image': array([[[109, 105, 94],\n", + " [107, 103, 92],\n", + " [ 34, 61, 70],\n", + " ...,\n", + " [153, 161, 162],\n", + " [153, 160, 164],\n", + " [153, 160, 164]],\n", + "\n", + " [[120, 107, 94],\n", + " [120, 107, 94],\n", + " [ 51, 80, 79],\n", + " ...,\n", + " [150, 157, 163],\n", + " [152, 159, 167],\n", + " [150, 157, 165]],\n", + "\n", + " [[111, 85, 74],\n", + " [130, 104, 93],\n", + " [ 75, 105, 99],\n", + " ...,\n", + " [149, 155, 168],\n", + " [146, 154, 169],\n", + " [144, 152, 167]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 75, 60],\n", + " [ 68, 81, 66],\n", + " [ 71, 81, 67],\n", + " ...,\n", + " [ 64, 96, 107],\n", + " [ 60, 92, 101],\n", + " [ 57, 89, 98]],\n", + "\n", + " [[ 56, 67, 48],\n", + " [ 56, 67, 48],\n", + " [ 54, 62, 47],\n", + " ...,\n", + " [ 56, 88, 99],\n", + " [ 54, 85, 99],\n", + " [ 57, 88, 102]],\n", + "\n", + " [[ 50, 59, 39],\n", + " [ 49, 58, 38],\n", + " [ 46, 55, 35],\n", + " ...,\n", + " [ 56, 87, 101],\n", + " [ 57, 88, 104],\n", + " [ 60, 91, 107]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0366597 , -1.50608 , 2.3327332 , -2.5523531 , -1.5953492 ,\n", + " 3.3971448 , 0.41204247, 0.14764796, -0.00976385, 0.75563943,\n", + " 0.65026915, 0.05629408, 0.054681 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0., 0., -0.], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 5, 5, 3],\n", + " [ 3, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 5, 5, 3],\n", + " [ 5, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 13, 10, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[187, 194, 202],\n", + " [187, 194, 202],\n", + " [186, 193, 201],\n", + " ...,\n", + " [211, 225, 184],\n", + " [211, 225, 184],\n", + " [211, 225, 184]],\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [186, 193, 201],\n", + " ...,\n", + " [211, 225, 184],\n", + " [211, 225, 184],\n", + " [211, 225, 184]],\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [186, 193, 201],\n", + " ...,\n", + " [212, 226, 185],\n", + " [211, 225, 184],\n", + " [211, 225, 184]]], dtype=uint8), 'image': array([[[111, 103, 77],\n", + " [114, 106, 80],\n", + " [ 49, 75, 62],\n", + " ...,\n", + " [ 47, 53, 55],\n", + " [ 49, 48, 50],\n", + " [ 51, 50, 52]],\n", + "\n", + " [[118, 100, 78],\n", + " [119, 101, 79],\n", + " [ 61, 89, 74],\n", + " ...,\n", + " [ 49, 52, 51],\n", + " [ 57, 48, 51],\n", + " [ 58, 49, 52]],\n", + "\n", + " [[113, 86, 64],\n", + " [126, 99, 77],\n", + " [ 82, 111, 93],\n", + " ...,\n", + " [ 53, 53, 47],\n", + " [ 64, 49, 50],\n", + " [ 65, 50, 51]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 75, 56],\n", + " [ 70, 76, 57],\n", + " [ 69, 75, 56],\n", + " ...,\n", + " [ 63, 94, 108],\n", + " [ 66, 98, 109],\n", + " [ 66, 98, 109]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " [ 55, 61, 42],\n", + " ...,\n", + " [ 65, 96, 110],\n", + " [ 68, 97, 109],\n", + " [ 67, 96, 108]],\n", + "\n", + " [[ 48, 55, 33],\n", + " [ 50, 57, 35],\n", + " [ 49, 55, 36],\n", + " ...,\n", + " [ 65, 97, 108],\n", + " [ 68, 97, 109],\n", + " [ 67, 96, 108]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0452423 , -1.5262693 , 2.3518927 , -2.553457 , -1.5919966 ,\n", + " 3.3911233 , 0.405479 , 0.1441851 , -0.00875834, 0.75636166,\n", + " 0.6493327 , 0.05772375, 0.05433304, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 1.0, 'world_vector': array([0., 0., 0.], dtype=float32)}, 'is_first': False, 'is_last': True, 'is_terminal': True, 'observation': {'hand_image': array([[[ 4, 4, 0],\n", + " [ 6, 5, 0],\n", + " [ 8, 6, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 193, 202],\n", + " [189, 193, 202],\n", + " [188, 192, 201],\n", + " ...,\n", + " [211, 224, 185],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[188, 192, 201],\n", + " [188, 192, 201],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[187, 191, 200],\n", + " [187, 191, 200],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]]], dtype=uint8), 'image': array([[[108, 101, 82],\n", + " [104, 97, 78],\n", + " [ 46, 72, 72],\n", + " ...,\n", + " [ 12, 19, 13],\n", + " [ 7, 15, 16],\n", + " [ 39, 47, 48]],\n", + "\n", + " [[115, 102, 78],\n", + " [119, 106, 82],\n", + " [ 61, 91, 83],\n", + " ...,\n", + " [ 16, 20, 15],\n", + " [ 23, 26, 29],\n", + " [ 51, 54, 57]],\n", + "\n", + " [[114, 90, 67],\n", + " [124, 100, 77],\n", + " [ 71, 102, 86],\n", + " ...,\n", + " [ 22, 25, 24],\n", + " [ 43, 43, 45],\n", + " [ 48, 48, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 72, 55],\n", + " [ 71, 77, 60],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 67, 96, 106],\n", + " [ 66, 98, 107],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 55, 61, 44],\n", + " ...,\n", + " [ 69, 98, 106],\n", + " [ 69, 98, 106],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 46, 52, 33],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [ 68, 97, 107],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0464277 , -1.5272526 , 2.3538566 , -2.5545247 , -1.5915655 ,\n", + " 3.390477 , 0.40496254, 0.14374796, -0.00899097, 0.75652575,\n", + " 0.64914 , 0.05783392, 0.05423374, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 1.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 1.0, 'world_vector': array([0., 0., 0.], dtype=float32)}, 'is_first': False, 'is_last': True, 'is_terminal': True, 'observation': {'hand_image': array([[[ 4, 4, 0],\n", + " [ 6, 5, 0],\n", + " [ 8, 6, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 193, 202],\n", + " [189, 193, 202],\n", + " [188, 192, 201],\n", + " ...,\n", + " [211, 224, 185],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[188, 192, 201],\n", + " [188, 192, 201],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[187, 191, 200],\n", + " [187, 191, 200],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]]], dtype=uint8), 'image': array([[[108, 101, 82],\n", + " [104, 97, 78],\n", + " [ 46, 72, 72],\n", + " ...,\n", + " [ 12, 19, 13],\n", + " [ 7, 15, 16],\n", + " [ 39, 47, 48]],\n", + "\n", + " [[115, 102, 78],\n", + " [119, 106, 82],\n", + " [ 61, 91, 83],\n", + " ...,\n", + " [ 16, 20, 15],\n", + " [ 23, 26, 29],\n", + " [ 51, 54, 57]],\n", + "\n", + " [[114, 90, 67],\n", + " [124, 100, 77],\n", + " [ 71, 102, 86],\n", + " ...,\n", + " [ 22, 25, 24],\n", + " [ 43, 43, 45],\n", + " [ 48, 48, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 72, 55],\n", + " [ 71, 77, 60],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 67, 96, 106],\n", + " [ 66, 98, 107],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 55, 61, 44],\n", + " ...,\n", + " [ 69, 98, 106],\n", + " [ 69, 98, 106],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 46, 52, 33],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [ 68, 97, 107],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0464277 , -1.5272526 , 2.3538566 , -2.5545247 , -1.5915655 ,\n", + " 3.390477 , 0.40496254, 0.14374796, -0.00899097, 0.75652575,\n", + " 0.64914 , 0.05783392, 0.05423374, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00038095, 0. , 0.00057143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00228571, 0.00685714, 0.00354286], dtype=float32)}, 'is_first': True, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 47, 33, 26],\n", + " [ 49, 35, 29],\n", + " [ 48, 34, 27],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 19, 12, 11],\n", + " [ 22, 16, 15]],\n", + "\n", + " [[ 48, 34, 27],\n", + " [ 48, 34, 27],\n", + " [ 47, 33, 26],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " [[ 48, 34, 27],\n", + " [ 48, 34, 27],\n", + " [ 47, 33, 26],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[207, 233, 169],\n", + " [207, 233, 169],\n", + " [207, 233, 169],\n", + " ...,\n", + " [158, 176, 198],\n", + " [159, 177, 199],\n", + " [159, 177, 199]],\n", + "\n", + " [[207, 233, 169],\n", + " [207, 233, 169],\n", + " [208, 234, 170],\n", + " ...,\n", + " [159, 177, 199],\n", + " [158, 176, 198],\n", + " [158, 176, 198]],\n", + "\n", + " [[207, 233, 169],\n", + " [208, 234, 170],\n", + " [208, 234, 170],\n", + " ...,\n", + " [158, 176, 198],\n", + " [157, 175, 197],\n", + " [159, 177, 199]]], dtype=uint8), 'image': array([[[112, 105, 76],\n", + " [114, 107, 78],\n", + " [ 53, 77, 65],\n", + " ...,\n", + " [ 98, 92, 94],\n", + " [ 99, 91, 90],\n", + " [ 99, 91, 90]],\n", + "\n", + " [[124, 105, 78],\n", + " [122, 103, 76],\n", + " [ 68, 93, 76],\n", + " ...,\n", + " [ 97, 92, 90],\n", + " [ 97, 92, 88],\n", + " [ 97, 92, 88]],\n", + "\n", + " [[119, 88, 61],\n", + " [129, 98, 71],\n", + " [ 79, 107, 82],\n", + " ...,\n", + " [ 97, 89, 86],\n", + " [ 99, 92, 86],\n", + " [ 96, 89, 83]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 74, 58],\n", + " [ 74, 77, 61],\n", + " [ 70, 75, 61],\n", + " ...,\n", + " [ 70, 95, 102],\n", + " [ 72, 97, 104],\n", + " [ 75, 100, 107]],\n", + "\n", + " [[ 61, 64, 46],\n", + " [ 60, 63, 45],\n", + " [ 58, 61, 43],\n", + " ...,\n", + " [ 72, 97, 102],\n", + " [ 71, 96, 101],\n", + " [ 73, 98, 103]],\n", + "\n", + " [[ 56, 58, 40],\n", + " [ 53, 55, 37],\n", + " [ 52, 55, 37],\n", + " ...,\n", + " [ 68, 96, 100],\n", + " [ 70, 95, 100],\n", + " [ 71, 96, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2969947 , -1.6309904 , 2.4249759 , -2.406821 , -1.5576328 ,\n", + " 2.7911022 , 0.36736205, 0.05681828, -0.01433784, 0.6336503 ,\n", + " 0.77331245, 0.01563 , 0.01519585, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00114286, 0. , 0.01580952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00954286, 0.01148571, 0.00508571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 49, 33, 26],\n", + " [ 51, 35, 29],\n", + " [ 52, 36, 30],\n", + " ...,\n", + " [ 18, 10, 12],\n", + " [ 20, 13, 14],\n", + " [ 20, 13, 14]],\n", + "\n", + " [[ 49, 33, 26],\n", + " [ 49, 33, 26],\n", + " [ 49, 33, 26],\n", + " ...,\n", + " [ 20, 13, 14],\n", + " [ 21, 14, 15],\n", + " [ 21, 14, 15]],\n", + "\n", + " [[ 47, 33, 26],\n", + " [ 47, 33, 26],\n", + " [ 49, 33, 26],\n", + " ...,\n", + " [ 20, 13, 14],\n", + " [ 21, 14, 15],\n", + " [ 21, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[209, 234, 173],\n", + " [209, 234, 173],\n", + " [208, 233, 172],\n", + " ...,\n", + " [158, 176, 198],\n", + " [159, 177, 199],\n", + " [159, 177, 199]],\n", + "\n", + " [[208, 233, 172],\n", + " [208, 233, 172],\n", + " [210, 235, 174],\n", + " ...,\n", + " [161, 178, 200],\n", + " [161, 178, 200],\n", + " [161, 178, 200]],\n", + "\n", + " [[208, 233, 172],\n", + " [209, 234, 173],\n", + " [211, 236, 175],\n", + " ...,\n", + " [161, 178, 200],\n", + " [161, 178, 200],\n", + " [161, 178, 200]]], dtype=uint8), 'image': array([[[116, 108, 82],\n", + " [111, 103, 77],\n", + " [ 49, 74, 67],\n", + " ...,\n", + " [ 97, 90, 82],\n", + " [ 99, 92, 84],\n", + " [ 99, 92, 84]],\n", + "\n", + " [[127, 110, 83],\n", + " [117, 100, 73],\n", + " [ 66, 94, 79],\n", + " ...,\n", + " [ 95, 88, 80],\n", + " [ 99, 92, 84],\n", + " [ 96, 89, 81]],\n", + "\n", + " [[120, 92, 64],\n", + " [127, 99, 71],\n", + " [ 81, 108, 86],\n", + " ...,\n", + " [ 95, 88, 78],\n", + " [ 92, 85, 75],\n", + " [ 89, 82, 72]],\n", + "\n", + " ...,\n", + "\n", + " [[ 74, 77, 61],\n", + " [ 79, 82, 66],\n", + " [ 75, 78, 60],\n", + " ...,\n", + " [ 72, 98, 98],\n", + " [ 72, 98, 98],\n", + " [ 72, 98, 98]],\n", + "\n", + " [[ 65, 69, 48],\n", + " [ 67, 71, 50],\n", + " [ 61, 65, 44],\n", + " ...,\n", + " [ 72, 98, 98],\n", + " [ 73, 97, 95],\n", + " [ 73, 97, 95]],\n", + "\n", + " [[ 59, 62, 41],\n", + " [ 58, 61, 40],\n", + " [ 51, 55, 34],\n", + " ...,\n", + " [ 70, 96, 96],\n", + " [ 72, 96, 96],\n", + " [ 73, 97, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2966354 , -1.6166404 , 2.418114 , -2.4144275 , -1.5576924 ,\n", + " 2.7917013 , 0.3713127 , 0.05632875, -0.01605012, 0.63372785,\n", + " 0.7732474 , 0.01564922, 0.01525001, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00095238, 0.01561905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01108571, 0.01057143, 0.00765714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 38, 29, 22],\n", + " [ 42, 33, 25],\n", + " [ 42, 33, 25],\n", + " ...,\n", + " [ 21, 15, 14],\n", + " [ 20, 14, 12],\n", + " [ 20, 14, 12]],\n", + "\n", + " [[ 37, 28, 20],\n", + " [ 38, 29, 22],\n", + " [ 39, 30, 23],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 21, 15, 14],\n", + " [ 21, 15, 14]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 35, 28, 22],\n", + " [ 38, 29, 22],\n", + " ...,\n", + " [ 20, 14, 12],\n", + " [ 21, 15, 14],\n", + " [ 21, 15, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[199, 224, 170],\n", + " [198, 223, 169],\n", + " [196, 224, 167],\n", + " ...,\n", + " [159, 177, 199],\n", + " [159, 177, 199],\n", + " [161, 178, 200]],\n", + "\n", + " [[201, 226, 172],\n", + " [198, 223, 169],\n", + " [196, 224, 167],\n", + " ...,\n", + " [161, 178, 200],\n", + " [159, 177, 199],\n", + " [161, 178, 200]],\n", + "\n", + " [[201, 226, 172],\n", + " [201, 226, 172],\n", + " [197, 225, 168],\n", + " ...,\n", + " [159, 177, 199],\n", + " [158, 176, 198],\n", + " [159, 177, 199]]], dtype=uint8), 'image': array([[[119, 106, 66],\n", + " [116, 103, 63],\n", + " [ 52, 71, 59],\n", + " ...,\n", + " [ 84, 75, 68],\n", + " [ 83, 74, 67],\n", + " [ 80, 71, 64]],\n", + "\n", + " [[128, 108, 68],\n", + " [128, 108, 68],\n", + " [ 64, 89, 68],\n", + " ...,\n", + " [ 81, 71, 59],\n", + " [ 80, 71, 62],\n", + " [ 79, 70, 61]],\n", + "\n", + " [[120, 93, 58],\n", + " [130, 103, 68],\n", + " [ 80, 106, 80],\n", + " ...,\n", + " [ 78, 68, 54],\n", + " [ 76, 68, 54],\n", + " [ 78, 70, 56]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 75, 54],\n", + " [ 70, 73, 52],\n", + " [ 74, 74, 54],\n", + " ...,\n", + " [ 78, 101, 106],\n", + " [ 77, 100, 105],\n", + " [ 74, 97, 102]],\n", + "\n", + " [[ 63, 66, 43],\n", + " [ 63, 66, 43],\n", + " [ 61, 64, 41],\n", + " ...,\n", + " [ 72, 97, 102],\n", + " [ 72, 97, 102],\n", + " [ 71, 96, 101]],\n", + "\n", + " [[ 55, 55, 31],\n", + " [ 55, 55, 31],\n", + " [ 52, 55, 32],\n", + " ...,\n", + " [ 66, 95, 96],\n", + " [ 70, 95, 100],\n", + " [ 69, 94, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3029208 , -1.591381 , 2.4043546 , -2.4257529 , -1.5580639 ,\n", + " 2.7932708 , 0.37898293, 0.05273136, -0.01882746, 0.63674 ,\n", + " 0.7707726 , 0.0154203 , 0.01529887, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0192381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01171429, 0.00902857, 0.00897143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 16, 12, 8],\n", + " [ 17, 14, 10],\n", + " [ 15, 11, 7],\n", + " ...,\n", + " [ 16, 12, 10],\n", + " [ 18, 14, 12],\n", + " [ 19, 15, 14]],\n", + "\n", + " [[ 20, 14, 11],\n", + " [ 18, 12, 8],\n", + " [ 16, 12, 8],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14]],\n", + "\n", + " [[ 20, 14, 11],\n", + " [ 18, 12, 8],\n", + " [ 18, 12, 8],\n", + " ...,\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14],\n", + " [ 19, 15, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[211, 232, 185],\n", + " [210, 230, 184],\n", + " [210, 230, 184],\n", + " ...,\n", + " [159, 174, 197],\n", + " [160, 175, 198],\n", + " [160, 175, 198]],\n", + "\n", + " [[212, 233, 186],\n", + " [211, 232, 185],\n", + " [211, 232, 185],\n", + " ...,\n", + " [161, 177, 199],\n", + " [161, 177, 199],\n", + " [161, 177, 199]],\n", + "\n", + " [[212, 233, 186],\n", + " [212, 233, 186],\n", + " [212, 233, 186],\n", + " ...,\n", + " [161, 177, 199],\n", + " [160, 175, 198],\n", + " [161, 177, 199]]], dtype=uint8), 'image': array([[[111, 105, 81],\n", + " [108, 102, 78],\n", + " [ 49, 76, 71],\n", + " ...,\n", + " [ 69, 69, 47],\n", + " [ 70, 68, 49],\n", + " [ 70, 68, 49]],\n", + "\n", + " [[121, 105, 82],\n", + " [123, 107, 84],\n", + " [ 65, 92, 83],\n", + " ...,\n", + " [ 72, 71, 54],\n", + " [ 72, 67, 49],\n", + " [ 78, 73, 55]],\n", + "\n", + " [[117, 90, 68],\n", + " [129, 102, 80],\n", + " [ 78, 107, 89],\n", + " ...,\n", + " [ 75, 74, 59],\n", + " [ 76, 74, 55],\n", + " [ 81, 79, 60]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 75, 52],\n", + " [ 83, 86, 63],\n", + " [ 78, 81, 58],\n", + " ...,\n", + " [ 70, 100, 105],\n", + " [ 67, 99, 106],\n", + " [ 66, 98, 105]],\n", + "\n", + " [[ 60, 63, 40],\n", + " [ 56, 59, 36],\n", + " [ 57, 61, 40],\n", + " ...,\n", + " [ 68, 93, 100],\n", + " [ 68, 96, 102],\n", + " [ 68, 96, 102]],\n", + "\n", + " [[ 51, 54, 33],\n", + " [ 51, 54, 33],\n", + " [ 51, 54, 36],\n", + " ...,\n", + " [ 69, 94, 99],\n", + " [ 71, 97, 99],\n", + " [ 71, 97, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.314199 , -1.5623087 , 2.39123 , -2.4421766 , -1.557824 ,\n", + " 2.7923725 , 0.3873519 , 0.0471875 , -0.0229582 , 0.64069533,\n", + " 0.7674803 , 0.01556755, 0.01552759, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01411429, 0.00897143, 0.01274286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 20, 17, 13],\n", + " [ 18, 15, 11],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 17, 13, 11],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " [[ 19, 16, 12],\n", + " [ 18, 15, 11],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 17, 13, 11],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " [[ 21, 16, 10],\n", + " [ 20, 15, 9],\n", + " [ 20, 14, 11],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12],\n", + " [ 18, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 230, 185],\n", + " [210, 230, 185],\n", + " [209, 228, 184],\n", + " ...,\n", + " [156, 172, 190],\n", + " [156, 172, 190],\n", + " [158, 175, 192]],\n", + "\n", + " [[210, 230, 185],\n", + " [210, 230, 185],\n", + " [211, 231, 187],\n", + " ...,\n", + " [158, 175, 192],\n", + " [158, 175, 192],\n", + " [158, 175, 192]],\n", + "\n", + " [[211, 231, 187],\n", + " [211, 231, 187],\n", + " [211, 231, 187],\n", + " ...,\n", + " [158, 175, 192],\n", + " [158, 175, 192],\n", + " [157, 174, 191]]], dtype=uint8), 'image': array([[[110, 105, 72],\n", + " [108, 103, 70],\n", + " [ 48, 77, 74],\n", + " ...,\n", + " [ 17, 20, 21],\n", + " [ 19, 26, 20],\n", + " [ 36, 43, 37]],\n", + "\n", + " [[118, 103, 73],\n", + " [121, 106, 76],\n", + " [ 55, 87, 76],\n", + " ...,\n", + " [ 13, 19, 17],\n", + " [ 36, 44, 31],\n", + " [ 47, 55, 42]],\n", + "\n", + " [[116, 89, 63],\n", + " [126, 99, 73],\n", + " [ 74, 106, 87],\n", + " ...,\n", + " [ 25, 34, 27],\n", + " [ 56, 65, 45],\n", + " [ 61, 70, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 53],\n", + " [ 75, 78, 60],\n", + " [ 73, 75, 64],\n", + " ...,\n", + " [ 67, 97, 100],\n", + " [ 68, 99, 99],\n", + " [ 67, 98, 98]],\n", + "\n", + " [[ 61, 65, 44],\n", + " [ 64, 68, 47],\n", + " [ 59, 61, 43],\n", + " ...,\n", + " [ 66, 96, 101],\n", + " [ 66, 96, 99],\n", + " [ 66, 96, 99]],\n", + "\n", + " [[ 60, 59, 42],\n", + " [ 57, 56, 39],\n", + " [ 49, 51, 33],\n", + " ...,\n", + " [ 65, 95, 100],\n", + " [ 61, 94, 98],\n", + " [ 61, 94, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3272011 , -1.5332474 , 2.3811233 , -2.4609396 , -1.5572494 ,\n", + " 2.7913055 , 0.3946254 , 0.04107245, -0.02822383, 0.6452304 ,\n", + " 0.7636719 , 0.01568247, 0.01539803, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.01790476], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01222857, 0.00857143, 0.01382857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 26, 18, 13],\n", + " [ 26, 18, 13],\n", + " [ 26, 18, 14],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 18, 13, 14],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 24, 17, 12],\n", + " [ 23, 16, 10],\n", + " [ 23, 16, 10],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 24, 17, 12],\n", + " [ 23, 16, 10],\n", + " [ 23, 16, 10],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[206, 228, 182],\n", + " [206, 228, 182],\n", + " [205, 227, 180],\n", + " ...,\n", + " [153, 170, 188],\n", + " [154, 171, 189],\n", + " [154, 171, 189]],\n", + "\n", + " [[207, 230, 183],\n", + " [207, 230, 183],\n", + " [207, 230, 183],\n", + " ...,\n", + " [153, 170, 188],\n", + " [153, 170, 188],\n", + " [153, 170, 188]],\n", + "\n", + " [[206, 228, 182],\n", + " [207, 230, 183],\n", + " [207, 230, 183],\n", + " ...,\n", + " [153, 170, 188],\n", + " [154, 171, 189],\n", + " [153, 170, 188]]], dtype=uint8), 'image': array([[[108, 102, 78],\n", + " [108, 102, 78],\n", + " [ 55, 82, 75],\n", + " ...,\n", + " [ 71, 79, 66],\n", + " [ 69, 77, 64],\n", + " [ 70, 78, 65]],\n", + "\n", + " [[111, 94, 76],\n", + " [123, 106, 88],\n", + " [ 66, 94, 81],\n", + " ...,\n", + " [ 69, 77, 66],\n", + " [ 67, 80, 67],\n", + " [ 67, 80, 67]],\n", + "\n", + " [[113, 85, 68],\n", + " [130, 102, 85],\n", + " [ 80, 108, 95],\n", + " ...,\n", + " [ 70, 78, 67],\n", + " [ 68, 83, 69],\n", + " [ 68, 83, 69]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 73, 45],\n", + " [ 72, 77, 49],\n", + " [ 69, 76, 50],\n", + " ...,\n", + " [ 63, 93, 98],\n", + " [ 68, 96, 102],\n", + " [ 68, 96, 102]],\n", + "\n", + " [[ 60, 65, 35],\n", + " [ 61, 66, 36],\n", + " [ 58, 63, 33],\n", + " ...,\n", + " [ 62, 92, 97],\n", + " [ 64, 89, 96],\n", + " [ 63, 88, 95]],\n", + "\n", + " [[ 57, 58, 31],\n", + " [ 56, 57, 30],\n", + " [ 53, 57, 27],\n", + " ...,\n", + " [ 59, 89, 94],\n", + " [ 62, 87, 94],\n", + " [ 59, 84, 91]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3431368 , -1.4991382 , 2.3722975 , -2.4858325 , -1.5566143 ,\n", + " 2.7856972 , 0.40199557, 0.03360351, -0.03550917, 0.6491278 ,\n", + " 0.76036435, 0.01573002, 0.01522294, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00019048, 0. , 0.01238095], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01257143, 0.01074286, 0.01348571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 3, 4, 2],\n", + " [ 6, 7, 4],\n", + " [ 9, 7, 5],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 17, 12, 13],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 9, 7, 5],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 8, 6, 4],\n", + " [ 9, 7, 5],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[203, 227, 182],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [158, 174, 187],\n", + " [161, 174, 188],\n", + " [161, 174, 188]],\n", + "\n", + " [[201, 226, 181],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [160, 176, 189],\n", + " [162, 176, 189],\n", + " [161, 174, 188]],\n", + "\n", + " [[203, 227, 182],\n", + " [203, 227, 182],\n", + " [203, 227, 182],\n", + " ...,\n", + " [161, 177, 190],\n", + " [162, 176, 189],\n", + " [161, 174, 188]]], dtype=uint8), 'image': array([[[115, 109, 85],\n", + " [112, 106, 82],\n", + " [ 56, 83, 78],\n", + " ...,\n", + " [ 80, 93, 88],\n", + " [ 77, 92, 87],\n", + " [ 78, 93, 88]],\n", + "\n", + " [[124, 108, 85],\n", + " [126, 110, 87],\n", + " [ 54, 85, 71],\n", + " ...,\n", + " [ 77, 92, 87],\n", + " [ 77, 92, 89],\n", + " [ 74, 89, 86]],\n", + "\n", + " [[116, 89, 67],\n", + " [137, 110, 88],\n", + " [ 69, 101, 82],\n", + " ...,\n", + " [ 74, 92, 86],\n", + " [ 72, 90, 84],\n", + " [ 69, 87, 81]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 72, 54],\n", + " [ 73, 75, 57],\n", + " [ 69, 70, 57],\n", + " ...,\n", + " [ 82, 89, 83],\n", + " [ 82, 87, 77],\n", + " [ 81, 86, 76]],\n", + "\n", + " [[ 60, 66, 49],\n", + " [ 62, 68, 51],\n", + " [ 59, 65, 48],\n", + " ...,\n", + " [ 82, 86, 79],\n", + " [ 84, 86, 77],\n", + " [ 84, 86, 77]],\n", + "\n", + " [[ 53, 58, 46],\n", + " [ 56, 61, 49],\n", + " [ 49, 55, 38],\n", + " ...,\n", + " [ 86, 87, 81],\n", + " [ 90, 88, 80],\n", + " [ 90, 88, 80]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3573966 , -1.4627742 , 2.364023 , -2.5137966 , -1.5562791 ,\n", + " 2.7826176 , 0.40912914, 0.02653525, -0.04385389, 0.6533296 ,\n", + " 0.756757 , 0.0157517 , 0.01520285, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00133333, -0.01009524, 0.01047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01257143, 0.01045714, 0.01451429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 34, 26, 21],\n", + " [ 33, 25, 20],\n", + " [ 29, 24, 20],\n", + " ...,\n", + " [ 19, 15, 14],\n", + " [ 18, 13, 14],\n", + " [ 18, 13, 14]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " [ 31, 24, 18],\n", + " ...,\n", + " [ 18, 14, 12],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " [[ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " [ 33, 25, 20],\n", + " ...,\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15],\n", + " [ 19, 14, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [205, 226, 182],\n", + " ...,\n", + " [154, 171, 189],\n", + " [156, 172, 190],\n", + " [156, 172, 190]],\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [206, 228, 183],\n", + " ...,\n", + " [154, 171, 189],\n", + " [154, 171, 189],\n", + " [156, 172, 190]],\n", + "\n", + " [[205, 226, 182],\n", + " [205, 226, 182],\n", + " [206, 228, 183],\n", + " ...,\n", + " [156, 172, 190],\n", + " [156, 172, 190],\n", + " [154, 171, 189]]], dtype=uint8), 'image': array([[[112, 105, 84],\n", + " [109, 102, 81],\n", + " [ 38, 66, 70],\n", + " ...,\n", + " [ 43, 54, 35],\n", + " [ 42, 52, 40],\n", + " [ 41, 51, 39]],\n", + "\n", + " [[122, 104, 82],\n", + " [123, 105, 83],\n", + " [ 61, 93, 86],\n", + " ...,\n", + " [ 43, 54, 33],\n", + " [ 40, 50, 36],\n", + " [ 40, 50, 36]],\n", + "\n", + " [[118, 91, 69],\n", + " [127, 100, 78],\n", + " [ 75, 106, 90],\n", + " ...,\n", + " [ 42, 53, 32],\n", + " [ 41, 49, 36],\n", + " [ 41, 49, 36]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 75, 52],\n", + " [ 69, 79, 56],\n", + " [ 64, 73, 55],\n", + " ...,\n", + " [ 92, 93, 80],\n", + " [ 97, 98, 87],\n", + " [ 95, 96, 85]],\n", + "\n", + " [[ 55, 66, 45],\n", + " [ 55, 66, 45],\n", + " [ 54, 65, 46],\n", + " ...,\n", + " [ 97, 94, 78],\n", + " [ 99, 97, 85],\n", + " [ 99, 97, 85]],\n", + "\n", + " [[ 54, 64, 48],\n", + " [ 48, 58, 42],\n", + " [ 47, 55, 40],\n", + " ...,\n", + " [ 99, 96, 80],\n", + " [ 99, 98, 83],\n", + " [101, 100, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3698246 , -1.4250783 , 2.3521204 , -2.5393493 , -1.5560521 ,\n", + " 2.7788906 , 0.41707364, 0.0197808 , -0.05207061, 0.6565796 ,\n", + " 0.75394094, 0.01570738, 0.0151506 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00933333, 0.008 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01325714, 0.01108571, 0.0148 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 33, 26, 18],\n", + " [ 33, 26, 18],\n", + " [ 33, 26, 18],\n", + " ...,\n", + " [ 21, 17, 16],\n", + " [ 21, 17, 17],\n", + " [ 20, 16, 16]],\n", + "\n", + " [[ 30, 23, 17],\n", + " [ 30, 23, 17],\n", + " [ 29, 22, 16],\n", + " ...,\n", + " [ 20, 16, 15],\n", + " [ 21, 17, 17],\n", + " [ 21, 17, 17]],\n", + "\n", + " [[ 27, 22, 16],\n", + " [ 26, 21, 15],\n", + " [ 26, 21, 15],\n", + " ...,\n", + " [ 20, 16, 16],\n", + " [ 21, 17, 17],\n", + " [ 21, 17, 17]],\n", + "\n", + " ...,\n", + "\n", + " [[205, 229, 184],\n", + " [205, 229, 184],\n", + " [204, 228, 183],\n", + " ...,\n", + " [164, 175, 189],\n", + " [165, 176, 190],\n", + " [165, 176, 190]],\n", + "\n", + " [[205, 229, 184],\n", + " [205, 229, 184],\n", + " [205, 229, 184],\n", + " ...,\n", + " [165, 176, 190],\n", + " [165, 176, 190],\n", + " [165, 176, 190]],\n", + "\n", + " [[205, 229, 184],\n", + " [206, 230, 185],\n", + " [206, 230, 185],\n", + " ...,\n", + " [164, 175, 189],\n", + " [165, 176, 190],\n", + " [164, 175, 189]]], dtype=uint8), 'image': array([[[110, 107, 75],\n", + " [111, 108, 76],\n", + " [ 41, 73, 64],\n", + " ...,\n", + " [ 76, 50, 50],\n", + " [ 80, 51, 41],\n", + " [ 84, 55, 45]],\n", + "\n", + " [[121, 106, 78],\n", + " [126, 111, 83],\n", + " [ 59, 92, 77],\n", + " ...,\n", + " [ 80, 54, 52],\n", + " [ 84, 55, 43],\n", + " [ 84, 55, 43]],\n", + "\n", + " [[116, 89, 65],\n", + " [130, 103, 79],\n", + " [ 73, 107, 87],\n", + " ...,\n", + " [ 84, 57, 51],\n", + " [ 86, 56, 49],\n", + " [ 87, 57, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 74, 73, 58],\n", + " [ 76, 75, 60],\n", + " [ 72, 71, 56],\n", + " ...,\n", + " [104, 103, 88],\n", + " [104, 103, 88],\n", + " [105, 104, 89]],\n", + "\n", + " [[ 63, 65, 47],\n", + " [ 65, 67, 49],\n", + " [ 62, 64, 48],\n", + " ...,\n", + " [105, 104, 87],\n", + " [106, 108, 92],\n", + " [109, 111, 95]],\n", + "\n", + " [[ 56, 58, 40],\n", + " [ 54, 56, 38],\n", + " [ 49, 51, 33],\n", + " ...,\n", + " [105, 108, 90],\n", + " [103, 108, 94],\n", + " [103, 108, 94]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3830068 , -1.3866261 , 2.3424206 , -2.5729144 , -1.5544709 ,\n", + " 2.7728264 , 0.42537263, 0.01276265, -0.0608653 , 0.65915596,\n", + " 0.7516113 , 0.01741757, 0.01704437, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.0047619 , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01417143, 0.01262857, 0.01457143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[145, 165, 180],\n", + " [147, 167, 182],\n", + " [147, 167, 182],\n", + " ...,\n", + " [ 21, 18, 19],\n", + " [ 21, 17, 17],\n", + " [ 23, 18, 19]],\n", + "\n", + " [[147, 167, 182],\n", + " [147, 167, 182],\n", + " [147, 167, 182],\n", + " ...,\n", + " [ 19, 17, 17],\n", + " [ 23, 18, 19],\n", + " [ 23, 18, 19]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 21, 17, 17],\n", + " [ 23, 18, 19],\n", + " [ 23, 18, 19]],\n", + "\n", + " ...,\n", + "\n", + " [[208, 230, 185],\n", + " [208, 230, 185],\n", + " [207, 229, 184],\n", + " ...,\n", + " [158, 173, 189],\n", + " [161, 174, 190],\n", + " [161, 174, 190]],\n", + "\n", + " [[208, 230, 185],\n", + " [208, 230, 185],\n", + " [208, 230, 185],\n", + " ...,\n", + " [158, 173, 189],\n", + " [160, 172, 189],\n", + " [160, 172, 189]],\n", + "\n", + " [[207, 229, 184],\n", + " [208, 230, 185],\n", + " [209, 231, 187],\n", + " ...,\n", + " [158, 173, 189],\n", + " [160, 172, 189],\n", + " [160, 172, 189]]], dtype=uint8), 'image': array([[[111, 104, 73],\n", + " [111, 104, 73],\n", + " [ 48, 75, 68],\n", + " ...,\n", + " [ 79, 86, 90],\n", + " [ 81, 88, 94],\n", + " [ 82, 89, 95]],\n", + "\n", + " [[120, 101, 72],\n", + " [118, 99, 70],\n", + " [ 66, 94, 79],\n", + " ...,\n", + " [ 78, 83, 88],\n", + " [ 85, 87, 93],\n", + " [ 86, 88, 94]],\n", + "\n", + " [[118, 86, 62],\n", + " [129, 97, 73],\n", + " [ 79, 108, 86],\n", + " ...,\n", + " [ 85, 87, 95],\n", + " [ 85, 87, 93],\n", + " [ 87, 89, 95]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 52],\n", + " [ 72, 79, 57],\n", + " [ 71, 77, 58],\n", + " ...,\n", + " [113, 111, 99],\n", + " [113, 111, 101],\n", + " [111, 109, 99]],\n", + "\n", + " [[ 59, 66, 44],\n", + " [ 58, 65, 43],\n", + " [ 56, 62, 43],\n", + " ...,\n", + " [114, 112, 100],\n", + " [113, 111, 99],\n", + " [114, 112, 100]],\n", + "\n", + " [[ 52, 55, 39],\n", + " [ 51, 54, 38],\n", + " [ 50, 53, 37],\n", + " ...,\n", + " [114, 112, 100],\n", + " [114, 112, 100],\n", + " [115, 113, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.396764 , -1.3478969 , 2.3317156 , -2.606684 , -1.5517172 ,\n", + " 2.7645218 , 0.4339046 , 0.00549856, -0.06976949, 0.6610677 ,\n", + " 0.74982053, 0.01980009, 0.01914801, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00438095, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01577143, 0.01262857, 0.01222857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[156, 168, 180],\n", + " [155, 167, 178],\n", + " [153, 167, 178],\n", + " ...,\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21]],\n", + "\n", + " [[155, 167, 178],\n", + " [155, 167, 178],\n", + " [151, 166, 177],\n", + " ...,\n", + " [ 20, 20, 20],\n", + " [ 20, 20, 20],\n", + " [ 20, 20, 20]],\n", + "\n", + " [[153, 167, 178],\n", + " [151, 166, 177],\n", + " [151, 166, 177],\n", + " ...,\n", + " [ 20, 20, 20],\n", + " [ 21, 21, 21],\n", + " [ 21, 21, 21]],\n", + "\n", + " ...,\n", + "\n", + " [[204, 229, 182],\n", + " [204, 229, 182],\n", + " [203, 228, 180],\n", + " ...,\n", + " [148, 168, 190],\n", + " [148, 168, 190],\n", + " [148, 168, 190]],\n", + "\n", + " [[204, 229, 182],\n", + " [203, 228, 180],\n", + " [203, 228, 180],\n", + " ...,\n", + " [150, 170, 192],\n", + " [149, 169, 191],\n", + " [147, 167, 189]],\n", + "\n", + " [[204, 229, 182],\n", + " [204, 229, 182],\n", + " [203, 228, 180],\n", + " ...,\n", + " [150, 170, 192],\n", + " [150, 170, 192],\n", + " [150, 170, 192]]], dtype=uint8), 'image': array([[[111, 103, 89],\n", + " [107, 99, 85],\n", + " [ 44, 74, 79],\n", + " ...,\n", + " [106, 102, 105],\n", + " [ 99, 102, 103],\n", + " [ 99, 102, 103]],\n", + "\n", + " [[123, 107, 84],\n", + " [122, 106, 83],\n", + " [ 61, 93, 86],\n", + " ...,\n", + " [104, 104, 100],\n", + " [102, 102, 102],\n", + " [101, 101, 101]],\n", + "\n", + " [[124, 95, 72],\n", + " [130, 101, 78],\n", + " [ 73, 104, 88],\n", + " ...,\n", + " [105, 102, 97],\n", + " [102, 102, 96],\n", + " [102, 102, 96]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 52],\n", + " [ 74, 77, 59],\n", + " [ 69, 73, 52],\n", + " ...,\n", + " [117, 119, 103],\n", + " [119, 118, 101],\n", + " [113, 112, 95]],\n", + "\n", + " [[ 59, 62, 44],\n", + " [ 60, 63, 45],\n", + " [ 60, 63, 45],\n", + " ...,\n", + " [115, 117, 99],\n", + " [118, 120, 104],\n", + " [119, 121, 105]],\n", + "\n", + " [[ 51, 54, 36],\n", + " [ 53, 56, 38],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [117, 119, 103],\n", + " [119, 121, 105],\n", + " [119, 121, 105]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4100306e+00, -1.3073558e+00, 2.3148079e+00, -2.6330740e+00,\n", + " -1.5499452e+00, 2.7541919e+00, 4.4335869e-01, -2.3262841e-03,\n", + " -7.8644946e-02, 6.6207409e-01, 7.4887121e-01, 2.1015564e-02,\n", + " 2.0203222e-02, 0.0000000e+00, 0.0000000e+00], dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00152381, -0.00171429, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01742857, 0.01297143, 0.01085714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[147, 166, 183],\n", + " [147, 166, 183],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[148, 167, 184],\n", + " [148, 167, 184],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[148, 167, 184],\n", + " [148, 167, 184],\n", + " [147, 166, 183],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 228, 182],\n", + " [210, 228, 182],\n", + " [209, 227, 179],\n", + " ...,\n", + " [156, 172, 192],\n", + " [156, 172, 192],\n", + " [156, 172, 192]],\n", + "\n", + " [[211, 229, 183],\n", + " [211, 229, 183],\n", + " [210, 228, 180],\n", + " ...,\n", + " [158, 174, 194],\n", + " [157, 173, 193],\n", + " [157, 173, 193]],\n", + "\n", + " [[211, 229, 183],\n", + " [211, 229, 183],\n", + " [211, 230, 181],\n", + " ...,\n", + " [156, 172, 192],\n", + " [156, 172, 192],\n", + " [154, 170, 191]]], dtype=uint8), 'image': array([[[110, 111, 82],\n", + " [104, 105, 76],\n", + " [ 35, 70, 73],\n", + " ...,\n", + " [111, 113, 102],\n", + " [115, 116, 110],\n", + " [115, 116, 110]],\n", + "\n", + " [[124, 112, 81],\n", + " [119, 107, 76],\n", + " [ 49, 84, 76],\n", + " ...,\n", + " [111, 113, 102],\n", + " [114, 115, 109],\n", + " [114, 115, 109]],\n", + "\n", + " [[121, 98, 66],\n", + " [123, 100, 68],\n", + " [ 74, 108, 88],\n", + " ...,\n", + " [110, 112, 103],\n", + " [115, 116, 110],\n", + " [116, 117, 111]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 48],\n", + " [ 71, 78, 52],\n", + " [ 65, 74, 54],\n", + " ...,\n", + " [122, 123, 112],\n", + " [121, 123, 112],\n", + " [121, 123, 112]],\n", + "\n", + " [[ 60, 68, 39],\n", + " [ 60, 68, 39],\n", + " [ 52, 64, 40],\n", + " ...,\n", + " [120, 121, 108],\n", + " [124, 125, 112],\n", + " [124, 125, 112]],\n", + "\n", + " [[ 55, 61, 42],\n", + " [ 51, 57, 38],\n", + " [ 44, 53, 35],\n", + " ...,\n", + " [122, 123, 110],\n", + " [122, 123, 110],\n", + " [122, 123, 110]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4241116 , -1.2713832 , 2.294727 , -2.6514275 , -1.5482687 ,\n", + " 2.7406254 , 0.4530185 , -0.01090289, -0.08589576, 0.66217947,\n", + " 0.74872 , 0.02204888, 0.02122683, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00380952, -0.0007619 , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01782857, 0.01325714, 0.01085714], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[149, 166, 182],\n", + " [149, 166, 182],\n", + " [151, 166, 182],\n", + " ...,\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22],\n", + " [ 25, 23, 23]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[149, 166, 182],\n", + " [149, 166, 182],\n", + " [149, 166, 182],\n", + " ...,\n", + " [ 24, 23, 21],\n", + " [ 24, 22, 22],\n", + " [ 25, 23, 23]],\n", + "\n", + " ...,\n", + "\n", + " [[212, 228, 180],\n", + " [212, 228, 180],\n", + " [211, 226, 180],\n", + " ...,\n", + " [159, 173, 191],\n", + " [159, 173, 191],\n", + " [159, 173, 191]],\n", + "\n", + " [[213, 229, 181],\n", + " [212, 228, 180],\n", + " [213, 228, 183],\n", + " ...,\n", + " [158, 172, 190],\n", + " [158, 172, 190],\n", + " [158, 172, 190]],\n", + "\n", + " [[213, 229, 181],\n", + " [213, 229, 181],\n", + " [213, 228, 183],\n", + " ...,\n", + " [156, 171, 189],\n", + " [158, 172, 190],\n", + " [158, 172, 190]]], dtype=uint8), 'image': array([[[105, 107, 73],\n", + " [107, 109, 75],\n", + " [ 37, 72, 64],\n", + " ...,\n", + " [183, 184, 175],\n", + " [171, 167, 170],\n", + " [167, 163, 166]],\n", + "\n", + " [[116, 105, 71],\n", + " [120, 109, 75],\n", + " [ 53, 86, 71],\n", + " ...,\n", + " [179, 176, 173],\n", + " [170, 166, 169],\n", + " [163, 159, 162]],\n", + "\n", + " [[124, 101, 71],\n", + " [124, 101, 71],\n", + " [ 75, 107, 88],\n", + " ...,\n", + " [179, 173, 175],\n", + " [167, 166, 168],\n", + " [160, 159, 161]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 73, 54],\n", + " [ 72, 78, 59],\n", + " [ 70, 76, 59],\n", + " ...,\n", + " [126, 127, 114],\n", + " [126, 127, 116],\n", + " [123, 124, 113]],\n", + "\n", + " [[ 58, 64, 45],\n", + " [ 57, 63, 44],\n", + " [ 53, 62, 42],\n", + " ...,\n", + " [127, 126, 111],\n", + " [127, 125, 113],\n", + " [128, 126, 114]],\n", + "\n", + " [[ 54, 57, 39],\n", + " [ 51, 54, 36],\n", + " [ 44, 53, 33],\n", + " ...,\n", + " [129, 128, 113],\n", + " [127, 128, 115],\n", + " [127, 128, 115]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4396536 , -1.2382988 , 2.2719407 , -2.66238 , -1.5463403 ,\n", + " 2.7252848 , 0.46257424, -0.020392 , -0.09226454, 0.66218996,\n", + " 0.74868333, 0.02275231, 0.02144971, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00285714, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01782857, 0.01342857, 0.01022857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[137, 159, 181],\n", + " [137, 159, 181],\n", + " [138, 160, 182],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 24, 20, 18],\n", + " [ 28, 24, 23]],\n", + "\n", + " [[137, 159, 181],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22]],\n", + "\n", + " [[137, 159, 181],\n", + " [137, 159, 181],\n", + " [137, 159, 181],\n", + " ...,\n", + " [ 23, 21, 21],\n", + " [ 23, 21, 21],\n", + " [ 24, 22, 22]],\n", + "\n", + " ...,\n", + "\n", + " [[211, 227, 179],\n", + " [211, 227, 179],\n", + " [211, 227, 179],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 168, 190],\n", + " [150, 168, 190]],\n", + "\n", + " [[211, 227, 179],\n", + " [211, 227, 179],\n", + " [211, 227, 179],\n", + " ...,\n", + " [151, 169, 191],\n", + " [149, 166, 189],\n", + " [149, 166, 189]],\n", + "\n", + " [[210, 226, 178],\n", + " [210, 226, 178],\n", + " [211, 227, 179],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 168, 190],\n", + " [148, 165, 187]]], dtype=uint8), 'image': array([[[111, 106, 75],\n", + " [107, 102, 71],\n", + " [ 53, 78, 69],\n", + " ...,\n", + " [162, 162, 162],\n", + " [167, 167, 169],\n", + " [173, 173, 175]],\n", + "\n", + " [[119, 104, 74],\n", + " [121, 106, 76],\n", + " [ 65, 92, 74],\n", + " ...,\n", + " [161, 161, 161],\n", + " [167, 167, 167],\n", + " [175, 175, 175]],\n", + "\n", + " [[111, 88, 58],\n", + " [125, 102, 72],\n", + " [ 79, 106, 84],\n", + " ...,\n", + " [159, 162, 161],\n", + " [168, 168, 168],\n", + " [174, 174, 174]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 73, 44],\n", + " [ 70, 78, 49],\n", + " [ 69, 76, 52],\n", + " ...,\n", + " [131, 132, 121],\n", + " [134, 132, 122],\n", + " [133, 131, 121]],\n", + "\n", + " [[ 60, 68, 39],\n", + " [ 58, 66, 37],\n", + " [ 56, 66, 41],\n", + " ...,\n", + " [133, 131, 119],\n", + " [136, 133, 119],\n", + " [139, 136, 122]],\n", + "\n", + " [[ 55, 59, 34],\n", + " [ 50, 54, 29],\n", + " [ 46, 56, 33],\n", + " ...,\n", + " [137, 134, 118],\n", + " [141, 135, 120],\n", + " [139, 133, 118]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4565248 , -1.2037634 , 2.2444725 , -2.6683176 , -1.5432638 ,\n", + " 2.7085657 , 0.47299433, -0.03085847, -0.0988073 , 0.6621888 ,\n", + " 0.74867994, 0.02358894, 0.0206851 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00380952, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01788571, 0.01325714, 0.01051429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[143, 159, 179],\n", + " [145, 161, 181],\n", + " [144, 160, 180],\n", + " ...,\n", + " [ 27, 23, 22],\n", + " [ 27, 23, 22],\n", + " [ 27, 23, 22]],\n", + "\n", + " [[143, 159, 179],\n", + " [144, 160, 180],\n", + " [143, 159, 179],\n", + " ...,\n", + " [ 24, 20, 18],\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19]],\n", + "\n", + " [[141, 159, 179],\n", + " [141, 159, 179],\n", + " [141, 159, 179],\n", + " ...,\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19],\n", + " [ 25, 21, 19]],\n", + "\n", + " ...,\n", + "\n", + " [[210, 227, 176],\n", + " [210, 227, 176],\n", + " [208, 225, 175],\n", + " ...,\n", + " [147, 164, 186],\n", + " [150, 165, 187],\n", + " [150, 165, 187]],\n", + "\n", + " [[208, 225, 175],\n", + " [208, 225, 175],\n", + " [208, 225, 175],\n", + " ...,\n", + " [149, 166, 189],\n", + " [150, 165, 187],\n", + " [150, 165, 187]],\n", + "\n", + " [[207, 224, 174],\n", + " [208, 225, 175],\n", + " [208, 225, 175],\n", + " ...,\n", + " [149, 166, 189],\n", + " [149, 164, 186],\n", + " [150, 165, 187]]], dtype=uint8), 'image': array([[[108, 104, 68],\n", + " [109, 105, 69],\n", + " [ 45, 71, 56],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " [[120, 103, 66],\n", + " [125, 108, 71],\n", + " [ 66, 93, 71],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " [[119, 89, 57],\n", + " [128, 98, 66],\n", + " [ 80, 108, 81],\n", + " ...,\n", + " [254, 255, 251],\n", + " [254, 255, 251],\n", + " [254, 255, 251]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 73, 50],\n", + " [ 76, 80, 57],\n", + " [ 73, 76, 58],\n", + " ...,\n", + " [137, 133, 124],\n", + " [141, 134, 124],\n", + " [137, 130, 120]],\n", + "\n", + " [[ 59, 62, 37],\n", + " [ 60, 63, 38],\n", + " [ 60, 63, 38],\n", + " ...,\n", + " [136, 134, 124],\n", + " [136, 134, 124],\n", + " [138, 136, 126]],\n", + "\n", + " [[ 53, 56, 33],\n", + " [ 48, 51, 28],\n", + " [ 48, 51, 26],\n", + " ...,\n", + " [136, 137, 126],\n", + " [139, 137, 129],\n", + " [140, 138, 130]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.471241 , -1.1722492 , 2.2166338 , -2.6705124 , -1.5408925 ,\n", + " 2.6938715 , 0.4829263 , -0.04071204, -0.10468176, 0.6621572 ,\n", + " 0.74871343, 0.02410153, 0.0198809 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0047619, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01811429, 0.01285714, 0.01 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[136, 158, 179],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 21, 19, 17],\n", + " [ 21, 19, 17],\n", + " [ 23, 21, 19]],\n", + "\n", + " [[137, 159, 181],\n", + " [136, 158, 179],\n", + " [136, 158, 179],\n", + " ...,\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18],\n", + " [ 23, 21, 19]],\n", + "\n", + " [[138, 161, 180],\n", + " [138, 161, 180],\n", + " [137, 160, 179],\n", + " ...,\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18],\n", + " [ 22, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[204, 221, 170],\n", + " [204, 221, 170],\n", + " [203, 220, 169],\n", + " ...,\n", + " [141, 158, 181],\n", + " [142, 159, 182],\n", + " [142, 159, 182]],\n", + "\n", + " [[204, 221, 170],\n", + " [203, 220, 169],\n", + " [203, 220, 169],\n", + " ...,\n", + " [142, 159, 182],\n", + " [142, 159, 182],\n", + " [141, 158, 181]],\n", + "\n", + " [[204, 221, 170],\n", + " [203, 220, 169],\n", + " [204, 221, 170],\n", + " ...,\n", + " [141, 158, 181],\n", + " [142, 159, 182],\n", + " [141, 158, 181]]], dtype=uint8), 'image': array([[[107, 101, 63],\n", + " [109, 103, 65],\n", + " [ 47, 75, 62],\n", + " ...,\n", + " [155, 155, 157],\n", + " [155, 155, 151],\n", + " [156, 156, 152]],\n", + "\n", + " [[118, 102, 68],\n", + " [122, 106, 72],\n", + " [ 57, 86, 68],\n", + " ...,\n", + " [157, 157, 159],\n", + " [156, 156, 152],\n", + " [156, 156, 152]],\n", + "\n", + " [[123, 94, 71],\n", + " [125, 96, 73],\n", + " [ 66, 98, 77],\n", + " ...,\n", + " [158, 157, 159],\n", + " [156, 156, 152],\n", + " [156, 156, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 70, 55],\n", + " [ 77, 76, 61],\n", + " [ 76, 78, 60],\n", + " ...,\n", + " [140, 141, 130],\n", + " [140, 141, 128],\n", + " [134, 135, 122]],\n", + "\n", + " [[ 63, 62, 45],\n", + " [ 65, 64, 47],\n", + " [ 58, 57, 42],\n", + " ...,\n", + " [143, 141, 133],\n", + " [140, 141, 128],\n", + " [137, 138, 125]],\n", + "\n", + " [[ 56, 53, 39],\n", + " [ 57, 54, 40],\n", + " [ 53, 51, 39],\n", + " ...,\n", + " [142, 138, 129],\n", + " [142, 140, 128],\n", + " [143, 141, 129]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4855387 , -1.1409982 , 2.1876225 , -2.6713645 , -1.5381626 ,\n", + " 2.679764 , 0.49294764, -0.05065339, -0.11059836, 0.66218203,\n", + " 0.74868876, 0.02483457, 0.0190603 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00533333, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01862857, 0.01217143, 0.00902857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 23, 19, 17]],\n", + "\n", + " [[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " [[136, 161, 182],\n", + " [135, 159, 181],\n", + " [135, 159, 181],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[199, 216, 166],\n", + " [199, 216, 166],\n", + " [198, 215, 164],\n", + " ...,\n", + " [135, 150, 172],\n", + " [136, 151, 174],\n", + " [136, 151, 174]],\n", + "\n", + " [[198, 215, 164],\n", + " [198, 215, 164],\n", + " [198, 215, 164],\n", + " ...,\n", + " [136, 151, 174],\n", + " [136, 151, 174],\n", + " [136, 151, 174]],\n", + "\n", + " [[198, 215, 164],\n", + " [198, 215, 164],\n", + " [198, 215, 164],\n", + " ...,\n", + " [136, 151, 174],\n", + " [136, 151, 174],\n", + " [136, 151, 174]]], dtype=uint8), 'image': array([[[113, 102, 79],\n", + " [112, 101, 78],\n", + " [ 54, 74, 69],\n", + " ...,\n", + " [224, 220, 209],\n", + " [224, 220, 211],\n", + " [225, 221, 212]],\n", + "\n", + " [[120, 101, 76],\n", + " [118, 99, 74],\n", + " [ 61, 85, 71],\n", + " ...,\n", + " [224, 220, 211],\n", + " [226, 219, 211],\n", + " [228, 221, 213]],\n", + "\n", + " [[116, 84, 62],\n", + " [128, 96, 74],\n", + " [ 76, 101, 82],\n", + " ...,\n", + " [224, 220, 211],\n", + " [226, 222, 213],\n", + " [227, 223, 214]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 73, 50],\n", + " [ 73, 76, 53],\n", + " [ 69, 72, 49],\n", + " ...,\n", + " [140, 140, 134],\n", + " [139, 139, 133],\n", + " [140, 140, 134]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 58, 62, 37],\n", + " [ 59, 63, 38],\n", + " ...,\n", + " [142, 140, 132],\n", + " [141, 138, 133],\n", + " [145, 142, 137]],\n", + "\n", + " [[ 52, 55, 32],\n", + " [ 49, 52, 29],\n", + " [ 48, 52, 31],\n", + " ...,\n", + " [145, 143, 133],\n", + " [142, 137, 133],\n", + " [146, 141, 137]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.500686 , -1.1089195 , 2.1561332 , -2.6699607 , -1.5347141 ,\n", + " 2.6646018 , 0.5029847 , -0.06102105, -0.11659307, 0.6621161 ,\n", + " 0.7487434 , 0.02574205, 0.0179702 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00761905, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01902857, 0.0112 , 0.00971429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[118, 147, 169],\n", + " [118, 147, 169],\n", + " [119, 148, 170],\n", + " ...,\n", + " [ 18, 17, 15],\n", + " [ 20, 16, 15],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[119, 145, 168],\n", + " [118, 144, 167],\n", + " [117, 145, 168],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 20, 16, 15],\n", + " [ 23, 19, 17]],\n", + "\n", + " [[118, 144, 167],\n", + " [118, 144, 167],\n", + " [117, 145, 168],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[177, 172, 159],\n", + " [178, 173, 160],\n", + " [179, 177, 157],\n", + " ...,\n", + " [138, 151, 174],\n", + " [138, 151, 174],\n", + " [138, 151, 174]],\n", + "\n", + " [[176, 171, 158],\n", + " [177, 172, 159],\n", + " [178, 176, 156],\n", + " ...,\n", + " [139, 152, 175],\n", + " [138, 151, 174],\n", + " [139, 152, 175]],\n", + "\n", + " [[173, 169, 156],\n", + " [175, 170, 157],\n", + " [177, 175, 154],\n", + " ...,\n", + " [140, 153, 176],\n", + " [140, 153, 176],\n", + " [141, 154, 177]]], dtype=uint8), 'image': array([[[107, 106, 78],\n", + " [101, 100, 72],\n", + " [ 36, 74, 57],\n", + " ...,\n", + " [207, 203, 192],\n", + " [198, 190, 187],\n", + " [186, 178, 175]],\n", + "\n", + " [[120, 108, 81],\n", + " [110, 98, 71],\n", + " [ 58, 95, 72],\n", + " ...,\n", + " [205, 201, 192],\n", + " [198, 192, 194],\n", + " [186, 180, 182]],\n", + "\n", + " [[108, 84, 57],\n", + " [122, 98, 71],\n", + " [ 72, 106, 82],\n", + " ...,\n", + " [207, 199, 196],\n", + " [197, 191, 195],\n", + " [186, 180, 184]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 53],\n", + " [ 72, 75, 57],\n", + " [ 70, 72, 56],\n", + " ...,\n", + " [146, 142, 133],\n", + " [144, 140, 131],\n", + " [142, 138, 129]],\n", + "\n", + " [[ 60, 64, 43],\n", + " [ 60, 64, 43],\n", + " [ 59, 63, 42],\n", + " ...,\n", + " [148, 141, 133],\n", + " [150, 143, 135],\n", + " [148, 141, 133]],\n", + "\n", + " [[ 54, 57, 36],\n", + " [ 54, 57, 36],\n", + " [ 48, 51, 33],\n", + " ...,\n", + " [149, 142, 134],\n", + " [148, 141, 133],\n", + " [149, 142, 134]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5155468 , -1.0804976 , 2.1259022 , -2.6659186 , -1.5311936 ,\n", + " 2.6497624 , 0.51242495, -0.07151319, -0.12196171, 0.66212773,\n", + " 0.74872553, 0.02683827, 0.01663197, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00590476, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01891429, 0.01205714, 0.01451429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 150, 173],\n", + " [120, 149, 172],\n", + " [121, 150, 173],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 16, 14, 14],\n", + " [ 17, 15, 15]],\n", + "\n", + " [[119, 148, 170],\n", + " [119, 148, 170],\n", + " [118, 147, 169],\n", + " ...,\n", + " [ 15, 12, 13],\n", + " [ 16, 14, 14],\n", + " [ 16, 14, 14]],\n", + "\n", + " [[121, 147, 170],\n", + " [121, 147, 170],\n", + " [120, 146, 169],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 160, 168],\n", + " [154, 160, 168],\n", + " [154, 159, 170],\n", + " ...,\n", + " [152, 157, 175],\n", + " [152, 157, 175],\n", + " [152, 157, 175]],\n", + "\n", + " [[154, 160, 168],\n", + " [153, 159, 167],\n", + " [154, 159, 170],\n", + " ...,\n", + " [152, 157, 175],\n", + " [152, 157, 175],\n", + " [152, 157, 175]],\n", + "\n", + " [[154, 160, 168],\n", + " [154, 160, 168],\n", + " [153, 158, 169],\n", + " ...,\n", + " [151, 156, 173],\n", + " [151, 156, 173],\n", + " [151, 156, 173]]], dtype=uint8), 'image': array([[[111, 100, 77],\n", + " [113, 102, 79],\n", + " [ 51, 71, 68],\n", + " ...,\n", + " [137, 132, 128],\n", + " [136, 136, 130],\n", + " [138, 138, 132]],\n", + "\n", + " [[120, 101, 76],\n", + " [124, 105, 80],\n", + " [ 62, 86, 74],\n", + " ...,\n", + " [139, 136, 131],\n", + " [138, 135, 130],\n", + " [140, 137, 132]],\n", + "\n", + " [[120, 86, 62],\n", + " [130, 96, 72],\n", + " [ 80, 102, 84],\n", + " ...,\n", + " [138, 135, 130],\n", + " [140, 137, 132],\n", + " [141, 138, 133]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 72, 49],\n", + " [ 74, 77, 54],\n", + " [ 73, 76, 55],\n", + " ...,\n", + " [148, 144, 137],\n", + " [148, 144, 135],\n", + " [143, 139, 130]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 60, 64, 39],\n", + " [ 59, 62, 39],\n", + " ...,\n", + " [148, 144, 135],\n", + " [149, 145, 136],\n", + " [147, 143, 134]],\n", + "\n", + " [[ 53, 56, 31],\n", + " [ 52, 55, 30],\n", + " [ 45, 49, 24],\n", + " ...,\n", + " [149, 145, 136],\n", + " [149, 145, 136],\n", + " [149, 145, 136]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5310063 , -1.052627 , 2.096078 , -2.660988 , -1.5265115 ,\n", + " 2.6343114 , 0.5215374 , -0.08259036, -0.12778392, 0.66204727,\n", + " 0.74878216, 0.0282663 , 0.01482305, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00361905, 0. , 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01805714, 0.01268571, 0.01908571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 145, 168],\n", + " [120, 144, 167],\n", + " [120, 144, 167],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 19, 15, 14],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[120, 144, 167],\n", + " [120, 144, 167],\n", + " [118, 142, 166],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 19, 15, 14],\n", + " [ 21, 17, 16]],\n", + "\n", + " [[120, 144, 167],\n", + " [120, 144, 167],\n", + " [118, 142, 166],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 21, 17, 16],\n", + " [ 24, 20, 18]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 157, 175],\n", + " [154, 157, 175],\n", + " [154, 158, 173],\n", + " ...,\n", + " [150, 157, 176],\n", + " [150, 157, 176],\n", + " [150, 157, 176]],\n", + "\n", + " [[150, 153, 171],\n", + " [150, 153, 171],\n", + " [151, 155, 171],\n", + " ...,\n", + " [151, 158, 177],\n", + " [151, 158, 177],\n", + " [150, 157, 176]],\n", + "\n", + " [[147, 150, 168],\n", + " [148, 151, 169],\n", + " [149, 153, 168],\n", + " ...,\n", + " [152, 159, 178],\n", + " [151, 158, 177],\n", + " [151, 158, 177]]], dtype=uint8), 'image': array([[[113, 102, 79],\n", + " [107, 96, 73],\n", + " [ 49, 73, 71],\n", + " ...,\n", + " [159, 158, 158],\n", + " [158, 157, 161],\n", + " [158, 157, 161]],\n", + "\n", + " [[118, 99, 74],\n", + " [119, 100, 75],\n", + " [ 61, 86, 77],\n", + " ...,\n", + " [159, 159, 161],\n", + " [157, 157, 161],\n", + " [154, 154, 158]],\n", + "\n", + " [[109, 77, 53],\n", + " [135, 103, 79],\n", + " [ 77, 104, 86],\n", + " ...,\n", + " [158, 157, 164],\n", + " [152, 151, 160],\n", + " [147, 146, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 69, 49],\n", + " [ 72, 72, 52],\n", + " [ 66, 69, 46],\n", + " ...,\n", + " [148, 146, 138],\n", + " [147, 144, 141],\n", + " [146, 143, 140]],\n", + "\n", + " [[ 63, 63, 41],\n", + " [ 62, 62, 40],\n", + " [ 57, 57, 37],\n", + " ...,\n", + " [145, 146, 137],\n", + " [146, 143, 140],\n", + " [146, 143, 140]],\n", + "\n", + " [[ 57, 55, 34],\n", + " [ 56, 54, 33],\n", + " [ 50, 50, 28],\n", + " ...,\n", + " [148, 149, 138],\n", + " [148, 145, 140],\n", + " [149, 146, 141]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5453887 , -1.0169486 , 2.0630584 , -2.6606886 , -1.5223564 ,\n", + " 2.6199625 , 0.531208 , -0.0935498 , -0.13631946, 0.6620134 ,\n", + " 0.748793 , 0.02956677, 0.01315534, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0.0007619 , 0.00152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01308571, 0.01091429, 0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[100, 127, 159],\n", + " [ 98, 126, 158],\n", + " [100, 127, 159],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 12, 12, 12],\n", + " [ 12, 12, 12]],\n", + "\n", + " [[ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " [[100, 127, 159],\n", + " [ 98, 126, 158],\n", + " [ 98, 126, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " ...,\n", + "\n", + " [[146, 148, 168],\n", + " [146, 148, 168],\n", + " [142, 147, 167],\n", + " ...,\n", + " [149, 153, 175],\n", + " [149, 153, 175],\n", + " [149, 153, 175]],\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [145, 149, 169],\n", + " ...,\n", + " [149, 153, 175],\n", + " [149, 153, 175],\n", + " [149, 153, 175]],\n", + "\n", + " [[146, 148, 168],\n", + " [147, 149, 169],\n", + " [145, 149, 169],\n", + " ...,\n", + " [152, 156, 178],\n", + " [149, 153, 175],\n", + " [149, 153, 175]]], dtype=uint8), 'image': array([[[105, 98, 77],\n", + " [103, 96, 75],\n", + " [ 48, 72, 74],\n", + " ...,\n", + " [158, 158, 154],\n", + " [157, 152, 148],\n", + " [159, 154, 150]],\n", + "\n", + " [[119, 101, 79],\n", + " [113, 95, 73],\n", + " [ 53, 80, 75],\n", + " ...,\n", + " [156, 155, 155],\n", + " [156, 153, 150],\n", + " [156, 153, 150]],\n", + "\n", + " [[124, 94, 76],\n", + " [123, 93, 75],\n", + " [ 69, 97, 84],\n", + " ...,\n", + " [156, 155, 157],\n", + " [148, 147, 149],\n", + " [148, 147, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 72, 71, 54],\n", + " [ 78, 77, 60],\n", + " [ 76, 75, 58],\n", + " ...,\n", + " [151, 144, 134],\n", + " [147, 145, 135],\n", + " [145, 143, 133]],\n", + "\n", + " [[ 59, 63, 42],\n", + " [ 57, 61, 40],\n", + " [ 55, 58, 37],\n", + " ...,\n", + " [151, 144, 134],\n", + " [148, 144, 135],\n", + " [148, 144, 135]],\n", + "\n", + " [[ 50, 53, 35],\n", + " [ 52, 55, 37],\n", + " [ 48, 51, 33],\n", + " ...,\n", + " [151, 144, 134],\n", + " [149, 145, 136],\n", + " [149, 145, 136]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.556693 , -0.9751294 , 2.0289373 , -2.666302 , -1.5200332 ,\n", + " 2.6086464 , 0.5411583 , -0.10370405, -0.14767449, 0.66197366,\n", + " 0.748819 , 0.03023103, 0.01212799, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0.00095238, 0.01142857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00805714, 0. , 0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[102, 127, 159],\n", + " [100, 126, 158],\n", + " [102, 128, 158],\n", + " ...,\n", + " [ 12, 12, 12],\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13]],\n", + "\n", + " [[102, 127, 159],\n", + " [100, 126, 158],\n", + " [100, 127, 157],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14]],\n", + "\n", + " [[104, 128, 158],\n", + " [104, 128, 158],\n", + " [102, 128, 158],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [149, 152, 170],\n", + " [150, 153, 171],\n", + " [150, 153, 171]],\n", + "\n", + " [[147, 149, 169],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [149, 152, 170],\n", + " [148, 151, 169],\n", + " [149, 152, 170]],\n", + "\n", + " [[146, 148, 168],\n", + " [147, 149, 169],\n", + " [147, 149, 169],\n", + " ...,\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " [148, 151, 169]]], dtype=uint8), 'image': array([[[108, 103, 74],\n", + " [105, 100, 71],\n", + " [ 44, 71, 62],\n", + " ...,\n", + " [158, 154, 155],\n", + " [158, 154, 157],\n", + " [157, 153, 156]],\n", + "\n", + " [[119, 102, 73],\n", + " [123, 106, 77],\n", + " [ 62, 93, 77],\n", + " ...,\n", + " [155, 154, 154],\n", + " [156, 155, 157],\n", + " [154, 153, 155]],\n", + "\n", + " [[120, 91, 66],\n", + " [127, 98, 73],\n", + " [ 71, 100, 80],\n", + " ...,\n", + " [152, 152, 156],\n", + " [148, 147, 151],\n", + " [148, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 68, 49],\n", + " [ 67, 73, 54],\n", + " [ 64, 71, 49],\n", + " ...,\n", + " [150, 146, 137],\n", + " [147, 143, 134],\n", + " [142, 138, 129]],\n", + "\n", + " [[ 56, 63, 41],\n", + " [ 56, 63, 41],\n", + " [ 57, 64, 42],\n", + " ...,\n", + " [148, 144, 135],\n", + " [148, 144, 135],\n", + " [148, 144, 135]],\n", + "\n", + " [[ 50, 56, 39],\n", + " [ 50, 56, 39],\n", + " [ 46, 52, 33],\n", + " ...,\n", + " [148, 146, 136],\n", + " [147, 145, 135],\n", + " [147, 145, 135]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5630758 , -0.9333814 , 1.9979777 , -2.6762102 , -1.5196503 ,\n", + " 2.6031442 , 0.5501979 , -0.11132966, -0.16009057, 0.66227996,\n", + " 0.7485525 , 0.03024558, 0.01181715, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00704762, 0. , 0.02038095], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01777143, 0. , 0.00914286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[112, 130, 150],\n", + " [ 97, 115, 135],\n", + " [ 83, 101, 109],\n", + " ...,\n", + " [ 17, 15, 15],\n", + " [ 18, 16, 16],\n", + " [ 18, 16, 16]],\n", + "\n", + " [[112, 129, 151],\n", + " [102, 120, 142],\n", + " [ 89, 106, 117],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15]],\n", + "\n", + " [[113, 133, 155],\n", + " [105, 125, 147],\n", + " [ 97, 113, 126],\n", + " ...,\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15],\n", + " [ 17, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 148, 168],\n", + " [144, 148, 168],\n", + " [144, 148, 168],\n", + " ...,\n", + " [139, 149, 172],\n", + " [139, 149, 172],\n", + " [139, 149, 172]],\n", + "\n", + " [[139, 144, 163],\n", + " [140, 145, 165],\n", + " [139, 144, 163],\n", + " ...,\n", + " [140, 150, 174],\n", + " [139, 149, 172],\n", + " [140, 150, 174]],\n", + "\n", + " [[139, 144, 163],\n", + " [139, 144, 163],\n", + " [139, 144, 163],\n", + " ...,\n", + " [140, 150, 174],\n", + " [140, 150, 174],\n", + " [139, 149, 172]]], dtype=uint8), 'image': array([[[107, 100, 71],\n", + " [109, 102, 73],\n", + " [ 46, 69, 65],\n", + " ...,\n", + " [152, 155, 158],\n", + " [153, 156, 157],\n", + " [152, 155, 156]],\n", + "\n", + " [[114, 95, 68],\n", + " [122, 103, 76],\n", + " [ 52, 78, 65],\n", + " ...,\n", + " [150, 153, 156],\n", + " [152, 155, 156],\n", + " [151, 154, 155]],\n", + "\n", + " [[118, 86, 64],\n", + " [127, 95, 73],\n", + " [ 76, 101, 82],\n", + " ...,\n", + " [150, 152, 158],\n", + " [149, 152, 155],\n", + " [148, 151, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 73, 76, 53],\n", + " [ 75, 78, 55],\n", + " [ 68, 72, 47],\n", + " ...,\n", + " [150, 148, 140],\n", + " [148, 146, 138],\n", + " [141, 139, 131]],\n", + "\n", + " [[ 58, 61, 38],\n", + " [ 59, 62, 39],\n", + " [ 55, 58, 35],\n", + " ...,\n", + " [150, 146, 139],\n", + " [150, 146, 137],\n", + " [147, 143, 134]],\n", + "\n", + " [[ 53, 53, 33],\n", + " [ 51, 51, 31],\n", + " [ 49, 52, 31],\n", + " ...,\n", + " [148, 144, 137],\n", + " [148, 144, 137],\n", + " [150, 146, 139]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.569471 , -0.8993924 , 1.9853354 , -2.6972144 , -1.5198654 ,\n", + " 2.60294 , 0.5531312 , -0.11650646, -0.1728949 , 0.6645783 ,\n", + " 0.7465192 , 0.03010343, 0.01176925, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00095238, -0.00552381, 0.01314286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01902857, -0.00474286, 0.00297143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 13, 13, 13],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 150, 168],\n", + " [147, 150, 168],\n", + " [147, 150, 168],\n", + " ...,\n", + " [135, 146, 169],\n", + " [136, 147, 170],\n", + " [136, 147, 170]],\n", + "\n", + " [[148, 151, 169],\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " ...,\n", + " [133, 143, 167],\n", + " [134, 144, 168],\n", + " [134, 144, 168]],\n", + "\n", + " [[149, 152, 170],\n", + " [148, 151, 169],\n", + " [148, 151, 169],\n", + " ...,\n", + " [133, 143, 167],\n", + " [134, 144, 168],\n", + " [134, 144, 168]]], dtype=uint8), 'image': array([[[109, 101, 77],\n", + " [109, 101, 77],\n", + " [ 42, 67, 62],\n", + " ...,\n", + " [155, 155, 151],\n", + " [156, 151, 149],\n", + " [157, 152, 150]],\n", + "\n", + " [[122, 103, 78],\n", + " [124, 105, 80],\n", + " [ 62, 86, 72],\n", + " ...,\n", + " [152, 152, 152],\n", + " [152, 151, 151],\n", + " [153, 152, 152]],\n", + "\n", + " [[126, 93, 76],\n", + " [127, 94, 77],\n", + " [ 78, 105, 85],\n", + " ...,\n", + " [150, 153, 154],\n", + " [151, 151, 153],\n", + " [146, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 70, 48],\n", + " [ 67, 74, 52],\n", + " [ 66, 73, 51],\n", + " ...,\n", + " [148, 146, 136],\n", + " [146, 143, 138],\n", + " [143, 140, 135]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 57, 63, 44],\n", + " [ 60, 63, 45],\n", + " ...,\n", + " [149, 144, 140],\n", + " [149, 144, 140],\n", + " [147, 142, 138]],\n", + "\n", + " [[ 51, 52, 41],\n", + " [ 52, 53, 42],\n", + " [ 47, 50, 34],\n", + " ...,\n", + " [152, 145, 139],\n", + " [149, 144, 140],\n", + " [148, 143, 139]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5539145 , -0.88672715, 1.9930334 , -2.7198975 , -1.5237097 ,\n", + " 2.630835 , 0.5523699 , -0.10781911, -0.17971404, 0.6691864 ,\n", + " 0.7424202 , 0.02894053, 0.0128143 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00266667, -0.00590476, 0.0112381 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01988572, -0.00342857, 0.0024 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 15, 15, 15],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " ...,\n", + " [ 16, 14, 14],\n", + " [ 14, 14, 14],\n", + " [ 15, 15, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[147, 150, 168],\n", + " [147, 150, 168],\n", + " [147, 150, 168],\n", + " ...,\n", + " [134, 144, 168],\n", + " [134, 144, 168],\n", + " [134, 144, 168]],\n", + "\n", + " [[148, 151, 169],\n", + " [148, 151, 169],\n", + " [147, 150, 168],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [132, 142, 165]],\n", + "\n", + " [[148, 151, 169],\n", + " [147, 150, 168],\n", + " [146, 149, 166],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [132, 142, 165]]], dtype=uint8), 'image': array([[[104, 101, 71],\n", + " [101, 98, 68],\n", + " [ 43, 70, 63],\n", + " ...,\n", + " [157, 157, 161],\n", + " [155, 154, 156],\n", + " [154, 153, 155]],\n", + "\n", + " [[116, 101, 73],\n", + " [119, 104, 76],\n", + " [ 63, 91, 76],\n", + " ...,\n", + " [154, 154, 158],\n", + " [154, 153, 157],\n", + " [152, 151, 155]],\n", + "\n", + " [[110, 86, 59],\n", + " [125, 101, 74],\n", + " [ 75, 102, 80],\n", + " ...,\n", + " [152, 155, 158],\n", + " [146, 146, 150],\n", + " [147, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 69, 46],\n", + " [ 69, 72, 49],\n", + " [ 65, 68, 52],\n", + " ...,\n", + " [146, 145, 130],\n", + " [147, 143, 134],\n", + " [146, 142, 133]],\n", + "\n", + " [[ 62, 65, 40],\n", + " [ 61, 64, 39],\n", + " [ 55, 57, 41],\n", + " ...,\n", + " [146, 144, 132],\n", + " [146, 143, 138],\n", + " [147, 144, 139]],\n", + "\n", + " [[ 55, 55, 35],\n", + " [ 54, 54, 34],\n", + " [ 46, 48, 30],\n", + " ...,\n", + " [146, 144, 136],\n", + " [143, 143, 137],\n", + " [143, 143, 137]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5357125 , -0.88817865, 2.0125475 , -2.7420037 , -1.524392 ,\n", + " 2.6583688 , 0.5487908 , -0.09566072, -0.18201101, 0.6725909 ,\n", + " 0.7392859 , 0.02962112, 0.01415604, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00285714, -0.00609524, 0.01180952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01977143, -0.00291429, 0.00234286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 13, 11],\n", + " [ 15, 13, 11],\n", + " [ 14, 12, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 14, 12, 10],\n", + " [ 14, 12, 10],\n", + " [ 12, 11, 9],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14]],\n", + "\n", + " [[ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 10, 11, 9],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14],\n", + " [ 17, 16, 14]],\n", + "\n", + " ...,\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [139, 146, 167],\n", + " [138, 146, 170],\n", + " [138, 146, 170]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [138, 144, 166],\n", + " [136, 144, 168],\n", + " [135, 143, 167]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [138, 144, 166],\n", + " [135, 143, 167],\n", + " [135, 143, 167]]], dtype=uint8), 'image': array([[[109, 97, 68],\n", + " [112, 100, 71],\n", + " [ 49, 69, 64],\n", + " ...,\n", + " [153, 152, 152],\n", + " [156, 155, 155],\n", + " [155, 154, 154]],\n", + "\n", + " [[119, 97, 71],\n", + " [124, 102, 76],\n", + " [ 59, 83, 69],\n", + " ...,\n", + " [151, 151, 151],\n", + " [150, 150, 150],\n", + " [150, 150, 150]],\n", + "\n", + " [[117, 83, 59],\n", + " [128, 94, 70],\n", + " [ 74, 99, 80],\n", + " ...,\n", + " [150, 150, 154],\n", + " [145, 145, 147],\n", + " [144, 144, 146]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 71, 43],\n", + " [ 72, 77, 49],\n", + " [ 67, 71, 48],\n", + " ...,\n", + " [141, 142, 133],\n", + " [140, 140, 134],\n", + " [139, 139, 133]],\n", + "\n", + " [[ 54, 59, 31],\n", + " [ 55, 60, 32],\n", + " [ 52, 59, 33],\n", + " ...,\n", + " [143, 144, 133],\n", + " [143, 144, 135],\n", + " [143, 144, 135]],\n", + "\n", + " [[ 53, 57, 32],\n", + " [ 51, 55, 30],\n", + " [ 45, 52, 28],\n", + " ...,\n", + " [143, 144, 135],\n", + " [143, 144, 135],\n", + " [143, 144, 135]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.5170557 , -0.8910335 , 2.0314274 , -2.7631152 , -1.5247868 ,\n", + " 2.6843548 , 0.545448 , -0.08352495, -0.18336658, 0.67522 ,\n", + " 0.7368057 , 0.03068829, 0.01592391, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00419048, -0.0072381 , 0.0127619 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01982857, -0.00034286, 0.00194286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 15, 15],\n", + " [ 14, 14, 14],\n", + " [ 13, 13, 13],\n", + " ...,\n", + " [ 19, 18, 16],\n", + " [ 19, 18, 16],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 12, 12, 12],\n", + " [ 12, 12, 12],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " [[ 10, 12, 12],\n", + " [ 10, 12, 12],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 17, 16, 14],\n", + " [ 18, 17, 15],\n", + " [ 18, 17, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 157, 170],\n", + " [153, 157, 170],\n", + " [151, 155, 171],\n", + " ...,\n", + " [147, 149, 169],\n", + " [148, 149, 172],\n", + " [148, 149, 172]],\n", + "\n", + " [[154, 158, 171],\n", + " [154, 158, 171],\n", + " [154, 158, 173],\n", + " ...,\n", + " [148, 150, 170],\n", + " [148, 149, 172],\n", + " [149, 151, 173]],\n", + "\n", + " [[153, 157, 170],\n", + " [154, 158, 171],\n", + " [154, 158, 173],\n", + " ...,\n", + " [148, 150, 170],\n", + " [147, 148, 171],\n", + " [148, 149, 172]]], dtype=uint8), 'image': array([[[103, 96, 79],\n", + " [105, 98, 81],\n", + " [ 48, 72, 72],\n", + " ...,\n", + " [153, 154, 148],\n", + " [152, 152, 148],\n", + " [152, 152, 148]],\n", + "\n", + " [[112, 95, 77],\n", + " [112, 95, 77],\n", + " [ 58, 83, 76],\n", + " ...,\n", + " [152, 153, 149],\n", + " [151, 151, 151],\n", + " [149, 149, 149]],\n", + "\n", + " [[112, 82, 64],\n", + " [121, 91, 73],\n", + " [ 75, 101, 86],\n", + " ...,\n", + " [148, 151, 152],\n", + " [147, 147, 151],\n", + " [145, 145, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 69, 46],\n", + " [ 68, 71, 48],\n", + " [ 65, 68, 43],\n", + " ...,\n", + " [139, 137, 129],\n", + " [142, 140, 130],\n", + " [139, 137, 127]],\n", + "\n", + " [[ 58, 61, 38],\n", + " [ 54, 57, 34],\n", + " [ 49, 52, 27],\n", + " ...,\n", + " [141, 139, 131],\n", + " [142, 140, 130],\n", + " [142, 140, 130]],\n", + "\n", + " [[ 51, 51, 31],\n", + " [ 51, 51, 31],\n", + " [ 45, 49, 21],\n", + " ...,\n", + " [140, 138, 130],\n", + " [142, 140, 130],\n", + " [143, 141, 131]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4967704 , -0.89257985, 2.048069 , -2.7829902 , -1.5259126 ,\n", + " 2.7121851 , 0.5425027 , -0.0710614 , -0.1848708 , 0.67793345,\n", + " 0.734238 , 0.03148263, 0.01759424, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00457143, -0.00704762, 0.01142857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, -5.7142857e-05, 2.0571428e-03], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 17, 16],\n", + " [ 11, 16, 15],\n", + " [ 10, 12, 12],\n", + " ...,\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15],\n", + " [ 15, 16, 14]],\n", + "\n", + " [[ 11, 13, 13],\n", + " [ 11, 13, 13],\n", + " [ 12, 12, 12],\n", + " ...,\n", + " [ 15, 16, 14],\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15]],\n", + "\n", + " [[ 11, 13, 13],\n", + " [ 11, 13, 13],\n", + " [ 12, 12, 12],\n", + " ...,\n", + " [ 15, 16, 14],\n", + " [ 16, 17, 15],\n", + " [ 16, 17, 15]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [149, 151, 172],\n", + " [148, 153, 173],\n", + " [148, 153, 173]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [150, 153, 173],\n", + " [147, 152, 172],\n", + " [147, 152, 172]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [151, 154, 174],\n", + " [147, 152, 172],\n", + " [147, 152, 172]]], dtype=uint8), 'image': array([[[103, 99, 61],\n", + " [104, 100, 62],\n", + " [ 49, 76, 58],\n", + " ...,\n", + " [153, 154, 150],\n", + " [152, 152, 148],\n", + " [152, 152, 148]],\n", + "\n", + " [[114, 101, 63],\n", + " [117, 104, 66],\n", + " [ 52, 80, 55],\n", + " ...,\n", + " [150, 153, 152],\n", + " [151, 151, 151],\n", + " [150, 150, 150]],\n", + "\n", + " [[113, 87, 56],\n", + " [123, 97, 66],\n", + " [ 75, 104, 74],\n", + " ...,\n", + " [146, 152, 154],\n", + " [147, 147, 149],\n", + " [145, 145, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 45],\n", + " [ 70, 73, 48],\n", + " [ 65, 69, 46],\n", + " ...,\n", + " [138, 136, 124],\n", + " [134, 136, 120],\n", + " [133, 135, 119]],\n", + "\n", + " [[ 58, 61, 36],\n", + " [ 56, 59, 34],\n", + " [ 54, 57, 34],\n", + " ...,\n", + " [138, 137, 122],\n", + " [138, 137, 122],\n", + " [138, 137, 122]],\n", + "\n", + " [[ 54, 57, 32],\n", + " [ 51, 54, 29],\n", + " [ 47, 51, 30],\n", + " ...,\n", + " [139, 138, 123],\n", + " [138, 137, 122],\n", + " [138, 137, 122]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4744623 , -0.8928798 , 2.0601006 , -2.8009937 , -1.5276612 ,\n", + " 2.7427585 , 0.54119325, -0.05844606, -0.18594792, 0.6808814 ,\n", + " 0.7314025 , 0.03243729, 0.01996658, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00514286, -0.00666667, 0.01314286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01982857, 0. , 0.00194286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 14, 14],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12],\n", + " [ 17, 16, 14]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 8, 11, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 15, 13, 11],\n", + " [ 16, 14, 12],\n", + " [ 16, 14, 12]],\n", + "\n", + " ...,\n", + "\n", + " [[143, 150, 169],\n", + " [143, 150, 169],\n", + " [143, 150, 169],\n", + " ...,\n", + " [145, 149, 171],\n", + " [145, 149, 169],\n", + " [145, 149, 169]],\n", + "\n", + " [[145, 152, 172],\n", + " [145, 152, 172],\n", + " [145, 152, 172],\n", + " ...,\n", + " [145, 149, 171],\n", + " [146, 151, 170],\n", + " [145, 149, 169]],\n", + "\n", + " [[145, 152, 172],\n", + " [146, 153, 173],\n", + " [146, 153, 173],\n", + " ...,\n", + " [145, 149, 171],\n", + " [146, 151, 170],\n", + " [145, 149, 169]]], dtype=uint8), 'image': array([[[111, 99, 81],\n", + " [106, 94, 76],\n", + " [ 49, 73, 73],\n", + " ...,\n", + " [156, 156, 150],\n", + " [152, 153, 149],\n", + " [152, 153, 149]],\n", + "\n", + " [[121, 101, 81],\n", + " [115, 95, 75],\n", + " [ 55, 82, 75],\n", + " ...,\n", + " [152, 153, 149],\n", + " [152, 152, 152],\n", + " [150, 150, 150]],\n", + "\n", + " [[120, 88, 68],\n", + " [125, 93, 73],\n", + " [ 68, 96, 81],\n", + " ...,\n", + " [149, 152, 151],\n", + " [146, 146, 148],\n", + " [145, 145, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 68, 45],\n", + " [ 66, 70, 47],\n", + " [ 62, 65, 40],\n", + " ...,\n", + " [131, 129, 121],\n", + " [134, 130, 121],\n", + " [132, 128, 119]],\n", + "\n", + " [[ 59, 63, 38],\n", + " [ 57, 61, 36],\n", + " [ 54, 58, 35],\n", + " ...,\n", + " [133, 129, 120],\n", + " [133, 129, 120],\n", + " [134, 130, 121]],\n", + "\n", + " [[ 55, 59, 36],\n", + " [ 50, 54, 31],\n", + " [ 45, 49, 28],\n", + " ...,\n", + " [134, 130, 121],\n", + " [133, 129, 120],\n", + " [135, 131, 122]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.451041 , -0.89231616, 2.0696073 , -2.817558 , -1.5302237 ,\n", + " 2.7739055 , 0.54045534, -0.0457982 , -0.18693021, 0.6836287 ,\n", + " 0.728724 , 0.03323772, 0.02254771, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00457143, -0.00438095, 0.01333333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01977143, 0. , 0.00228571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 14, 14],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 13, 14, 11],\n", + " [ 13, 14, 11],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " [ 10, 10, 10],\n", + " ...,\n", + " [ 14, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " ...,\n", + "\n", + " [[142, 151, 172],\n", + " [142, 151, 172],\n", + " [141, 149, 171],\n", + " ...,\n", + " [143, 145, 167],\n", + " [144, 146, 168],\n", + " [144, 146, 168]],\n", + "\n", + " [[141, 149, 171],\n", + " [141, 149, 171],\n", + " [141, 149, 171],\n", + " ...,\n", + " [144, 146, 168],\n", + " [142, 144, 166],\n", + " [141, 142, 165]],\n", + "\n", + " [[142, 151, 172],\n", + " [142, 151, 172],\n", + " [142, 151, 172],\n", + " ...,\n", + " [144, 146, 168],\n", + " [143, 145, 167],\n", + " [143, 145, 167]]], dtype=uint8), 'image': array([[[109, 102, 71],\n", + " [111, 104, 73],\n", + " [ 44, 69, 62],\n", + " ...,\n", + " [151, 155, 150],\n", + " [151, 154, 153],\n", + " [149, 152, 151]],\n", + "\n", + " [[117, 95, 69],\n", + " [121, 99, 73],\n", + " [ 66, 90, 76],\n", + " ...,\n", + " [150, 153, 154],\n", + " [147, 150, 151],\n", + " [147, 150, 151]],\n", + "\n", + " [[114, 80, 56],\n", + " [124, 90, 66],\n", + " [ 77, 99, 79],\n", + " ...,\n", + " [148, 151, 152],\n", + " [143, 146, 149],\n", + " [143, 146, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 59, 71, 47],\n", + " [ 62, 74, 50],\n", + " [ 61, 70, 50],\n", + " ...,\n", + " [125, 123, 113],\n", + " [123, 124, 113],\n", + " [120, 121, 110]],\n", + "\n", + " [[ 47, 62, 35],\n", + " [ 48, 63, 36],\n", + " [ 52, 64, 40],\n", + " ...,\n", + " [124, 125, 114],\n", + " [126, 124, 114],\n", + " [125, 123, 113]],\n", + "\n", + " [[ 45, 57, 31],\n", + " [ 43, 55, 29],\n", + " [ 42, 54, 28],\n", + " ...,\n", + " [127, 128, 117],\n", + " [128, 126, 116],\n", + " [127, 125, 115]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4267576 , -0.8909252 , 2.0777245 , -2.832958 , -1.5336002 ,\n", + " 2.8063924 , 0.5396934 , -0.03303649, -0.18807867, 0.6865643 ,\n", + " 0.7258565 , 0.03373102, 0.02497944, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00495238, -0.0047619 , 0.01333333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.01977143, 0. , 0.00177143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 15, 12],\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 14, 12, 10]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " [[ 8, 12, 9],\n", + " [ 8, 12, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 155, 172],\n", + " [149, 155, 172],\n", + " [148, 154, 171],\n", + " ...,\n", + " [144, 148, 168],\n", + " [145, 149, 169],\n", + " [145, 149, 169]],\n", + "\n", + " [[149, 155, 172],\n", + " [151, 156, 173],\n", + " [151, 156, 173],\n", + " ...,\n", + " [144, 148, 168],\n", + " [144, 148, 168],\n", + " [145, 149, 169]],\n", + "\n", + " [[149, 155, 172],\n", + " [148, 154, 171],\n", + " [148, 154, 171],\n", + " ...,\n", + " [145, 149, 169],\n", + " [145, 149, 169],\n", + " [145, 149, 169]]], dtype=uint8), 'image': array([[[111, 104, 71],\n", + " [103, 96, 63],\n", + " [ 54, 74, 67],\n", + " ...,\n", + " [154, 153, 153],\n", + " [153, 152, 152],\n", + " [153, 152, 152]],\n", + "\n", + " [[120, 105, 68],\n", + " [111, 96, 59],\n", + " [ 62, 83, 70],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[120, 89, 60],\n", + " [122, 91, 62],\n", + " [ 78, 100, 82],\n", + " ...,\n", + " [152, 152, 156],\n", + " [146, 145, 152],\n", + " [144, 143, 150]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 74, 48],\n", + " [ 69, 76, 50],\n", + " [ 64, 71, 47],\n", + " ...,\n", + " [115, 112, 96],\n", + " [115, 114, 97],\n", + " [114, 113, 96]],\n", + "\n", + " [[ 54, 58, 33],\n", + " [ 57, 61, 36],\n", + " [ 60, 67, 41],\n", + " ...,\n", + " [119, 118, 103],\n", + " [118, 117, 102],\n", + " [117, 116, 101]],\n", + "\n", + " [[ 52, 55, 34],\n", + " [ 48, 51, 30],\n", + " [ 45, 49, 24],\n", + " ...,\n", + " [120, 119, 104],\n", + " [118, 117, 102],\n", + " [118, 117, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.4022958 , -0.88859874, 2.0833273 , -2.8456123 , -1.5374321 ,\n", + " 2.8394403 , 0.5391687 , -0.02050408, -0.18934578, 0.6896574 ,\n", + " 0.7228375 , 0.03391662, 0.02698694, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00438095, -0.00533333, 0.012 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([0.02 , 0. , 0.0008], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 14, 13, 8],\n", + " [ 14, 13, 8],\n", + " [ 14, 13, 8]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[148, 154, 171],\n", + " [148, 154, 171],\n", + " [149, 151, 172],\n", + " ...,\n", + " [140, 147, 168],\n", + " [142, 148, 170],\n", + " [142, 148, 170]],\n", + "\n", + " [[147, 153, 170],\n", + " [146, 151, 169],\n", + " [149, 151, 172],\n", + " ...,\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " [142, 148, 170]],\n", + "\n", + " [[146, 151, 169],\n", + " [147, 153, 170],\n", + " [149, 151, 172],\n", + " ...,\n", + " [142, 148, 170],\n", + " [143, 149, 171],\n", + " [142, 148, 170]]], dtype=uint8), 'image': array([[[103, 98, 69],\n", + " [105, 100, 71],\n", + " [ 44, 71, 66],\n", + " ...,\n", + " [154, 153, 157],\n", + " [156, 152, 157],\n", + " [154, 150, 155]],\n", + "\n", + " [[111, 94, 69],\n", + " [123, 106, 81],\n", + " [ 63, 90, 81],\n", + " ...,\n", + " [152, 152, 156],\n", + " [151, 150, 154],\n", + " [152, 151, 155]],\n", + "\n", + " [[111, 79, 57],\n", + " [128, 96, 74],\n", + " [ 75, 100, 83],\n", + " ...,\n", + " [147, 149, 155],\n", + " [146, 146, 150],\n", + " [144, 144, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 66, 41],\n", + " [ 67, 70, 45],\n", + " [ 68, 69, 40],\n", + " ...,\n", + " [105, 104, 89],\n", + " [108, 107, 90],\n", + " [108, 107, 90]],\n", + "\n", + " [[ 56, 60, 30],\n", + " [ 59, 63, 33],\n", + " [ 60, 65, 28],\n", + " ...,\n", + " [110, 109, 94],\n", + " [111, 110, 95],\n", + " [107, 106, 91]],\n", + "\n", + " [[ 51, 53, 21],\n", + " [ 51, 53, 21],\n", + " [ 49, 52, 13],\n", + " ...,\n", + " [112, 111, 96],\n", + " [113, 112, 97],\n", + " [111, 110, 95]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.377918 , -0.88633186, 2.08747 , -2.8563583 , -1.5414795 ,\n", + " 2.8719716 , 0.5387577 , -0.00821607, -0.19039896, 0.6925935 ,\n", + " 0.7199444 , 0.03404387, 0.02890096, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.0047619 , -0.00609524, 0.01047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , -0.00022857, 0.0004 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 14, 13, 8]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 8, 11, 10],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [151, 155, 172],\n", + " ...,\n", + " [134, 142, 165],\n", + " [135, 143, 167],\n", + " [135, 143, 167]],\n", + "\n", + " [[148, 151, 169],\n", + " [150, 153, 171],\n", + " [150, 153, 171],\n", + " ...,\n", + " [136, 144, 168],\n", + " [136, 144, 168],\n", + " [136, 144, 168]],\n", + "\n", + " [[143, 146, 164],\n", + " [147, 150, 168],\n", + " [146, 149, 166],\n", + " ...,\n", + " [136, 144, 168],\n", + " [136, 144, 168],\n", + " [136, 144, 168]]], dtype=uint8), 'image': array([[[108, 98, 66],\n", + " [108, 98, 66],\n", + " [ 51, 73, 57],\n", + " ...,\n", + " [154, 153, 153],\n", + " [153, 153, 149],\n", + " [152, 152, 148]],\n", + "\n", + " [[113, 97, 65],\n", + " [113, 97, 65],\n", + " [ 62, 88, 64],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[110, 82, 54],\n", + " [121, 93, 65],\n", + " [ 74, 100, 74],\n", + " ...,\n", + " [149, 151, 157],\n", + " [145, 147, 153],\n", + " [145, 147, 153]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 69, 42],\n", + " [ 72, 73, 46],\n", + " [ 71, 71, 49],\n", + " ...,\n", + " [ 95, 88, 78],\n", + " [ 93, 89, 78],\n", + " [ 93, 89, 78]],\n", + "\n", + " [[ 62, 63, 36],\n", + " [ 61, 62, 35],\n", + " [ 58, 58, 36],\n", + " ...,\n", + " [ 94, 91, 77],\n", + " [ 94, 92, 80],\n", + " [ 97, 95, 83]],\n", + "\n", + " [[ 57, 55, 34],\n", + " [ 52, 50, 29],\n", + " [ 48, 48, 28],\n", + " ...,\n", + " [ 98, 95, 81],\n", + " [ 98, 94, 83],\n", + " [100, 96, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3518052 , -0.8849166 , 2.0911217 , -2.8662055 , -1.5458258 ,\n", + " 2.9062974 , 0.53834844, 0.00483379, -0.19096382, 0.6955367 ,\n", + " 0.71699756, 0.0343527 , 0.03103506, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00609524, -0.00457143, 0.00780952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " ...,\n", + " [ 12, 12, 10],\n", + " [ 12, 12, 10],\n", + " [ 13, 14, 11]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9],\n", + " [ 12, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [151, 155, 172],\n", + " [151, 155, 172],\n", + " ...,\n", + " [127, 137, 161],\n", + " [128, 139, 162],\n", + " [128, 139, 162]],\n", + "\n", + " [[149, 152, 170],\n", + " [151, 155, 172],\n", + " [151, 155, 172],\n", + " ...,\n", + " [128, 139, 162],\n", + " [127, 137, 161],\n", + " [127, 137, 161]],\n", + "\n", + " [[149, 152, 170],\n", + " [149, 152, 170],\n", + " [150, 153, 171],\n", + " ...,\n", + " [128, 139, 162],\n", + " [127, 137, 161],\n", + " [127, 137, 161]]], dtype=uint8), 'image': array([[[104, 104, 71],\n", + " [100, 100, 67],\n", + " [ 47, 79, 68],\n", + " ...,\n", + " [158, 152, 154],\n", + " [156, 150, 154],\n", + " [156, 150, 154]],\n", + "\n", + " [[111, 97, 62],\n", + " [114, 100, 65],\n", + " [ 61, 93, 72],\n", + " ...,\n", + " [156, 153, 150],\n", + " [154, 150, 151],\n", + " [153, 149, 150]],\n", + "\n", + " [[105, 79, 48],\n", + " [122, 96, 65],\n", + " [ 77, 105, 78],\n", + " ...,\n", + " [153, 153, 149],\n", + " [148, 147, 147],\n", + " [147, 146, 146]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 47],\n", + " [ 70, 73, 50],\n", + " [ 67, 71, 50],\n", + " ...,\n", + " [ 77, 77, 71],\n", + " [ 75, 75, 71],\n", + " [ 74, 74, 70]],\n", + "\n", + " [[ 56, 59, 34],\n", + " [ 59, 62, 37],\n", + " [ 53, 55, 37],\n", + " ...,\n", + " [ 80, 78, 68],\n", + " [ 78, 75, 70],\n", + " [ 79, 76, 71]],\n", + "\n", + " [[ 52, 50, 31],\n", + " [ 54, 52, 33],\n", + " [ 48, 47, 32],\n", + " ...,\n", + " [ 83, 80, 66],\n", + " [ 83, 79, 72],\n", + " [ 86, 82, 75]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.3262916 , -0.88400537, 2.0940309 , -2.8750207 , -1.5502566 ,\n", + " 2.9384649 , 0.5379629 , 0.01745152, -0.19117709, 0.69790804,\n", + " 0.71456426, 0.03479953, 0.03333521, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00666667, -0.00095238, 0.00533333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 14, 13],\n", + " [ 5, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 5, 10, 9],\n", + " [ 5, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " [[ 7, 10, 9],\n", + " [ 7, 10, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 12, 12, 10]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [153, 156, 173],\n", + " ...,\n", + " [134, 142, 165],\n", + " [134, 142, 165],\n", + " [134, 142, 165]],\n", + "\n", + " [[153, 156, 173],\n", + " [153, 156, 173],\n", + " [153, 156, 173],\n", + " ...,\n", + " [131, 139, 163],\n", + " [133, 140, 164],\n", + " [133, 140, 164]],\n", + "\n", + " [[153, 156, 173],\n", + " [154, 157, 175],\n", + " [154, 157, 175],\n", + " ...,\n", + " [133, 140, 164],\n", + " [133, 140, 164],\n", + " [131, 139, 163]]], dtype=uint8), 'image': array([[[106, 97, 60],\n", + " [107, 98, 61],\n", + " [ 44, 68, 56],\n", + " ...,\n", + " [152, 150, 157],\n", + " [152, 152, 156],\n", + " [152, 152, 156]],\n", + "\n", + " [[116, 95, 64],\n", + " [123, 102, 71],\n", + " [ 60, 85, 68],\n", + " ...,\n", + " [152, 151, 158],\n", + " [150, 150, 154],\n", + " [151, 151, 155]],\n", + "\n", + " [[113, 82, 55],\n", + " [127, 96, 69],\n", + " [ 78, 103, 82],\n", + " ...,\n", + " [152, 152, 156],\n", + " [145, 148, 151],\n", + " [141, 144, 147]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 66, 45],\n", + " [ 67, 71, 50],\n", + " [ 61, 71, 46],\n", + " ...,\n", + " [ 44, 73, 85],\n", + " [ 43, 71, 86],\n", + " [ 48, 76, 91]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 53, 60, 36],\n", + " [ 50, 61, 31],\n", + " ...,\n", + " [ 50, 79, 89],\n", + " [ 44, 73, 85],\n", + " [ 45, 74, 86]],\n", + "\n", + " [[ 50, 54, 31],\n", + " [ 50, 54, 31],\n", + " [ 42, 52, 25],\n", + " ...,\n", + " [ 65, 97, 106],\n", + " [ 57, 88, 102],\n", + " [ 52, 83, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.299629 , -0.8831771 , 2.0949645 , -2.8811018 , -1.5560879 ,\n", + " 2.9703782 , 0.5377479 , 0.03023179, -0.1911558 , 0.6998129 ,\n", + " 0.71258867, 0.03465674, 0.03575221, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.004 , 0. , 0.00552381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 10, 8],\n", + " [ 10, 9, 7],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 11, 10, 8],\n", + " [ 11, 10, 8],\n", + " [ 10, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[160, 160, 174],\n", + " [160, 160, 174],\n", + " [157, 159, 173],\n", + " ...,\n", + " [137, 143, 165],\n", + " [137, 143, 165],\n", + " [137, 143, 165]],\n", + "\n", + " [[159, 159, 173],\n", + " [160, 160, 174],\n", + " [159, 162, 175],\n", + " ...,\n", + " [137, 143, 165],\n", + " [136, 142, 164],\n", + " [136, 142, 164]],\n", + "\n", + " [[158, 158, 171],\n", + " [158, 158, 171],\n", + " [157, 159, 173],\n", + " ...,\n", + " [136, 142, 164],\n", + " [136, 142, 164],\n", + " [135, 141, 163]]], dtype=uint8), 'image': array([[[111, 101, 71],\n", + " [104, 94, 64],\n", + " [ 49, 69, 66],\n", + " ...,\n", + " [155, 154, 156],\n", + " [153, 152, 152],\n", + " [151, 150, 150]],\n", + "\n", + " [[119, 101, 69],\n", + " [119, 101, 69],\n", + " [ 62, 84, 68],\n", + " ...,\n", + " [152, 155, 156],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[118, 87, 58],\n", + " [126, 95, 66],\n", + " [ 78, 101, 78],\n", + " ...,\n", + " [149, 154, 159],\n", + " [144, 147, 150],\n", + " [143, 146, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 71, 49],\n", + " [ 65, 72, 50],\n", + " [ 63, 71, 42],\n", + " ...,\n", + " [ 64, 89, 94],\n", + " [ 63, 91, 97],\n", + " [ 63, 91, 97]],\n", + "\n", + " [[ 53, 60, 36],\n", + " [ 51, 58, 34],\n", + " [ 49, 56, 30],\n", + " ...,\n", + " [ 59, 87, 93],\n", + " [ 63, 87, 97],\n", + " [ 64, 88, 98]],\n", + "\n", + " [[ 53, 57, 34],\n", + " [ 52, 56, 33],\n", + " [ 42, 49, 25],\n", + " ...,\n", + " [ 55, 84, 92],\n", + " [ 57, 84, 93],\n", + " [ 57, 84, 93]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2725835 , -0.88199025, 2.093504 , -2.8825166 , -1.5627569 ,\n", + " 3.0007691 , 0.53741384, 0.0429778 , -0.19125551, 0.70110106,\n", + " 0.7112887 , 0.03353317, 0.03743315, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00647619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 12],\n", + " [ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 7, 7, 7],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 7, 7, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[155, 156, 172],\n", + " [155, 156, 172],\n", + " [155, 155, 171],\n", + " ...,\n", + " [129, 140, 163],\n", + " [131, 141, 164],\n", + " [131, 141, 164]],\n", + "\n", + " [[152, 154, 169],\n", + " [153, 155, 171],\n", + " [157, 156, 172],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [131, 141, 164]],\n", + "\n", + " [[155, 156, 172],\n", + " [153, 155, 171],\n", + " [155, 155, 171],\n", + " ...,\n", + " [132, 142, 165],\n", + " [132, 142, 165],\n", + " [131, 141, 164]]], dtype=uint8), 'image': array([[[107, 96, 71],\n", + " [107, 96, 71],\n", + " [ 51, 74, 66],\n", + " ...,\n", + " [151, 154, 155],\n", + " [153, 152, 154],\n", + " [153, 152, 154]],\n", + "\n", + " [[112, 95, 68],\n", + " [118, 101, 74],\n", + " [ 58, 83, 66],\n", + " ...,\n", + " [150, 153, 152],\n", + " [151, 150, 152],\n", + " [151, 150, 152]],\n", + "\n", + " [[118, 90, 62],\n", + " [130, 102, 74],\n", + " [ 69, 94, 73],\n", + " ...,\n", + " [150, 153, 154],\n", + " [146, 146, 150],\n", + " [147, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 70, 48],\n", + " [ 66, 73, 51],\n", + " [ 62, 69, 45],\n", + " ...,\n", + " [ 66, 94, 100],\n", + " [ 64, 92, 98],\n", + " [ 62, 90, 96]],\n", + "\n", + " [[ 55, 62, 38],\n", + " [ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " ...,\n", + " [ 68, 93, 100],\n", + " [ 64, 89, 96],\n", + " [ 64, 89, 96]],\n", + "\n", + " [[ 47, 51, 30],\n", + " [ 51, 55, 34],\n", + " [ 42, 49, 27],\n", + " ...,\n", + " [ 65, 90, 97],\n", + " [ 63, 88, 95],\n", + " [ 63, 88, 95]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.2453115 , -0.88051444, 2.0899243 , -2.8813412 , -1.5684208 ,\n", + " 3.031338 , 0.53737307, 0.05604258, -0.19126037, 0.70236003,\n", + " 0.7100228 , 0.03269348, 0.03858953, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00666667], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 14, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 14, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [155, 155, 173],\n", + " ...,\n", + " [134, 142, 165],\n", + " [134, 142, 164],\n", + " [134, 142, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [135, 143, 167],\n", + " [134, 142, 164],\n", + " [134, 142, 164]],\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [156, 156, 175],\n", + " ...,\n", + " [135, 143, 167],\n", + " [134, 142, 164],\n", + " [134, 142, 164]]], dtype=uint8), 'image': array([[[109, 98, 73],\n", + " [108, 97, 72],\n", + " [ 55, 75, 68],\n", + " ...,\n", + " [154, 153, 155],\n", + " [152, 152, 154],\n", + " [151, 151, 153]],\n", + "\n", + " [[114, 94, 72],\n", + " [121, 101, 79],\n", + " [ 62, 84, 68],\n", + " ...,\n", + " [153, 152, 154],\n", + " [151, 151, 153],\n", + " [150, 150, 152]],\n", + "\n", + " [[109, 77, 55],\n", + " [126, 94, 72],\n", + " [ 78, 101, 78],\n", + " ...,\n", + " [151, 151, 155],\n", + " [147, 147, 149],\n", + " [146, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 71, 46],\n", + " [ 69, 73, 48],\n", + " [ 67, 71, 48],\n", + " ...,\n", + " [ 63, 96, 98],\n", + " [ 62, 92, 95],\n", + " [ 61, 91, 94]],\n", + "\n", + " [[ 61, 65, 37],\n", + " [ 62, 66, 38],\n", + " [ 57, 61, 40],\n", + " ...,\n", + " [ 61, 94, 96],\n", + " [ 62, 92, 95],\n", + " [ 60, 90, 93]],\n", + "\n", + " [[ 53, 56, 31],\n", + " [ 53, 56, 31],\n", + " [ 46, 50, 29],\n", + " ...,\n", + " [ 61, 94, 96],\n", + " [ 62, 92, 95],\n", + " [ 61, 91, 94]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.219752 , -0.8787039 , 2.0854707 , -2.8789427 , -1.5713781 ,\n", + " 3.0608265 , 0.53732604, 0.06888904, -0.19135544, 0.70380074,\n", + " 0.7085868 , 0.03269861, 0.03873029, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00552381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 7, 14, 13],\n", + " [ 3, 10, 9],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 3, 10, 9],\n", + " [ 2, 9, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 5, 10, 9],\n", + " [ 4, 9, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 151, 172],\n", + " [149, 151, 172],\n", + " [148, 150, 170],\n", + " ...,\n", + " [126, 139, 162],\n", + " [127, 140, 163],\n", + " [127, 140, 163]],\n", + "\n", + " [[151, 154, 174],\n", + " [153, 155, 175],\n", + " [151, 154, 174],\n", + " ...,\n", + " [127, 140, 163],\n", + " [126, 139, 162],\n", + " [127, 140, 163]],\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [153, 155, 175],\n", + " ...,\n", + " [129, 141, 164],\n", + " [127, 140, 163],\n", + " [127, 140, 163]]], dtype=uint8), 'image': array([[[104, 99, 66],\n", + " [102, 97, 64],\n", + " [ 47, 71, 57],\n", + " ...,\n", + " [155, 154, 154],\n", + " [153, 152, 152],\n", + " [152, 151, 151]],\n", + "\n", + " [[114, 99, 71],\n", + " [118, 103, 75],\n", + " [ 57, 84, 66],\n", + " ...,\n", + " [153, 153, 155],\n", + " [152, 152, 154],\n", + " [147, 147, 149]],\n", + "\n", + " [[116, 89, 65],\n", + " [131, 104, 80],\n", + " [ 68, 95, 73],\n", + " ...,\n", + " [153, 152, 159],\n", + " [147, 147, 151],\n", + " [146, 146, 150]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 48],\n", + " [ 72, 75, 52],\n", + " [ 75, 76, 49],\n", + " ...,\n", + " [ 66, 91, 98],\n", + " [ 65, 90, 97],\n", + " [ 64, 89, 96]],\n", + "\n", + " [[ 57, 61, 36],\n", + " [ 55, 59, 34],\n", + " [ 51, 54, 29],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 90, 96],\n", + " [ 62, 90, 96]],\n", + "\n", + " [[ 55, 58, 33],\n", + " [ 51, 54, 29],\n", + " [ 48, 51, 26],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 62, 92, 95],\n", + " [ 63, 93, 96]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.194349 , -0.8765333 , 2.0798793 , -2.8753808 , -1.5737132 ,\n", + " 3.09029 , 0.5372993 , 0.08182718, -0.19148241, 0.7052792 ,\n", + " 0.7071188 , 0.03283391, 0.03855143, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 14, 13],\n", + " [ 5, 10, 9],\n", + " [ 7, 10, 9],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 7, 10, 9],\n", + " [ 6, 9, 8],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 7, 11, 8],\n", + " [ 6, 9, 7],\n", + " [ 8, 8, 8],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [155, 155, 173],\n", + " ...,\n", + " [129, 140, 163],\n", + " [131, 141, 164],\n", + " [131, 141, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [129, 140, 163],\n", + " [129, 140, 163],\n", + " [131, 141, 164]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [156, 156, 175],\n", + " ...,\n", + " [129, 140, 163],\n", + " [129, 140, 163],\n", + " [129, 140, 163]]], dtype=uint8), 'image': array([[[106, 98, 72],\n", + " [103, 95, 69],\n", + " [ 52, 74, 73],\n", + " ...,\n", + " [155, 154, 154],\n", + " [153, 152, 154],\n", + " [154, 153, 155]],\n", + "\n", + " [[115, 97, 65],\n", + " [113, 95, 63],\n", + " [ 64, 88, 74],\n", + " ...,\n", + " [153, 153, 155],\n", + " [154, 153, 157],\n", + " [151, 150, 154]],\n", + "\n", + " [[111, 84, 51],\n", + " [120, 93, 60],\n", + " [ 83, 106, 83],\n", + " ...,\n", + " [152, 152, 156],\n", + " [150, 149, 156],\n", + " [146, 145, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 63, 69, 52],\n", + " [ 69, 75, 58],\n", + " [ 65, 71, 52],\n", + " ...,\n", + " [ 71, 96, 101],\n", + " [ 78, 100, 103],\n", + " [ 75, 97, 100]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 57, 63, 44],\n", + " [ 56, 63, 41],\n", + " ...,\n", + " [ 67, 95, 101],\n", + " [ 75, 100, 105],\n", + " [ 75, 100, 105]],\n", + "\n", + " [[ 47, 50, 34],\n", + " [ 50, 53, 37],\n", + " [ 45, 52, 30],\n", + " ...,\n", + " [ 64, 91, 102],\n", + " [ 69, 96, 105],\n", + " [ 71, 98, 107]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1694727 , -0.87412244, 2.0735097 , -2.8713624 , -1.5762395 ,\n", + " 3.1185782 , 0.53730077, 0.09447004, -0.19158448, 0.7065261 ,\n", + " 0.7058712 , 0.03291051, 0.03851758, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.004], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 10, 10],\n", + " [ 8, 8, 8],\n", + " [ 7, 8, 5],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [158, 157, 173],\n", + " ...,\n", + " [122, 136, 161],\n", + " [123, 137, 162],\n", + " [123, 137, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [159, 158, 174],\n", + " ...,\n", + " [123, 137, 162],\n", + " [123, 137, 162],\n", + " [123, 137, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [158, 157, 173],\n", + " ...,\n", + " [123, 137, 162],\n", + " [124, 138, 163],\n", + " [124, 138, 163]]], dtype=uint8), 'image': array([[[106, 103, 71],\n", + " [104, 101, 69],\n", + " [ 49, 72, 66],\n", + " ...,\n", + " [155, 156, 150],\n", + " [155, 156, 147],\n", + " [153, 154, 145]],\n", + "\n", + " [[117, 102, 74],\n", + " [119, 104, 76],\n", + " [ 59, 85, 70],\n", + " ...,\n", + " [151, 155, 148],\n", + " [153, 155, 146],\n", + " [152, 154, 145]],\n", + "\n", + " [[110, 82, 54],\n", + " [128, 100, 72],\n", + " [ 74, 99, 78],\n", + " ...,\n", + " [149, 153, 148],\n", + " [145, 149, 142],\n", + " [142, 146, 139]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 67, 49],\n", + " [ 67, 70, 52],\n", + " [ 66, 69, 51],\n", + " ...,\n", + " [ 61, 91, 96],\n", + " [ 60, 93, 97],\n", + " [ 58, 91, 95]],\n", + "\n", + " [[ 57, 61, 40],\n", + " [ 58, 62, 41],\n", + " [ 49, 55, 36],\n", + " ...,\n", + " [ 59, 92, 96],\n", + " [ 60, 93, 97],\n", + " [ 59, 92, 96]],\n", + "\n", + " [[ 52, 56, 35],\n", + " [ 50, 54, 33],\n", + " [ 44, 50, 31],\n", + " ...,\n", + " [ 59, 88, 96],\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1447768 , -0.8713878 , 2.066051 , -2.8665173 , -1.5785502 ,\n", + " 3.1462178 , 0.53738934, 0.10712075, -0.19166769, 0.70760185,\n", + " 0.7047916 , 0.03304661, 0.03842265, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 14, 15, 12],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 12, 12, 7],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[157, 158, 176],\n", + " [157, 158, 176],\n", + " [156, 156, 175],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[156, 156, 175],\n", + " [156, 156, 175],\n", + " [157, 158, 176],\n", + " ...,\n", + " [113, 129, 156],\n", + " [112, 128, 155],\n", + " [113, 129, 156]],\n", + "\n", + " [[157, 158, 176],\n", + " [156, 156, 175],\n", + " [157, 158, 176],\n", + " ...,\n", + " [115, 131, 158],\n", + " [113, 129, 156],\n", + " [114, 130, 157]]], dtype=uint8), 'image': array([[[112, 100, 73],\n", + " [112, 100, 73],\n", + " [ 53, 71, 69],\n", + " ...,\n", + " [150, 154, 149],\n", + " [149, 153, 146],\n", + " [150, 154, 147]],\n", + "\n", + " [[120, 101, 72],\n", + " [117, 98, 69],\n", + " [ 67, 88, 75],\n", + " ...,\n", + " [149, 153, 148],\n", + " [148, 152, 147],\n", + " [148, 152, 147]],\n", + "\n", + " [[110, 82, 54],\n", + " [121, 93, 65],\n", + " [ 77, 99, 81],\n", + " ...,\n", + " [149, 152, 151],\n", + " [144, 147, 146],\n", + " [143, 146, 145]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 72, 48],\n", + " [ 64, 76, 52],\n", + " [ 63, 73, 50],\n", + " ...,\n", + " [ 60, 90, 95],\n", + " [ 58, 83, 90],\n", + " [ 58, 83, 90]],\n", + "\n", + " [[ 53, 63, 38],\n", + " [ 51, 61, 36],\n", + " [ 52, 59, 37],\n", + " ...,\n", + " [ 60, 90, 95],\n", + " [ 58, 83, 90],\n", + " [ 56, 81, 88]],\n", + "\n", + " [[ 48, 52, 29],\n", + " [ 50, 54, 31],\n", + " [ 46, 53, 31],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 87, 92],\n", + " [ 55, 80, 85]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.1198535 , -0.8682216 , 2.057539 , -2.860892 , -1.5811008 ,\n", + " 3.1736052 , 0.53747594, 0.11990292, -0.19179024, 0.70851207,\n", + " 0.7038783 , 0.03306015, 0.03838015, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00380952], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 154, 174],\n", + " [149, 154, 174],\n", + " [150, 153, 171],\n", + " ...,\n", + " [112, 132, 161],\n", + " [112, 132, 161],\n", + " [112, 132, 161]],\n", + "\n", + " [[151, 155, 175],\n", + " [151, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [110, 130, 159],\n", + " [109, 129, 157],\n", + " [111, 131, 160]],\n", + "\n", + " [[152, 156, 176],\n", + " [152, 156, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [112, 132, 161],\n", + " [113, 133, 162],\n", + " [115, 135, 163]]], dtype=uint8), 'image': array([[[104, 100, 62],\n", + " [ 97, 93, 55],\n", + " [ 39, 68, 67],\n", + " ...,\n", + " [152, 155, 154],\n", + " [151, 154, 153],\n", + " [150, 153, 152]],\n", + "\n", + " [[116, 101, 62],\n", + " [111, 96, 57],\n", + " [ 54, 84, 74],\n", + " ...,\n", + " [152, 155, 156],\n", + " [149, 152, 153],\n", + " [147, 150, 151]],\n", + "\n", + " [[118, 87, 60],\n", + " [122, 91, 64],\n", + " [ 73, 102, 84],\n", + " ...,\n", + " [150, 152, 160],\n", + " [144, 146, 154],\n", + " [144, 146, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 65, 69, 46],\n", + " [ 67, 71, 48],\n", + " [ 64, 68, 47],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " [ 56, 62, 43],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 98]],\n", + "\n", + " [[ 43, 52, 32],\n", + " [ 43, 52, 32],\n", + " [ 42, 50, 35],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 61, 93, 100],\n", + " [ 58, 90, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0955417 , -0.8648394 , 2.0481286 , -2.8546073 , -1.5833639 ,\n", + " 3.2002604 , 0.53765064, 0.1325151 , -0.191861 , 0.7093723 ,\n", + " 0.70301116, 0.03317269, 0.03828627, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00514286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00011429], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 8, 16, 12],\n", + " [ 4, 12, 9],\n", + " [ 8, 12, 9],\n", + " ...,\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " [[ 5, 11, 8],\n", + " [ 5, 11, 8],\n", + " [ 6, 9, 7],\n", + " ...,\n", + " [ 9, 11, 6],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " [[ 7, 11, 8],\n", + " [ 7, 11, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 9, 11, 6],\n", + " [ 10, 12, 7],\n", + " [ 10, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[155, 158, 176],\n", + " [155, 158, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [112, 127, 156],\n", + " [112, 127, 156],\n", + " [113, 128, 157]],\n", + "\n", + " [[154, 157, 175],\n", + " [155, 158, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [114, 129, 159],\n", + " [114, 129, 159],\n", + " [113, 128, 157]],\n", + "\n", + " [[155, 158, 176],\n", + " [156, 159, 177],\n", + " [155, 158, 176],\n", + " ...,\n", + " [117, 133, 162],\n", + " [119, 134, 163],\n", + " [117, 133, 162]]], dtype=uint8), 'image': array([[[108, 97, 74],\n", + " [108, 97, 74],\n", + " [ 43, 66, 60],\n", + " ...,\n", + " [151, 151, 151],\n", + " [156, 153, 150],\n", + " [157, 154, 151]],\n", + "\n", + " [[111, 91, 69],\n", + " [121, 101, 79],\n", + " [ 65, 89, 75],\n", + " ...,\n", + " [152, 151, 155],\n", + " [156, 152, 153],\n", + " [155, 151, 152]],\n", + "\n", + " [[106, 77, 56],\n", + " [125, 96, 75],\n", + " [ 81, 106, 87],\n", + " ...,\n", + " [152, 151, 160],\n", + " [151, 149, 156],\n", + " [147, 145, 152]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 63, 50],\n", + " [ 67, 69, 56],\n", + " [ 63, 69, 50],\n", + " ...,\n", + " [ 66, 96, 99],\n", + " [ 66, 96, 99],\n", + " [ 65, 95, 98]],\n", + "\n", + " [[ 57, 60, 44],\n", + " [ 59, 62, 46],\n", + " [ 59, 62, 44],\n", + " ...,\n", + " [ 65, 95, 98],\n", + " [ 61, 91, 94],\n", + " [ 62, 92, 95]],\n", + "\n", + " [[ 51, 54, 38],\n", + " [ 47, 50, 34],\n", + " [ 46, 49, 31],\n", + " ...,\n", + " [ 65, 95, 98],\n", + " [ 60, 90, 95],\n", + " [ 62, 92, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0713017 , -0.8611215 , 2.037857 , -2.8476746 , -1.585627 ,\n", + " 3.2267466 , 0.5377978 , 0.14518696, -0.19194975, 0.7101977 ,\n", + " 0.7021796 , 0.03325085, 0.03817584, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00514286], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00022857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 12],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [154, 157, 175],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[151, 154, 174],\n", + " [153, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [120, 136, 163]],\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [154, 157, 175],\n", + " ...,\n", + " [119, 135, 160],\n", + " [119, 135, 162],\n", + " [119, 135, 162]]], dtype=uint8), 'image': array([[[104, 103, 75],\n", + " [ 94, 93, 65],\n", + " [ 41, 71, 63],\n", + " ...,\n", + " [148, 153, 158],\n", + " [147, 152, 157],\n", + " [148, 153, 158]],\n", + "\n", + " [[116, 106, 78],\n", + " [106, 96, 68],\n", + " [ 58, 91, 74],\n", + " ...,\n", + " [146, 152, 154],\n", + " [147, 152, 157],\n", + " [146, 151, 156]],\n", + "\n", + " [[114, 90, 67],\n", + " [114, 90, 67],\n", + " [ 69, 101, 80],\n", + " ...,\n", + " [147, 152, 157],\n", + " [143, 148, 153],\n", + " [141, 146, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 71, 46],\n", + " [ 67, 77, 52],\n", + " [ 66, 73, 51],\n", + " ...,\n", + " [ 67, 96, 104],\n", + " [ 63, 95, 102],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 53, 63, 38],\n", + " [ 55, 65, 40],\n", + " [ 53, 60, 36],\n", + " ...,\n", + " [ 66, 96, 101],\n", + " [ 64, 96, 103],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 46, 56, 31],\n", + " [ 47, 57, 32],\n", + " [ 43, 50, 26],\n", + " ...,\n", + " [ 66, 94, 100],\n", + " [ 63, 92, 102],\n", + " [ 65, 94, 104]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.046966 , -0.8571275 , 2.026507 , -2.8399389 , -1.5878781 ,\n", + " 3.254046 , 0.5379952 , 0.15803353, -0.19198048, 0.71127206,\n", + " 0.7010918 , 0.03336547, 0.03806647, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, 0.0000000e+00, -5.7142857e-05], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 15, 16, 14],\n", + " [ 8, 9, 7],\n", + " [ 7, 8, 5],\n", + " ...,\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 9, 8, 4],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [153, 155, 175],\n", + " ...,\n", + " [121, 135, 160],\n", + " [121, 134, 162],\n", + " [121, 134, 162]],\n", + "\n", + " [[154, 156, 176],\n", + " [154, 156, 176],\n", + " [154, 156, 176],\n", + " ...,\n", + " [116, 130, 155],\n", + " [115, 128, 156],\n", + " [117, 131, 158]],\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [154, 156, 176],\n", + " ...,\n", + " [114, 128, 153],\n", + " [114, 127, 155],\n", + " [116, 130, 157]]], dtype=uint8), 'image': array([[[104, 102, 83],\n", + " [ 95, 93, 74],\n", + " [ 44, 68, 78],\n", + " ...,\n", + " [151, 154, 155],\n", + " [150, 153, 154],\n", + " [150, 153, 154]],\n", + "\n", + " [[110, 96, 77],\n", + " [114, 100, 81],\n", + " [ 54, 80, 80],\n", + " ...,\n", + " [147, 153, 155],\n", + " [147, 153, 155],\n", + " [148, 154, 156]],\n", + "\n", + " [[109, 81, 64],\n", + " [130, 102, 85],\n", + " [ 73, 100, 91],\n", + " ...,\n", + " [146, 151, 158],\n", + " [145, 151, 153],\n", + " [140, 146, 148]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 68, 45],\n", + " [ 66, 70, 47],\n", + " [ 64, 68, 47],\n", + " ...,\n", + " [ 61, 94, 98],\n", + " [ 63, 96, 100],\n", + " [ 63, 96, 100]],\n", + "\n", + " [[ 57, 61, 36],\n", + " [ 58, 62, 37],\n", + " [ 55, 62, 36],\n", + " ...,\n", + " [ 64, 97, 101],\n", + " [ 64, 97, 101],\n", + " [ 61, 94, 98]],\n", + "\n", + " [[ 52, 57, 29],\n", + " [ 50, 55, 27],\n", + " [ 46, 51, 23],\n", + " ...,\n", + " [ 65, 98, 102],\n", + " [ 64, 97, 101],\n", + " [ 61, 94, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.02317 , -0.8530019 , 2.0143075 , -2.8314588 , -1.5901531 ,\n", + " 3.280961 , 0.5382555 , 0.17071289, -0.1919237 , 0.7123984 ,\n", + " 0.69994664, 0.03346113, 0.0379937 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.004], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 13, 10, 6],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 12, 12, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[153, 155, 175],\n", + " [153, 155, 175],\n", + " [153, 156, 173],\n", + " ...,\n", + " [120, 136, 163],\n", + " [121, 137, 164],\n", + " [121, 137, 164]],\n", + "\n", + " [[150, 153, 173],\n", + " [151, 154, 174],\n", + " [151, 155, 172],\n", + " ...,\n", + " [121, 137, 164],\n", + " [122, 138, 165],\n", + " [123, 139, 166]],\n", + "\n", + " [[150, 153, 173],\n", + " [151, 154, 174],\n", + " [153, 156, 173],\n", + " ...,\n", + " [123, 139, 166],\n", + " [124, 140, 167],\n", + " [124, 140, 167]]], dtype=uint8), 'image': array([[[101, 98, 84],\n", + " [106, 103, 89],\n", + " [ 37, 68, 68],\n", + " ...,\n", + " [148, 156, 153],\n", + " [147, 155, 152],\n", + " [147, 155, 152]],\n", + "\n", + " [[111, 98, 85],\n", + " [118, 105, 92],\n", + " [ 46, 80, 75],\n", + " ...,\n", + " [148, 156, 155],\n", + " [147, 153, 151],\n", + " [148, 154, 152]],\n", + "\n", + " [[105, 80, 71],\n", + " [122, 97, 88],\n", + " [ 61, 93, 84],\n", + " ...,\n", + " [148, 153, 158],\n", + " [143, 149, 151],\n", + " [141, 147, 149]],\n", + "\n", + " ...,\n", + "\n", + " [[ 57, 68, 47],\n", + " [ 63, 74, 53],\n", + " [ 59, 75, 52],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 59, 87, 93],\n", + " [ 56, 84, 90]],\n", + "\n", + " [[ 50, 62, 38],\n", + " [ 49, 61, 37],\n", + " [ 48, 60, 36],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 59, 87, 93],\n", + " [ 57, 85, 91]],\n", + "\n", + " [[ 47, 57, 34],\n", + " [ 47, 57, 34],\n", + " [ 41, 53, 29],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 56, 86, 89],\n", + " [ 55, 85, 88]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9994097 , -0.84833604, 2.0012221 , -2.822284 , -1.5922841 ,\n", + " 3.3077435 , 0.5384106 , 0.18353929, -0.19198695, 0.7134904 ,\n", + " 0.6988444 , 0.03347312, 0.0377824 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 13, 13],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 9, 9],\n", + " [ 9, 9, 9],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 7, 8, 5],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " [[ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 8, 9, 7],\n", + " [ 9, 10, 8],\n", + " [ 10, 11, 9]],\n", + "\n", + " ...,\n", + "\n", + " [[152, 156, 178],\n", + " [152, 156, 178],\n", + " [154, 156, 176],\n", + " ...,\n", + " [116, 132, 159],\n", + " [117, 133, 160],\n", + " [117, 133, 160]],\n", + "\n", + " [[152, 156, 178],\n", + " [153, 157, 179],\n", + " [155, 157, 177],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]],\n", + "\n", + " [[152, 156, 178],\n", + " [153, 157, 179],\n", + " [155, 157, 177],\n", + " ...,\n", + " [119, 135, 162],\n", + " [119, 135, 162],\n", + " [119, 135, 162]]], dtype=uint8), 'image': array([[[102, 95, 74],\n", + " [107, 100, 79],\n", + " [ 37, 68, 66],\n", + " ...,\n", + " [147, 152, 159],\n", + " [152, 151, 160],\n", + " [151, 150, 159]],\n", + "\n", + " [[109, 93, 72],\n", + " [121, 105, 84],\n", + " [ 52, 84, 75],\n", + " ...,\n", + " [147, 152, 159],\n", + " [150, 152, 160],\n", + " [145, 147, 155]],\n", + "\n", + " [[110, 82, 63],\n", + " [124, 96, 77],\n", + " [ 66, 99, 84],\n", + " ...,\n", + " [147, 152, 161],\n", + " [147, 149, 157],\n", + " [144, 146, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 57, 65, 50],\n", + " [ 62, 70, 55],\n", + " [ 64, 69, 57],\n", + " ...,\n", + " [ 86, 118, 125],\n", + " [ 80, 114, 120],\n", + " [ 67, 101, 107]],\n", + "\n", + " [[ 56, 62, 45],\n", + " [ 57, 63, 46],\n", + " [ 55, 57, 44],\n", + " ...,\n", + " [ 67, 99, 108],\n", + " [ 77, 111, 119],\n", + " [ 73, 107, 115]],\n", + "\n", + " [[ 52, 55, 37],\n", + " [ 51, 54, 36],\n", + " [ 45, 47, 34],\n", + " ...,\n", + " [ 60, 92, 101],\n", + " [ 59, 93, 101],\n", + " [ 70, 104, 112]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9757934 , -0.84310704, 1.9867959 , -2.8126876 , -1.5945233 ,\n", + " 3.334057 , 0.53880805, 0.19645281, -0.19202068, 0.71446204,\n", + " 0.69784665, 0.03360009, 0.03775069, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02, 0. , -0. ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 9, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[151, 155, 175],\n", + " [151, 155, 175],\n", + " [149, 154, 174],\n", + " ...,\n", + " [125, 138, 161],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[148, 153, 173],\n", + " [148, 153, 173],\n", + " [148, 153, 173],\n", + " ...,\n", + " [126, 139, 162],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[148, 153, 173],\n", + " [147, 152, 172],\n", + " [149, 154, 174],\n", + " ...,\n", + " [127, 140, 163],\n", + " [127, 140, 163],\n", + " [127, 140, 163]]], dtype=uint8), 'image': array([[[107, 95, 75],\n", + " [108, 96, 76],\n", + " [ 49, 68, 72],\n", + " ...,\n", + " [155, 157, 163],\n", + " [153, 152, 159],\n", + " [152, 151, 158]],\n", + "\n", + " [[115, 95, 73],\n", + " [122, 102, 80],\n", + " [ 62, 85, 79],\n", + " ...,\n", + " [152, 154, 162],\n", + " [151, 153, 159],\n", + " [149, 151, 157]],\n", + "\n", + " [[118, 85, 68],\n", + " [128, 95, 78],\n", + " [ 78, 102, 90],\n", + " ...,\n", + " [148, 153, 162],\n", + " [148, 150, 158],\n", + " [145, 147, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 65, 51],\n", + " [ 67, 72, 58],\n", + " [ 64, 69, 55],\n", + " ...,\n", + " [ 83, 111, 126],\n", + " [ 89, 121, 132],\n", + " [ 82, 114, 125]],\n", + "\n", + " [[ 55, 61, 42],\n", + " [ 53, 59, 40],\n", + " [ 54, 57, 41],\n", + " ...,\n", + " [ 62, 90, 105],\n", + " [ 73, 102, 112],\n", + " [ 82, 111, 121]],\n", + "\n", + " [[ 48, 50, 32],\n", + " [ 52, 54, 36],\n", + " [ 47, 51, 30],\n", + " ...,\n", + " [ 58, 90, 101],\n", + " [ 60, 89, 99],\n", + " [ 58, 87, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9526203 , -0.8375662 , 1.9717474 , -2.8025048 , -1.5965704 ,\n", + " 3.3599634 , 0.539095 , 0.20931895, -0.1921031 , 0.7154446 ,\n", + " 0.6968444 , 0.03366781, 0.03759455, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.00495238], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 2.0000000e-02, 0.0000000e+00, -5.7142857e-05], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 13, 14, 10],\n", + " [ 10, 12, 7],\n", + " [ 8, 10, 5],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 7],\n", + " [ 11, 10, 8]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 8, 7, 3],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[145, 151, 173],\n", + " [145, 151, 173],\n", + " [144, 150, 172],\n", + " ...,\n", + " [125, 140, 163],\n", + " [125, 140, 163],\n", + " [125, 140, 163]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [125, 140, 163],\n", + " [124, 139, 162],\n", + " [123, 138, 161]],\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [144, 150, 172],\n", + " ...,\n", + " [121, 136, 158],\n", + " [121, 136, 158],\n", + " [121, 136, 158]]], dtype=uint8), 'image': array([[[103, 96, 77],\n", + " [102, 95, 76],\n", + " [ 50, 74, 74],\n", + " ...,\n", + " [150, 155, 160],\n", + " [150, 155, 160],\n", + " [150, 155, 160]],\n", + "\n", + " [[115, 97, 77],\n", + " [112, 94, 74],\n", + " [ 57, 85, 72],\n", + " ...,\n", + " [152, 154, 162],\n", + " [147, 152, 157],\n", + " [147, 152, 157]],\n", + "\n", + " [[120, 91, 68],\n", + " [123, 94, 71],\n", + " [ 73, 102, 82],\n", + " ...,\n", + " [148, 152, 164],\n", + " [143, 148, 155],\n", + " [142, 147, 154]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 68, 51],\n", + " [ 66, 72, 55],\n", + " [ 62, 67, 53],\n", + " ...,\n", + " [ 87, 121, 129],\n", + " [ 74, 108, 116],\n", + " [ 66, 100, 108]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 57, 64, 40],\n", + " [ 53, 58, 44],\n", + " ...,\n", + " [ 85, 119, 127],\n", + " [ 87, 121, 131],\n", + " [ 78, 112, 122]],\n", + "\n", + " [[ 55, 62, 40],\n", + " [ 48, 55, 33],\n", + " [ 45, 51, 34],\n", + " ...,\n", + " [ 64, 98, 104],\n", + " [ 75, 107, 116],\n", + " [ 82, 114, 123]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9290407 , -0.83140117, 1.9549513 , -2.7911823 , -1.5989178 ,\n", + " 3.386706 , 0.53953487, 0.22216113, -0.19215716, 0.7165386 ,\n", + " 0.6957205 , 0.0336923 , 0.03755271, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.0047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.02 , 0. , -0.00057143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 14, 11],\n", + " [ 8, 12, 9],\n", + " [ 8, 10, 5],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " [[ 8, 12, 9],\n", + " [ 8, 12, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 10, 9, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[152, 156, 176],\n", + " [152, 156, 176],\n", + " [151, 155, 175],\n", + " ...,\n", + " [126, 139, 162],\n", + " [126, 139, 162],\n", + " [126, 139, 162]],\n", + "\n", + " [[149, 154, 174],\n", + " [149, 154, 174],\n", + " [151, 155, 175],\n", + " ...,\n", + " [123, 135, 158],\n", + " [123, 135, 158],\n", + " [122, 134, 157]],\n", + "\n", + " [[148, 153, 173],\n", + " [148, 153, 173],\n", + " [149, 154, 174],\n", + " ...,\n", + " [120, 133, 156],\n", + " [119, 132, 155],\n", + " [120, 133, 156]]], dtype=uint8), 'image': array([[[107, 95, 77],\n", + " [108, 96, 78],\n", + " [ 49, 73, 73],\n", + " ...,\n", + " [153, 156, 157],\n", + " [152, 155, 156],\n", + " [150, 153, 154]],\n", + "\n", + " [[115, 94, 77],\n", + " [120, 99, 82],\n", + " [ 57, 84, 77],\n", + " ...,\n", + " [150, 158, 159],\n", + " [150, 156, 158],\n", + " [147, 153, 155]],\n", + "\n", + " [[112, 82, 64],\n", + " [125, 95, 77],\n", + " [ 75, 104, 86],\n", + " ...,\n", + " [147, 154, 158],\n", + " [142, 149, 153],\n", + " [140, 147, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 72, 60],\n", + " [ 67, 77, 65],\n", + " [ 62, 71, 62],\n", + " ...,\n", + " [ 64, 98, 108],\n", + " [ 66, 95, 105],\n", + " [ 66, 95, 105]],\n", + "\n", + " [[ 56, 64, 49],\n", + " [ 55, 63, 48],\n", + " [ 54, 61, 53],\n", + " ...,\n", + " [ 72, 106, 114],\n", + " [ 67, 99, 106],\n", + " [ 66, 98, 105]],\n", + "\n", + " [[ 50, 56, 39],\n", + " [ 48, 54, 37],\n", + " [ 39, 47, 34],\n", + " ...,\n", + " [ 89, 121, 128],\n", + " [ 87, 119, 126],\n", + " [ 74, 106, 113]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9072087 , -0.82534486, 1.9385734 , -2.780183 , -1.6008447 ,\n", + " 3.4115925 , 0.5398998 , 0.23507659, -0.19219233, 0.7176641 ,\n", + " 0.6945623 , 0.03376771, 0.03743221, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00361905, 0. , 0.00742857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.01845714, -0.00131429, -0.01542857], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 12, 12, 10],\n", + " [ 10, 11, 9],\n", + " [ 10, 12, 7],\n", + " ...,\n", + " [ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 10, 11, 9],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 5, 5, 3],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " [[ 10, 12, 7],\n", + " [ 10, 12, 7],\n", + " [ 9, 11, 6],\n", + " ...,\n", + " [ 6, 7, 4],\n", + " [ 7, 8, 5],\n", + " [ 8, 9, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[144, 150, 172],\n", + " [144, 150, 172],\n", + " [143, 149, 171],\n", + " ...,\n", + " [114, 127, 155],\n", + " [114, 127, 155],\n", + " [114, 127, 155]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [143, 149, 171],\n", + " ...,\n", + " [117, 131, 158],\n", + " [117, 131, 158],\n", + " [117, 131, 158]],\n", + "\n", + " [[143, 149, 171],\n", + " [143, 149, 171],\n", + " [140, 147, 168],\n", + " ...,\n", + " [121, 134, 162],\n", + " [121, 134, 162],\n", + " [121, 134, 162]]], dtype=uint8), 'image': array([[[109, 101, 77],\n", + " [110, 102, 78],\n", + " [ 47, 70, 66],\n", + " ...,\n", + " [150, 157, 165],\n", + " [149, 156, 162],\n", + " [148, 155, 161]],\n", + "\n", + " [[117, 101, 80],\n", + " [114, 98, 77],\n", + " [ 61, 86, 77],\n", + " ...,\n", + " [146, 156, 163],\n", + " [146, 156, 161],\n", + " [146, 156, 161]],\n", + "\n", + " [[108, 83, 65],\n", + " [118, 93, 75],\n", + " [ 77, 105, 90],\n", + " ...,\n", + " [146, 155, 165],\n", + " [142, 152, 157],\n", + " [143, 153, 158]],\n", + "\n", + " ...,\n", + "\n", + " [[ 55, 70, 43],\n", + " [ 59, 74, 47],\n", + " [ 58, 73, 46],\n", + " ...,\n", + " [ 61, 91, 96],\n", + " [ 63, 95, 102],\n", + " [ 60, 92, 99]],\n", + "\n", + " [[ 50, 62, 38],\n", + " [ 52, 64, 40],\n", + " [ 51, 62, 32],\n", + " ...,\n", + " [ 58, 91, 93],\n", + " [ 59, 93, 101],\n", + " [ 58, 92, 100]],\n", + "\n", + " [[ 48, 52, 31],\n", + " [ 50, 54, 33],\n", + " [ 45, 52, 26],\n", + " ...,\n", + " [ 67, 100, 102],\n", + " [ 60, 94, 102],\n", + " [ 60, 94, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8851256 , -0.8193839 , 1.9207363 , -2.767865 , -1.60294 ,\n", + " 3.4364538 , 0.5404305 , 0.24793711, -0.1918753 , 0.71865124,\n", + " 0.69353217, 0.03392792, 0.03744877, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.02457143], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([ 0.0012 , -0.00131429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " [[ 10, 11, 9],\n", + " [ 9, 10, 8],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 5, 5, 3],\n", + " [ 5, 5, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[149, 153, 175],\n", + " [149, 153, 175],\n", + " [151, 154, 174],\n", + " ...,\n", + " [119, 134, 159],\n", + " [123, 135, 160],\n", + " [123, 135, 160]],\n", + "\n", + " [[145, 149, 171],\n", + " [145, 149, 171],\n", + " [147, 149, 169],\n", + " ...,\n", + " [121, 135, 160],\n", + " [123, 135, 160],\n", + " [123, 135, 160]],\n", + "\n", + " [[146, 150, 172],\n", + " [145, 149, 171],\n", + " [147, 149, 169],\n", + " ...,\n", + " [119, 134, 159],\n", + " [123, 135, 160],\n", + " [123, 135, 160]]], dtype=uint8), 'image': array([[[110, 103, 82],\n", + " [104, 97, 76],\n", + " [ 46, 72, 70],\n", + " ...,\n", + " [149, 156, 162],\n", + " [149, 156, 162],\n", + " [150, 157, 163]],\n", + "\n", + " [[117, 102, 76],\n", + " [114, 99, 73],\n", + " [ 64, 92, 79],\n", + " ...,\n", + " [147, 154, 160],\n", + " [148, 155, 161],\n", + " [147, 154, 160]],\n", + "\n", + " [[114, 85, 60],\n", + " [124, 95, 70],\n", + " [ 75, 102, 84],\n", + " ...,\n", + " [145, 152, 158],\n", + " [143, 148, 157],\n", + " [143, 148, 157]],\n", + "\n", + " ...,\n", + "\n", + " [[ 61, 71, 48],\n", + " [ 63, 73, 50],\n", + " [ 61, 69, 54],\n", + " ...,\n", + " [ 63, 93, 98],\n", + " [ 66, 96, 101],\n", + " [ 65, 95, 100]],\n", + "\n", + " [[ 55, 65, 38],\n", + " [ 54, 64, 37],\n", + " [ 54, 64, 41],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 98],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 47, 57, 32],\n", + " [ 47, 57, 32],\n", + " [ 43, 53, 28],\n", + " ...,\n", + " [ 65, 94, 104],\n", + " [ 62, 91, 101],\n", + " [ 61, 90, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8629944 , -0.83263665, 1.9082379 , -2.740589 , -1.6067833 ,\n", + " 3.4631937 , 0.53934747, 0.260093 , -0.18352933, 0.72030914,\n", + " 0.69181347, 0.03324209, 0.03799796, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00190476, 0.00457143, 0.03009524], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00245714, -0.00251429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 9, 9, 9],\n", + " [ 8, 8, 8],\n", + " [ 8, 9, 7],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[ 10, 10, 10],\n", + " [ 9, 9, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[ 8, 11, 10],\n", + " [ 7, 10, 9],\n", + " [ 9, 10, 8],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 169, 180],\n", + " [168, 169, 180],\n", + " [167, 168, 179],\n", + " ...,\n", + " [120, 136, 163],\n", + " [120, 137, 161],\n", + " [120, 137, 161]],\n", + "\n", + " [[166, 167, 178],\n", + " [166, 167, 178],\n", + " [167, 168, 179],\n", + " ...,\n", + " [121, 137, 164],\n", + " [123, 140, 165],\n", + " [122, 139, 163]],\n", + "\n", + " [[165, 165, 177],\n", + " [165, 165, 177],\n", + " [165, 165, 177],\n", + " ...,\n", + " [123, 139, 166],\n", + " [124, 141, 166],\n", + " [126, 142, 167]]], dtype=uint8), 'image': array([[[104, 100, 75],\n", + " [104, 100, 75],\n", + " [ 50, 75, 70],\n", + " ...,\n", + " [151, 156, 163],\n", + " [153, 158, 165],\n", + " [151, 156, 163]],\n", + "\n", + " [[114, 101, 77],\n", + " [110, 97, 73],\n", + " [ 61, 89, 76],\n", + " ...,\n", + " [153, 160, 166],\n", + " [154, 159, 166],\n", + " [148, 153, 160]],\n", + "\n", + " [[119, 92, 70],\n", + " [123, 96, 74],\n", + " [ 74, 106, 85],\n", + " ...,\n", + " [146, 158, 162],\n", + " [145, 152, 158],\n", + " [142, 149, 155]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 55],\n", + " [ 71, 74, 58],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 61, 89, 104],\n", + " [ 63, 91, 106],\n", + " [ 65, 93, 108]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 50, 56, 39],\n", + " ...,\n", + " [ 63, 90, 103],\n", + " [ 63, 90, 103],\n", + " [ 63, 90, 103]],\n", + "\n", + " [[ 50, 56, 37],\n", + " [ 46, 52, 33],\n", + " [ 43, 49, 32],\n", + " ...,\n", + " [ 61, 88, 101],\n", + " [ 62, 89, 102],\n", + " [ 61, 88, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8570101 , -0.86013824, 1.9111109 , -2.715855 , -1.6078013 ,\n", + " 3.482787 , 0.5375394 , 0.26292184, -0.17109968, 0.72502565,\n", + " 0.68685734, 0.0334212 , 0.03804962, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.00266667, 0.00457143, 0.02761905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00937143, -0.00274286, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 99, 125, 148],\n", + " [ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " [ 98, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[100, 124, 147],\n", + " [100, 124, 147],\n", + " [100, 124, 147],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[163, 165, 178],\n", + " [163, 165, 178],\n", + " [163, 165, 178],\n", + " ...,\n", + " [124, 138, 163],\n", + " [125, 139, 166],\n", + " [125, 139, 166]],\n", + "\n", + " [[163, 165, 178],\n", + " [164, 166, 180],\n", + " [165, 167, 181],\n", + " ...,\n", + " [125, 140, 165],\n", + " [123, 137, 164],\n", + " [123, 137, 164]],\n", + "\n", + " [[163, 165, 178],\n", + " [163, 165, 178],\n", + " [164, 166, 180],\n", + " ...,\n", + " [125, 140, 165],\n", + " [125, 139, 166],\n", + " [125, 139, 166]]], dtype=uint8), 'image': array([[[107, 103, 94],\n", + " [102, 98, 89],\n", + " [ 39, 69, 74],\n", + " ...,\n", + " [142, 149, 155],\n", + " [141, 148, 154],\n", + " [141, 148, 154]],\n", + "\n", + " [[120, 105, 95],\n", + " [109, 94, 84],\n", + " [ 42, 76, 71],\n", + " ...,\n", + " [142, 149, 153],\n", + " [139, 146, 150],\n", + " [140, 147, 151]],\n", + "\n", + " [[116, 92, 78],\n", + " [120, 96, 82],\n", + " [ 57, 92, 82],\n", + " ...,\n", + " [144, 151, 155],\n", + " [141, 146, 151],\n", + " [141, 146, 151]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 70, 49],\n", + " [ 71, 75, 54],\n", + " [ 67, 71, 50],\n", + " ...,\n", + " [ 62, 91, 99],\n", + " [ 61, 90, 100],\n", + " [ 62, 91, 101]],\n", + "\n", + " [[ 58, 62, 41],\n", + " [ 57, 61, 40],\n", + " [ 53, 59, 40],\n", + " ...,\n", + " [ 62, 96, 104],\n", + " [ 58, 90, 101],\n", + " [ 57, 89, 100]],\n", + "\n", + " [[ 52, 55, 37],\n", + " [ 50, 53, 35],\n", + " [ 45, 51, 34],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 57, 91, 101],\n", + " [ 59, 93, 103]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8583746 , -0.8919328 , 1.9205203 , -2.6912768 , -1.6058615 ,\n", + " 3.5005364 , 0.5345636 , 0.26166272, -0.15828562, 0.73158187,\n", + " 0.67996943, 0.03315829, 0.03647007, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0032381 , 0.008 , 0.02761905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.0132 , -0.00371429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[106, 130, 153],\n", + " [106, 130, 153],\n", + " [106, 130, 153],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 4, 3, 1],\n", + " [ 4, 3, 1]],\n", + "\n", + " [[106, 130, 153],\n", + " [106, 130, 153],\n", + " [106, 130, 153],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[108, 129, 153],\n", + " [108, 129, 153],\n", + " [108, 129, 153],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [167, 164, 178],\n", + " ...,\n", + " [128, 143, 165],\n", + " [129, 143, 168],\n", + " [129, 143, 168]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [167, 164, 178],\n", + " ...,\n", + " [128, 143, 165],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [166, 163, 177],\n", + " ...,\n", + " [126, 142, 164],\n", + " [126, 141, 166],\n", + " [128, 142, 167]]], dtype=uint8), 'image': array([[[104, 101, 87],\n", + " [ 95, 92, 78],\n", + " [ 33, 69, 75],\n", + " ...,\n", + " [212, 215, 214],\n", + " [207, 207, 207],\n", + " [204, 204, 204]],\n", + "\n", + " [[109, 97, 79],\n", + " [115, 103, 85],\n", + " [ 50, 86, 84],\n", + " ...,\n", + " [212, 215, 214],\n", + " [208, 207, 207],\n", + " [205, 204, 204]],\n", + "\n", + " [[109, 85, 71],\n", + " [122, 98, 84],\n", + " [ 73, 104, 100],\n", + " ...,\n", + " [209, 212, 211],\n", + " [208, 208, 208],\n", + " [206, 206, 206]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 70, 55],\n", + " [ 67, 75, 60],\n", + " [ 66, 72, 55],\n", + " ...,\n", + " [ 61, 90, 102],\n", + " [ 62, 90, 105],\n", + " [ 62, 90, 105]],\n", + "\n", + " [[ 59, 65, 46],\n", + " [ 58, 64, 45],\n", + " [ 51, 57, 40],\n", + " ...,\n", + " [ 62, 91, 101],\n", + " [ 62, 91, 103],\n", + " [ 59, 88, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 49, 55, 36],\n", + " [ 43, 49, 30],\n", + " ...,\n", + " [ 62, 91, 101],\n", + " [ 60, 89, 101],\n", + " [ 60, 89, 101]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.866813 , -0.92632943, 1.9351735 , -2.669913 , -1.602569 ,\n", + " 3.5102549 , 0.5314049 , 0.25643107, -0.14546628, 0.73773456,\n", + " 0.67338413, 0.0331663 , 0.03466382, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0. , 0. , 0.01561905], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01651429, -0.0068 , -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[116, 138, 160],\n", + " [115, 137, 158],\n", + " [115, 137, 158],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[115, 137, 158],\n", + " [113, 136, 157],\n", + " [113, 136, 157],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[111, 135, 159],\n", + " [111, 135, 159],\n", + " [111, 135, 159],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 168, 182],\n", + " [168, 168, 182],\n", + " [167, 167, 181],\n", + " ...,\n", + " [127, 144, 168],\n", + " [129, 143, 168],\n", + " [129, 143, 168]],\n", + "\n", + " [[168, 168, 182],\n", + " [167, 167, 181],\n", + " [169, 169, 183],\n", + " ...,\n", + " [128, 145, 169],\n", + " [130, 144, 169],\n", + " [130, 144, 169]],\n", + "\n", + " [[168, 168, 182],\n", + " [168, 168, 182],\n", + " [167, 167, 181],\n", + " ...,\n", + " [129, 146, 170],\n", + " [131, 145, 170],\n", + " [130, 144, 169]]], dtype=uint8), 'image': array([[[102, 100, 77],\n", + " [104, 102, 79],\n", + " [ 38, 72, 69],\n", + " ...,\n", + " [181, 181, 183],\n", + " [182, 182, 182],\n", + " [185, 185, 185]],\n", + "\n", + " [[116, 100, 79],\n", + " [122, 106, 85],\n", + " [ 48, 83, 73],\n", + " ...,\n", + " [180, 180, 184],\n", + " [179, 182, 185],\n", + " [184, 187, 190]],\n", + "\n", + " [[118, 88, 72],\n", + " [133, 103, 87],\n", + " [ 74, 107, 92],\n", + " ...,\n", + " [176, 176, 180],\n", + " [179, 181, 187],\n", + " [185, 187, 193]],\n", + "\n", + " ...,\n", + "\n", + " [[ 70, 77, 51],\n", + " [ 71, 78, 52],\n", + " [ 68, 70, 52],\n", + " ...,\n", + " [ 62, 90, 96],\n", + " [ 62, 91, 99],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 56, 62, 43],\n", + " [ 58, 64, 45],\n", + " [ 58, 61, 45],\n", + " ...,\n", + " [ 64, 92, 98],\n", + " [ 62, 91, 99],\n", + " [ 62, 91, 99]],\n", + "\n", + " [[ 57, 59, 46],\n", + " [ 52, 54, 41],\n", + " [ 48, 50, 37],\n", + " ...,\n", + " [ 63, 92, 100],\n", + " [ 63, 90, 99],\n", + " [ 63, 90, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8797753 , -0.962189 , 1.9533105 , -2.6487052 , -1.5976962 ,\n", + " 3.5144436 , 0.52728325, 0.24874139, -0.1332232 , 0.7435012 ,\n", + " 0.66718084, 0.03253057, 0.03189772, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00361905, 0.00933333], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.0172 , -0.00937143, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[129, 148, 166],\n", + " [129, 148, 166],\n", + " [129, 148, 166],\n", + " ...,\n", + " [ 3, 4, 2],\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[129, 148, 166],\n", + " [129, 148, 166],\n", + " [128, 147, 165],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[129, 148, 166],\n", + " [129, 148, 166],\n", + " [128, 147, 165],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[168, 165, 180],\n", + " [168, 165, 180],\n", + " [168, 165, 180],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [168, 165, 180],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [128, 142, 167]],\n", + "\n", + " [[167, 164, 178],\n", + " [167, 164, 178],\n", + " [169, 167, 181],\n", + " ...,\n", + " [126, 142, 167],\n", + " [128, 142, 167],\n", + " [126, 141, 166]]], dtype=uint8), 'image': array([[[105, 98, 79],\n", + " [108, 101, 82],\n", + " [ 42, 71, 68],\n", + " ...,\n", + " [176, 175, 177],\n", + " [170, 168, 175],\n", + " [168, 166, 173]],\n", + "\n", + " [[119, 100, 83],\n", + " [123, 104, 87],\n", + " [ 54, 84, 76],\n", + " ...,\n", + " [175, 174, 176],\n", + " [173, 172, 176],\n", + " [169, 168, 172]],\n", + "\n", + " [[123, 92, 81],\n", + " [123, 92, 81],\n", + " [ 73, 104, 90],\n", + " ...,\n", + " [175, 174, 174],\n", + " [175, 171, 174],\n", + " [172, 168, 171]],\n", + "\n", + " ...,\n", + "\n", + " [[ 71, 78, 56],\n", + " [ 71, 78, 56],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 65, 97, 104],\n", + " [ 66, 96, 99],\n", + " [ 62, 92, 95]],\n", + "\n", + " [[ 55, 59, 36],\n", + " [ 59, 63, 40],\n", + " [ 59, 62, 44],\n", + " ...,\n", + " [ 64, 96, 103],\n", + " [ 63, 93, 98],\n", + " [ 62, 92, 97]],\n", + "\n", + " [[ 52, 55, 34],\n", + " [ 52, 55, 34],\n", + " [ 46, 50, 29],\n", + " ...,\n", + " [ 64, 96, 105],\n", + " [ 63, 92, 100],\n", + " [ 61, 90, 98]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.8943539 , -1.002066 , 1.9794574 , -2.6350296 , -1.5959238 ,\n", + " 3.5113556 , 0.52145064, 0.23904608, -0.12084077, 0.7473314 ,\n", + " 0.6629031 , 0.0326565 , 0.0314473 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00057143, -0.00780952, 0.0072381 ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01051429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[117, 136, 156],\n", + " [119, 137, 157],\n", + " [119, 137, 157],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[119, 137, 157],\n", + " [119, 137, 157],\n", + " [119, 137, 157],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " [[120, 138, 158],\n", + " [120, 138, 158],\n", + " [120, 138, 158],\n", + " ...,\n", + " [ 2, 3, 1],\n", + " [ 3, 4, 2],\n", + " [ 3, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [173, 169, 181],\n", + " ...,\n", + " [140, 146, 170],\n", + " [140, 146, 170],\n", + " [140, 146, 170]],\n", + "\n", + " [[175, 171, 183],\n", + " [176, 172, 184],\n", + " [175, 171, 183],\n", + " ...,\n", + " [139, 145, 169],\n", + " [138, 144, 168],\n", + " [139, 145, 169]],\n", + "\n", + " [[176, 172, 184],\n", + " [176, 172, 184],\n", + " [175, 171, 183],\n", + " ...,\n", + " [138, 144, 168],\n", + " [139, 145, 169],\n", + " [139, 145, 169]]], dtype=uint8), 'image': array([[[108, 105, 71],\n", + " [106, 103, 69],\n", + " [ 41, 73, 62],\n", + " ...,\n", + " [129, 132, 133],\n", + " [129, 135, 135],\n", + " [127, 133, 133]],\n", + "\n", + " [[123, 108, 78],\n", + " [121, 106, 76],\n", + " [ 56, 89, 72],\n", + " ...,\n", + " [130, 130, 132],\n", + " [128, 134, 136],\n", + " [129, 135, 137]],\n", + "\n", + " [[112, 86, 57],\n", + " [124, 98, 69],\n", + " [ 69, 101, 80],\n", + " ...,\n", + " [128, 130, 136],\n", + " [129, 135, 137],\n", + " [133, 139, 141]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 71, 49],\n", + " [ 69, 76, 54],\n", + " [ 67, 73, 56],\n", + " ...,\n", + " [ 60, 94, 102],\n", + " [ 59, 93, 103],\n", + " [ 59, 93, 103]],\n", + "\n", + " [[ 57, 64, 40],\n", + " [ 53, 60, 36],\n", + " [ 55, 58, 40],\n", + " ...,\n", + " [ 58, 93, 106],\n", + " [ 56, 91, 104],\n", + " [ 57, 92, 105]],\n", + "\n", + " [[ 53, 56, 35],\n", + " [ 51, 54, 33],\n", + " [ 47, 50, 32],\n", + " ...,\n", + " [ 58, 93, 106],\n", + " [ 57, 92, 107],\n", + " [ 56, 91, 106]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9088843 , -1.0461868 , 2.012751 , -2.6265728 , -1.5953134 ,\n", + " 3.5029898 , 0.5132325 , 0.2287732 , -0.10842803, 0.74936545,\n", + " 0.66052914, 0.03346506, 0.03213668, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([-0.00019048, -0.00876191, 0.00685714], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01068571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[119, 134, 152],\n", + " [119, 134, 152],\n", + " [119, 134, 152],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[118, 132, 151],\n", + " [118, 132, 151],\n", + " [118, 132, 151],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[118, 132, 151],\n", + " [118, 132, 151],\n", + " [118, 132, 151],\n", + " ...,\n", + " [ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [137, 144, 170],\n", + " [138, 145, 172],\n", + " [138, 145, 172]],\n", + "\n", + " [[175, 171, 183],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [140, 147, 173],\n", + " [138, 145, 172],\n", + " [138, 145, 172]],\n", + "\n", + " [[173, 169, 181],\n", + " [175, 171, 183],\n", + " [175, 171, 183],\n", + " ...,\n", + " [141, 148, 174],\n", + " [138, 145, 172],\n", + " [138, 145, 172]]], dtype=uint8), 'image': array([[[109, 104, 73],\n", + " [109, 104, 73],\n", + " [ 49, 76, 67],\n", + " ...,\n", + " [103, 104, 95],\n", + " [106, 103, 98],\n", + " [107, 104, 99]],\n", + "\n", + " [[119, 102, 73],\n", + " [116, 99, 70],\n", + " [ 57, 85, 70],\n", + " ...,\n", + " [104, 105, 96],\n", + " [105, 106, 97],\n", + " [105, 106, 97]],\n", + "\n", + " [[126, 99, 73],\n", + " [120, 93, 67],\n", + " [ 77, 106, 86],\n", + " ...,\n", + " [105, 106, 97],\n", + " [106, 107, 98],\n", + " [108, 109, 100]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 72, 51],\n", + " [ 74, 78, 57],\n", + " [ 73, 75, 62],\n", + " ...,\n", + " [ 63, 97, 107],\n", + " [ 61, 93, 102],\n", + " [ 60, 92, 101]],\n", + "\n", + " [[ 58, 65, 43],\n", + " [ 58, 65, 43],\n", + " [ 56, 61, 49],\n", + " ...,\n", + " [ 63, 97, 107],\n", + " [ 63, 95, 106],\n", + " [ 60, 92, 103]],\n", + "\n", + " [[ 54, 57, 41],\n", + " [ 52, 55, 39],\n", + " [ 45, 50, 36],\n", + " ...,\n", + " [ 60, 94, 104],\n", + " [ 64, 96, 107],\n", + " [ 60, 92, 103]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9227448 , -1.090619 , 2.0471349 , -2.6207187 , -1.5956005 ,\n", + " 3.4939177 , 0.5043737 , 0.21883176, -0.09658621, 0.750923 ,\n", + " 0.65860933, 0.03484912, 0.03367477, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.00971429, 0.00666667], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.01091429, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[121, 140, 158],\n", + " [119, 138, 155],\n", + " [121, 140, 158],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[119, 138, 155],\n", + " [119, 138, 155],\n", + " [119, 138, 155],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[119, 138, 155],\n", + " [119, 138, 155],\n", + " [119, 138, 155],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2]],\n", + "\n", + " ...,\n", + "\n", + " [[166, 165, 181],\n", + " [166, 165, 181],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [136, 146, 172],\n", + " [138, 147, 173]],\n", + "\n", + " [[165, 164, 180],\n", + " [165, 164, 180],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [138, 147, 173],\n", + " [136, 146, 172]],\n", + "\n", + " [[165, 164, 180],\n", + " [165, 164, 180],\n", + " [165, 164, 180],\n", + " ...,\n", + " [136, 146, 172],\n", + " [136, 146, 172],\n", + " [138, 147, 173]]], dtype=uint8), 'image': array([[[110, 103, 70],\n", + " [107, 100, 67],\n", + " [ 42, 72, 66],\n", + " ...,\n", + " [ 85, 82, 77],\n", + " [ 86, 83, 78],\n", + " [ 87, 84, 79]],\n", + "\n", + " [[124, 106, 74],\n", + " [117, 99, 67],\n", + " [ 57, 88, 74],\n", + " ...,\n", + " [ 85, 82, 77],\n", + " [ 87, 83, 76],\n", + " [ 90, 86, 79]],\n", + "\n", + " [[117, 90, 57],\n", + " [125, 98, 65],\n", + " [ 70, 102, 81],\n", + " ...,\n", + " [ 84, 81, 76],\n", + " [ 85, 83, 75],\n", + " [ 87, 85, 77]],\n", + "\n", + " ...,\n", + "\n", + " [[ 68, 71, 55],\n", + " [ 72, 75, 59],\n", + " [ 69, 72, 54],\n", + " ...,\n", + " [ 59, 93, 103],\n", + " [ 60, 94, 102],\n", + " [ 59, 93, 101]],\n", + "\n", + " [[ 60, 62, 49],\n", + " [ 60, 62, 49],\n", + " [ 58, 61, 45],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 63, 92, 102],\n", + " [ 63, 92, 102]],\n", + "\n", + " [[ 56, 57, 46],\n", + " [ 52, 53, 42],\n", + " [ 47, 49, 36],\n", + " ...,\n", + " [ 59, 93, 99],\n", + " [ 61, 90, 98],\n", + " [ 62, 91, 99]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9372277 , -1.138132 , 2.0828366 , -2.614289 , -1.5960315 ,\n", + " 3.4834929 , 0.4947406 , 0.20868209, -0.08435462, 0.7522201 ,\n", + " 0.656926 , 0.03664842, 0.03563798, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01047619, 0.00609524], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01731429, -0.0112 , -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[126, 144, 159],\n", + " [126, 144, 159],\n", + " [125, 143, 158],\n", + " ...,\n", + " [ 7, 5, 3],\n", + " [ 8, 6, 4],\n", + " [ 6, 4, 2]],\n", + "\n", + " [[128, 145, 161],\n", + " [126, 144, 159],\n", + " [125, 143, 158],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3]],\n", + "\n", + " [[130, 145, 161],\n", + " [129, 144, 159],\n", + " [126, 144, 159],\n", + " ...,\n", + " [ 6, 4, 2],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[175, 175, 189],\n", + " [175, 175, 189],\n", + " [174, 174, 188],\n", + " ...,\n", + " [145, 151, 166],\n", + " [145, 150, 168],\n", + " [145, 150, 168]],\n", + "\n", + " [[174, 174, 188],\n", + " [174, 174, 188],\n", + " [174, 174, 188],\n", + " ...,\n", + " [146, 152, 167],\n", + " [145, 150, 168],\n", + " [145, 150, 168]],\n", + "\n", + " [[173, 173, 187],\n", + " [174, 174, 188],\n", + " [174, 174, 188],\n", + " ...,\n", + " [146, 152, 167],\n", + " [146, 151, 169],\n", + " [145, 150, 168]]], dtype=uint8), 'image': array([[[101, 102, 89],\n", + " [ 97, 98, 85],\n", + " [ 48, 83, 84],\n", + " ...,\n", + " [ 90, 87, 82],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " [[112, 103, 84],\n", + " [114, 105, 86],\n", + " [ 41, 79, 73],\n", + " ...,\n", + " [ 84, 84, 78],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " [[114, 91, 72],\n", + " [134, 111, 92],\n", + " [ 60, 97, 84],\n", + " ...,\n", + " [ 89, 86, 81],\n", + " [ 83, 83, 77],\n", + " [ 82, 82, 76]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 70, 51],\n", + " [ 69, 75, 56],\n", + " [ 63, 72, 54],\n", + " ...,\n", + " [ 60, 89, 97],\n", + " [ 61, 90, 98],\n", + " [ 60, 89, 97]],\n", + "\n", + " [[ 56, 65, 47],\n", + " [ 55, 64, 46],\n", + " [ 54, 65, 46],\n", + " ...,\n", + " [ 63, 95, 104],\n", + " [ 59, 88, 96],\n", + " [ 60, 89, 97]],\n", + "\n", + " [[ 52, 58, 41],\n", + " [ 50, 56, 39],\n", + " [ 49, 58, 40],\n", + " ...,\n", + " [ 58, 90, 99],\n", + " [ 56, 88, 95],\n", + " [ 57, 89, 96]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9522376 , -1.1864475 , 2.1171021 , -2.6065881 , -1.5964864 ,\n", + " 3.4725657 , 0.48520797, 0.19863266, -0.07217506, 0.75350314,\n", + " 0.65520203, 0.03879603, 0.03792322, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01314286, 0.00419048], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01691429, -0.01308571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[142, 154, 166],\n", + " [142, 154, 166],\n", + " [142, 154, 166],\n", + " ...,\n", + " [ 9, 5, 3],\n", + " [ 8, 4, 0],\n", + " [ 10, 7, 3]],\n", + "\n", + " [[142, 154, 166],\n", + " [142, 154, 166],\n", + " [141, 153, 164],\n", + " ...,\n", + " [ 9, 5, 3],\n", + " [ 10, 7, 3],\n", + " [ 10, 7, 3]],\n", + "\n", + " [[141, 153, 164],\n", + " [141, 153, 164],\n", + " [141, 153, 164],\n", + " ...,\n", + " [ 9, 5, 1],\n", + " [ 10, 7, 3],\n", + " [ 10, 7, 3]],\n", + "\n", + " ...,\n", + "\n", + " [[177, 180, 191],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [196, 212, 171],\n", + " [197, 213, 172],\n", + " [197, 213, 172]],\n", + "\n", + " [[177, 180, 191],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [198, 214, 173],\n", + " [198, 214, 173],\n", + " [197, 213, 172]],\n", + "\n", + " [[176, 179, 190],\n", + " [177, 180, 191],\n", + " [177, 180, 191],\n", + " ...,\n", + " [198, 214, 173],\n", + " [198, 214, 173],\n", + " [197, 213, 172]]], dtype=uint8), 'image': array([[[104, 98, 85],\n", + " [103, 97, 84],\n", + " [ 45, 72, 81],\n", + " ...,\n", + " [138, 136, 143],\n", + " [126, 119, 130],\n", + " [113, 106, 117]],\n", + "\n", + " [[114, 99, 87],\n", + " [114, 99, 87],\n", + " [ 52, 83, 83],\n", + " ...,\n", + " [132, 131, 142],\n", + " [125, 120, 130],\n", + " [113, 108, 118]],\n", + "\n", + " [[123, 97, 84],\n", + " [122, 96, 83],\n", + " [ 70, 99, 96],\n", + " ...,\n", + " [133, 134, 149],\n", + " [116, 113, 129],\n", + " [102, 99, 115]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 54],\n", + " [ 71, 74, 58],\n", + " [ 67, 73, 56],\n", + " ...,\n", + " [ 56, 90, 100],\n", + " [ 54, 83, 95],\n", + " [ 61, 90, 102]],\n", + "\n", + " [[ 58, 61, 45],\n", + " [ 59, 62, 46],\n", + " [ 60, 62, 49],\n", + " ...,\n", + " [ 58, 90, 101],\n", + " [ 58, 87, 99],\n", + " [ 54, 83, 95]],\n", + "\n", + " [[ 52, 54, 43],\n", + " [ 50, 52, 41],\n", + " [ 45, 47, 38],\n", + " ...,\n", + " [ 61, 93, 104],\n", + " [ 60, 92, 103],\n", + " [ 54, 86, 97]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9681327 , -1.236956 , 2.1511526 , -2.5970871 , -1.5968579 ,\n", + " 3.4606204 , 0.4752485 , 0.18844554, -0.05988858, 0.7547231 ,\n", + " 0.65350336, 0.0411324 , 0.04042946, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01333333, 0.00342857], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01662857, -0.01388571, -0.02 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 63, 60, 69],\n", + " [ 66, 63, 73],\n", + " [ 70, 69, 79],\n", + " ...,\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " [[ 81, 84, 95],\n", + " [ 86, 89, 100],\n", + " [ 87, 93, 103],\n", + " ...,\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4]],\n", + "\n", + " [[101, 110, 124],\n", + " [103, 112, 126],\n", + " [108, 117, 129],\n", + " ...,\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4],\n", + " [ 10, 6, 4]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 188, 197],\n", + " [189, 188, 197],\n", + " [189, 188, 197],\n", + " ...,\n", + " [195, 211, 170],\n", + " [194, 212, 171],\n", + " [194, 212, 171]],\n", + "\n", + " [[188, 187, 196],\n", + " [188, 187, 196],\n", + " [188, 187, 196],\n", + " ...,\n", + " [197, 213, 172],\n", + " [196, 215, 173],\n", + " [194, 212, 171]],\n", + "\n", + " [[187, 186, 195],\n", + " [188, 187, 196],\n", + " [188, 187, 196],\n", + " ...,\n", + " [197, 213, 172],\n", + " [195, 214, 172],\n", + " [194, 212, 171]]], dtype=uint8), 'image': array([[[ 98, 101, 80],\n", + " [ 95, 98, 77],\n", + " [ 36, 74, 72],\n", + " ...,\n", + " [155, 157, 165],\n", + " [156, 158, 166],\n", + " [156, 158, 166]],\n", + "\n", + " [[107, 95, 77],\n", + " [114, 102, 84],\n", + " [ 53, 90, 81],\n", + " ...,\n", + " [151, 156, 165],\n", + " [151, 156, 165],\n", + " [151, 156, 165]],\n", + "\n", + " [[118, 92, 79],\n", + " [123, 97, 84],\n", + " [ 67, 102, 92],\n", + " ...,\n", + " [151, 155, 167],\n", + " [148, 152, 164],\n", + " [147, 151, 163]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 68, 53],\n", + " [ 74, 73, 58],\n", + " [ 70, 72, 56],\n", + " ...,\n", + " [ 59, 93, 101],\n", + " [ 63, 95, 102],\n", + " [ 64, 96, 103]],\n", + "\n", + " [[ 65, 65, 43],\n", + " [ 63, 63, 41],\n", + " [ 58, 58, 38],\n", + " ...,\n", + " [ 60, 92, 99],\n", + " [ 64, 96, 103],\n", + " [ 63, 95, 102]],\n", + "\n", + " [[ 56, 56, 32],\n", + " [ 56, 56, 32],\n", + " [ 50, 50, 28],\n", + " ...,\n", + " [ 65, 94, 102],\n", + " [ 63, 95, 104],\n", + " [ 61, 93, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.98276 , -1.2880267 , 2.1854315 , -2.5885217 , -1.5973486 ,\n", + " 3.4484115 , 0.46462205, 0.17914946, -0.04822387, 0.75541085,\n", + " 0.65233696, 0.04393689, 0.04336553, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0. , -0.01409524, 0.00152381], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01582857, -0.01577143, -0.0192 ], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 4, 3, 1],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7],\n", + " [ 11, 7, 5]],\n", + "\n", + " [[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 11, 7, 5],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " [[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 11, 7, 5],\n", + " [ 12, 8, 7],\n", + " [ 12, 8, 7]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 191, 200],\n", + " [189, 191, 200],\n", + " [188, 190, 198],\n", + " ...,\n", + " [201, 218, 174],\n", + " [201, 218, 174],\n", + " [201, 218, 174]],\n", + "\n", + " [[188, 190, 198],\n", + " [188, 190, 198],\n", + " [187, 188, 197],\n", + " ...,\n", + " [199, 216, 173],\n", + " [201, 218, 174],\n", + " [201, 218, 174]],\n", + "\n", + " [[186, 187, 196],\n", + " [187, 188, 197],\n", + " [187, 188, 197],\n", + " ...,\n", + " [201, 218, 174],\n", + " [199, 216, 173],\n", + " [199, 216, 173]]], dtype=uint8), 'image': array([[[106, 103, 89],\n", + " [102, 99, 85],\n", + " [ 43, 73, 78],\n", + " ...,\n", + " [152, 164, 170],\n", + " [150, 160, 165],\n", + " [151, 161, 166]],\n", + "\n", + " [[115, 103, 87],\n", + " [116, 104, 88],\n", + " [ 55, 86, 82],\n", + " ...,\n", + " [152, 162, 169],\n", + " [150, 157, 163],\n", + " [150, 157, 163]],\n", + "\n", + " [[113, 87, 72],\n", + " [122, 96, 81],\n", + " [ 70, 101, 87],\n", + " ...,\n", + " [150, 160, 167],\n", + " [146, 153, 161],\n", + " [147, 154, 162]],\n", + "\n", + " ...,\n", + "\n", + " [[ 67, 70, 54],\n", + " [ 72, 75, 59],\n", + " [ 64, 72, 59],\n", + " ...,\n", + " [ 61, 93, 104],\n", + " [ 64, 96, 105],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 61, 64, 48],\n", + " [ 60, 63, 47],\n", + " [ 57, 62, 52],\n", + " ...,\n", + " [ 59, 93, 103],\n", + " [ 64, 96, 105],\n", + " [ 60, 92, 101]],\n", + "\n", + " [[ 54, 55, 44],\n", + " [ 52, 53, 42],\n", + " [ 48, 53, 41],\n", + " ...,\n", + " [ 60, 94, 104],\n", + " [ 60, 92, 101],\n", + " [ 61, 93, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-2.9976504 , -1.3427528 , 2.2203817 , -2.5778692 , -1.5979112 ,\n", + " 3.4355202 , 0.45309877, 0.16995318, -0.03630768, 0.7559353 ,\n", + " 0.6512862 , 0.04700414, 0.04667673, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00114286, -0.01142857, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01417143, -0.01828571, -0.01594286], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 11, 8, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 7, 4],\n", + " [ 13, 7, 4]],\n", + "\n", + " [[ 10, 7, 3],\n", + " [ 10, 7, 3],\n", + " [ 9, 5, 1],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 9, 5, 1],\n", + " [ 9, 5, 1],\n", + " [ 9, 5, 1],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [184, 190, 198],\n", + " ...,\n", + " [201, 220, 176],\n", + " [202, 221, 177],\n", + " [202, 221, 177]],\n", + "\n", + " [[183, 189, 197],\n", + " [184, 190, 198],\n", + " [183, 189, 197],\n", + " ...,\n", + " [202, 221, 177],\n", + " [201, 220, 176],\n", + " [201, 220, 176]],\n", + "\n", + " [[184, 190, 198],\n", + " [183, 189, 197],\n", + " [183, 189, 197],\n", + " ...,\n", + " [201, 220, 176],\n", + " [201, 220, 176],\n", + " [201, 220, 176]]], dtype=uint8), 'image': array([[[107, 101, 86],\n", + " [102, 96, 81],\n", + " [ 39, 66, 77],\n", + " ...,\n", + " [154, 161, 167],\n", + " [154, 161, 167],\n", + " [153, 160, 166]],\n", + "\n", + " [[118, 98, 88],\n", + " [122, 102, 92],\n", + " [ 63, 91, 95],\n", + " ...,\n", + " [153, 160, 166],\n", + " [154, 159, 168],\n", + " [155, 160, 169]],\n", + "\n", + " [[121, 90, 79],\n", + " [128, 97, 86],\n", + " [ 70, 94, 92],\n", + " ...,\n", + " [153, 158, 167],\n", + " [147, 151, 163],\n", + " [145, 149, 161]],\n", + "\n", + " ...,\n", + "\n", + " [[ 60, 69, 60],\n", + " [ 67, 76, 67],\n", + " [ 65, 73, 60],\n", + " ...,\n", + " [ 60, 92, 101],\n", + " [ 59, 91, 100],\n", + " [ 61, 93, 102]],\n", + "\n", + " [[ 56, 64, 51],\n", + " [ 53, 61, 48],\n", + " [ 49, 57, 42],\n", + " ...,\n", + " [ 58, 90, 99],\n", + " [ 56, 88, 99],\n", + " [ 57, 89, 100]],\n", + "\n", + " [[ 53, 58, 46],\n", + " [ 51, 56, 44],\n", + " [ 43, 51, 36],\n", + " ...,\n", + " [ 57, 91, 99],\n", + " [ 57, 91, 101],\n", + " [ 58, 92, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0120497 , -1.400019 , 2.2567096 , -2.5662448 , -1.5984977 ,\n", + " 3.4220662 , 0.4402678 , 0.16121309, -0.02498595, 0.75610036,\n", + " 0.6506091 , 0.05005798, 0.05014284, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([ 0.00533333, -0.01238095, 0. ], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.01262857, -0.01931429, -0.01217143], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 6, 4, 2],\n", + " [ 8, 6, 4],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 5, 5, 3],\n", + " [ 5, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 11, 8, 4],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[190, 192, 201],\n", + " [190, 192, 201],\n", + " [189, 191, 200],\n", + " ...,\n", + " [203, 223, 178],\n", + " [204, 224, 180],\n", + " [204, 224, 180]],\n", + "\n", + " [[187, 188, 197],\n", + " [188, 190, 198],\n", + " [188, 190, 198],\n", + " ...,\n", + " [203, 223, 178],\n", + " [201, 220, 176],\n", + " [201, 220, 176]],\n", + "\n", + " [[187, 188, 197],\n", + " [187, 188, 197],\n", + " [188, 190, 198],\n", + " ...,\n", + " [203, 223, 178],\n", + " [202, 221, 177],\n", + " [201, 220, 176]]], dtype=uint8), 'image': array([[[110, 105, 85],\n", + " [109, 104, 84],\n", + " [ 40, 71, 69],\n", + " ...,\n", + " [152, 159, 165],\n", + " [150, 155, 164],\n", + " [150, 155, 164]],\n", + "\n", + " [[121, 104, 86],\n", + " [124, 107, 89],\n", + " [ 55, 87, 78],\n", + " ...,\n", + " [151, 161, 166],\n", + " [150, 155, 164],\n", + " [150, 155, 164]],\n", + "\n", + " [[111, 83, 64],\n", + " [127, 99, 80],\n", + " [ 76, 109, 92],\n", + " ...,\n", + " [147, 161, 167],\n", + " [146, 152, 163],\n", + " [141, 147, 158]],\n", + "\n", + " ...,\n", + "\n", + " [[ 64, 74, 51],\n", + " [ 70, 80, 57],\n", + " [ 65, 75, 48],\n", + " ...,\n", + " [ 58, 92, 98],\n", + " [ 61, 93, 100],\n", + " [ 61, 93, 100]],\n", + "\n", + " [[ 55, 65, 42],\n", + " [ 55, 65, 42],\n", + " [ 54, 64, 39],\n", + " ...,\n", + " [ 64, 96, 105],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 48, 57, 37],\n", + " [ 44, 53, 33],\n", + " [ 44, 53, 33],\n", + " ...,\n", + " [ 63, 92, 104],\n", + " [ 66, 93, 104],\n", + " [ 64, 91, 102]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.024355 , -1.4538416 , 2.2930508 , -2.5565522 , -1.5979952 ,\n", + " 3.4096293 , 0.42665708, 0.1540214 , -0.0161692 , 0.75591356,\n", + " 0.65040565, 0.05278223, 0.05273771, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0.0032381, 0. , 0.0047619], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0.00514286, -0.00582857, -0.00068571], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 13, 7, 4],\n", + " [ 14, 9, 5],\n", + " [ 14, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[191, 195, 203],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [205, 222, 178],\n", + " [205, 222, 178]],\n", + "\n", + " [[191, 195, 203],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [204, 221, 177],\n", + " [204, 221, 177]],\n", + "\n", + " [[189, 193, 202],\n", + " [191, 195, 203],\n", + " [191, 195, 203],\n", + " ...,\n", + " [204, 221, 177],\n", + " [204, 221, 177],\n", + " [204, 221, 177]]], dtype=uint8), 'image': array([[[109, 105, 94],\n", + " [107, 103, 92],\n", + " [ 34, 61, 70],\n", + " ...,\n", + " [153, 161, 162],\n", + " [153, 160, 164],\n", + " [153, 160, 164]],\n", + "\n", + " [[120, 107, 94],\n", + " [120, 107, 94],\n", + " [ 51, 80, 79],\n", + " ...,\n", + " [150, 157, 163],\n", + " [152, 159, 167],\n", + " [150, 157, 165]],\n", + "\n", + " [[111, 85, 74],\n", + " [130, 104, 93],\n", + " [ 75, 105, 99],\n", + " ...,\n", + " [149, 155, 168],\n", + " [146, 154, 169],\n", + " [144, 152, 167]],\n", + "\n", + " ...,\n", + "\n", + " [[ 62, 75, 60],\n", + " [ 68, 81, 66],\n", + " [ 71, 81, 67],\n", + " ...,\n", + " [ 64, 96, 107],\n", + " [ 60, 92, 101],\n", + " [ 57, 89, 98]],\n", + "\n", + " [[ 56, 67, 48],\n", + " [ 56, 67, 48],\n", + " [ 54, 62, 47],\n", + " ...,\n", + " [ 56, 88, 99],\n", + " [ 54, 85, 99],\n", + " [ 57, 88, 102]],\n", + "\n", + " [[ 50, 59, 39],\n", + " [ 49, 58, 38],\n", + " [ 46, 55, 35],\n", + " ...,\n", + " [ 56, 87, 101],\n", + " [ 57, 88, 104],\n", + " [ 60, 91, 107]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0366597 , -1.50608 , 2.3327332 , -2.5523531 , -1.5953492 ,\n", + " 3.3971448 , 0.41204247, 0.14764796, -0.00976385, 0.75563943,\n", + " 0.65026915, 0.05629408, 0.054681 , 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 0.0, 'world_vector': array([-0., 0., -0.], dtype=float32)}, 'is_first': False, 'is_last': False, 'is_terminal': False, 'observation': {'hand_image': array([[[ 5, 5, 3],\n", + " [ 3, 4, 2],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 5, 5, 3],\n", + " [ 5, 5, 3],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 10, 9, 5],\n", + " [ 11, 11, 6],\n", + " [ 11, 11, 6]],\n", + "\n", + " [[ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " [ 7, 5, 3],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 13, 10, 6],\n", + " [ 13, 10, 6]],\n", + "\n", + " ...,\n", + "\n", + " [[187, 194, 202],\n", + " [187, 194, 202],\n", + " [186, 193, 201],\n", + " ...,\n", + " [211, 225, 184],\n", + " [211, 225, 184],\n", + " [211, 225, 184]],\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [186, 193, 201],\n", + " ...,\n", + " [211, 225, 184],\n", + " [211, 225, 184],\n", + " [211, 225, 184]],\n", + "\n", + " [[185, 192, 200],\n", + " [185, 192, 200],\n", + " [186, 193, 201],\n", + " ...,\n", + " [212, 226, 185],\n", + " [211, 225, 184],\n", + " [211, 225, 184]]], dtype=uint8), 'image': array([[[111, 103, 77],\n", + " [114, 106, 80],\n", + " [ 49, 75, 62],\n", + " ...,\n", + " [ 47, 53, 55],\n", + " [ 49, 48, 50],\n", + " [ 51, 50, 52]],\n", + "\n", + " [[118, 100, 78],\n", + " [119, 101, 79],\n", + " [ 61, 89, 74],\n", + " ...,\n", + " [ 49, 52, 51],\n", + " [ 57, 48, 51],\n", + " [ 58, 49, 52]],\n", + "\n", + " [[113, 86, 64],\n", + " [126, 99, 77],\n", + " [ 82, 111, 93],\n", + " ...,\n", + " [ 53, 53, 47],\n", + " [ 64, 49, 50],\n", + " [ 65, 50, 51]],\n", + "\n", + " ...,\n", + "\n", + " [[ 69, 75, 56],\n", + " [ 70, 76, 57],\n", + " [ 69, 75, 56],\n", + " ...,\n", + " [ 63, 94, 108],\n", + " [ 66, 98, 109],\n", + " [ 66, 98, 109]],\n", + "\n", + " [[ 56, 63, 39],\n", + " [ 56, 63, 39],\n", + " [ 55, 61, 42],\n", + " ...,\n", + " [ 65, 96, 110],\n", + " [ 68, 97, 109],\n", + " [ 67, 96, 108]],\n", + "\n", + " [[ 48, 55, 33],\n", + " [ 50, 57, 35],\n", + " [ 49, 55, 36],\n", + " ...,\n", + " [ 65, 97, 108],\n", + " [ 68, 97, 109],\n", + " [ 67, 96, 108]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0452423 , -1.5262693 , 2.3518927 , -2.553457 , -1.5919966 ,\n", + " 3.3911233 , 0.405479 , 0.1441851 , -0.00875834, 0.75636166,\n", + " 0.6493327 , 0.05772375, 0.05433304, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 1.0, 'world_vector': array([0., 0., 0.], dtype=float32)}, 'is_first': False, 'is_last': True, 'is_terminal': True, 'observation': {'hand_image': array([[[ 4, 4, 0],\n", + " [ 6, 5, 0],\n", + " [ 8, 6, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 193, 202],\n", + " [189, 193, 202],\n", + " [188, 192, 201],\n", + " ...,\n", + " [211, 224, 185],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[188, 192, 201],\n", + " [188, 192, 201],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[187, 191, 200],\n", + " [187, 191, 200],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]]], dtype=uint8), 'image': array([[[108, 101, 82],\n", + " [104, 97, 78],\n", + " [ 46, 72, 72],\n", + " ...,\n", + " [ 12, 19, 13],\n", + " [ 7, 15, 16],\n", + " [ 39, 47, 48]],\n", + "\n", + " [[115, 102, 78],\n", + " [119, 106, 82],\n", + " [ 61, 91, 83],\n", + " ...,\n", + " [ 16, 20, 15],\n", + " [ 23, 26, 29],\n", + " [ 51, 54, 57]],\n", + "\n", + " [[114, 90, 67],\n", + " [124, 100, 77],\n", + " [ 71, 102, 86],\n", + " ...,\n", + " [ 22, 25, 24],\n", + " [ 43, 43, 45],\n", + " [ 48, 48, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 72, 55],\n", + " [ 71, 77, 60],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 67, 96, 106],\n", + " [ 66, 98, 107],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 55, 61, 44],\n", + " ...,\n", + " [ 69, 98, 106],\n", + " [ 69, 98, 106],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 46, 52, 33],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [ 68, 97, 107],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0464277 , -1.5272526 , 2.3538566 , -2.5545247 , -1.5915655 ,\n", + " 3.390477 , 0.40496254, 0.14374796, -0.00899097, 0.75652575,\n", + " 0.64914 , 0.05783392, 0.05423374, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 1.0}\n", + "{'action': {'gripper_closedness_action': 0.0, 'rotation_delta': array([0., 0., 0.], dtype=float32), 'terminate_episode': 1.0, 'world_vector': array([0., 0., 0.], dtype=float32)}, 'is_first': False, 'is_last': True, 'is_terminal': True, 'observation': {'hand_image': array([[[ 4, 4, 0],\n", + " [ 6, 5, 0],\n", + " [ 8, 6, 4],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 15, 11, 7]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " [[ 6, 5, 0],\n", + " [ 6, 5, 0],\n", + " [ 6, 4, 2],\n", + " ...,\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5],\n", + " [ 12, 9, 5]],\n", + "\n", + " ...,\n", + "\n", + " [[189, 193, 202],\n", + " [189, 193, 202],\n", + " [188, 192, 201],\n", + " ...,\n", + " [211, 224, 185],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[188, 192, 201],\n", + " [188, 192, 201],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]],\n", + "\n", + " [[187, 191, 200],\n", + " [187, 191, 200],\n", + " [187, 191, 200],\n", + " ...,\n", + " [212, 225, 186],\n", + " [209, 224, 185],\n", + " [209, 224, 185]]], dtype=uint8), 'image': array([[[108, 101, 82],\n", + " [104, 97, 78],\n", + " [ 46, 72, 72],\n", + " ...,\n", + " [ 12, 19, 13],\n", + " [ 7, 15, 16],\n", + " [ 39, 47, 48]],\n", + "\n", + " [[115, 102, 78],\n", + " [119, 106, 82],\n", + " [ 61, 91, 83],\n", + " ...,\n", + " [ 16, 20, 15],\n", + " [ 23, 26, 29],\n", + " [ 51, 54, 57]],\n", + "\n", + " [[114, 90, 67],\n", + " [124, 100, 77],\n", + " [ 71, 102, 86],\n", + " ...,\n", + " [ 22, 25, 24],\n", + " [ 43, 43, 45],\n", + " [ 48, 48, 50]],\n", + "\n", + " ...,\n", + "\n", + " [[ 66, 72, 55],\n", + " [ 71, 77, 60],\n", + " [ 69, 75, 58],\n", + " ...,\n", + " [ 67, 96, 106],\n", + " [ 66, 98, 107],\n", + " [ 63, 95, 104]],\n", + "\n", + " [[ 57, 63, 44],\n", + " [ 55, 61, 42],\n", + " [ 55, 61, 44],\n", + " ...,\n", + " [ 69, 98, 106],\n", + " [ 69, 98, 106],\n", + " [ 63, 92, 100]],\n", + "\n", + " [[ 51, 57, 38],\n", + " [ 46, 52, 33],\n", + " [ 45, 51, 32],\n", + " ...,\n", + " [ 68, 97, 107],\n", + " [ 67, 96, 104],\n", + " [ 63, 92, 100]]], dtype=uint8), 'image_with_depth': array([[[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " ...,\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]],\n", + "\n", + " [[0.],\n", + " [0.],\n", + " [0.],\n", + " ...,\n", + " [0.],\n", + " [0.],\n", + " [0.]]], dtype=float32), 'natural_language_embedding': array([-2.57158605e-03, 6.02926277e-02, -7.37616792e-02, 3.29931714e-02,\n", + " -1.16353203e-02, 1.19513914e-01, 8.21190514e-03, -8.53808690e-03,\n", + " -6.79049194e-02, 1.64459888e-02, 2.76963338e-02, 6.32510632e-02,\n", + " 4.95920777e-02, 1.69661436e-02, -7.80123398e-02, -2.72530336e-02,\n", + " 5.22834342e-03, -5.49627207e-02, -1.84190944e-02, 4.42720391e-02,\n", + " -2.43228097e-02, 5.38170673e-02, 1.62677001e-02, 7.71915838e-02,\n", + " 5.35852276e-02, 4.15176415e-04, 9.98403877e-02, 1.27209807e-02,\n", + " 1.78317539e-02, -8.67791474e-03, 4.10777628e-02, 4.94791046e-02,\n", + " -2.21885145e-02, 7.04456046e-02, 2.25713057e-03, -5.83644770e-02,\n", + " -2.18421370e-02, 7.38294097e-03, -4.87283356e-02, 2.55089551e-02,\n", + " 4.51700538e-02, -8.37735459e-02, 5.01492731e-02, -6.32911250e-02,\n", + " 2.00021593e-03, 3.89375314e-02, 4.25622910e-02, 3.82302292e-02,\n", + " -3.35912667e-02, -6.26519276e-03, -1.08870268e-02, 5.14937704e-03,\n", + " 1.18668023e-02, 4.01555840e-03, -5.30255884e-02, 2.93111466e-02,\n", + " -4.84070256e-02, -5.23491092e-02, -3.65836024e-02, 3.45632136e-02,\n", + " -3.12645063e-02, 5.15265241e-02, 9.16012470e-03, -6.12752698e-02,\n", + " 4.81236428e-02, -1.08801965e-02, 3.75676481e-03, 6.74141245e-03,\n", + " -6.43283427e-02, 7.06979074e-03, 1.00523792e-02, -3.90343019e-03,\n", + " 7.84378592e-03, -5.20281494e-02, -3.30404192e-02, 8.01846907e-02,\n", + " 3.13980505e-02, 6.07841276e-02, -1.72714777e-02, 5.02604023e-02,\n", + " -1.63720306e-02, -3.69528914e-03, 1.65575761e-02, -2.23633442e-02,\n", + " 7.42878718e-03, 3.24668400e-02, 2.92010810e-02, 2.94367038e-02,\n", + " 3.09225656e-02, -3.82606289e-03, -5.89156151e-02, -4.54527549e-02,\n", + " 3.59245464e-02, -2.56407615e-02, 5.63461753e-03, 1.80398244e-02,\n", + " 1.16252406e-02, 2.60653049e-02, 3.80461887e-02, -5.39722778e-02,\n", + " 4.16974314e-02, 6.03158250e-02, -6.42934740e-02, -7.83248395e-02,\n", + " -6.86524762e-03, 3.80139314e-02, 5.77860847e-02, 1.03577375e-02,\n", + " 1.01035656e-02, 3.25667635e-02, -2.89234053e-02, -3.34806480e-02,\n", + " -3.36649343e-02, 2.65200958e-02, 1.81382373e-02, -7.55209997e-02,\n", + " 3.49148549e-02, 1.01057859e-02, -4.79022302e-02, 3.03195510e-02,\n", + " -3.79341990e-02, -1.43008679e-02, -3.42715606e-02, -5.98295741e-02,\n", + " 1.34592783e-02, 2.79730670e-02, -8.33507478e-02, 4.15123105e-02,\n", + " -1.99438278e-02, 6.56811818e-02, 6.71588629e-02, 9.49943438e-02,\n", + " 6.41911402e-02, -5.04771853e-03, 5.97754447e-03, -1.84140932e-02,\n", + " -6.91598398e-04, 8.50071758e-02, -1.85156409e-02, -2.24846341e-02,\n", + " 5.62023707e-02, 1.78721864e-02, -4.16856892e-02, 1.18526137e-02,\n", + " 2.81594321e-02, -4.89021130e-02, -1.03083938e-01, 1.52442278e-02,\n", + " -1.67342927e-02, -6.90858588e-02, 3.62240858e-02, -2.81883515e-02,\n", + " 2.57765166e-02, -3.82834300e-02, 3.40406783e-02, 8.87050778e-02,\n", + " 2.60937735e-02, 8.98883045e-02, 3.16433460e-02, -5.07250316e-02,\n", + " 5.68391830e-02, 3.69277745e-02, 3.17989402e-02, 7.15628592e-03,\n", + " 4.45136391e-02, -2.32636146e-02, 2.44761314e-02, 1.87419076e-02,\n", + " 7.56445248e-03, 3.20580648e-03, 3.53567488e-02, -4.85384427e-02,\n", + " -3.34740169e-02, 7.03618824e-02, 6.59405366e-02, 9.33891237e-02,\n", + " 7.69008230e-03, 4.50974330e-02, 1.61097851e-02, 2.86435317e-02,\n", + " -3.35345529e-02, -3.80594879e-02, 5.55764586e-02, 1.74242705e-02,\n", + " -4.85781543e-02, 2.13113390e-02, 3.14885788e-02, -6.77365661e-02,\n", + " -7.05379322e-02, 7.90722221e-02, 3.78336571e-02, -7.46925846e-02,\n", + " 5.41023426e-02, -1.28568849e-02, -1.25438068e-03, -1.42236259e-02,\n", + " 4.15202565e-02, 8.18325281e-02, 8.10751866e-04, 1.53541351e-02,\n", + " 2.75699440e-02, -6.61135651e-04, 8.91992897e-02, 6.87056258e-02,\n", + " 3.81006510e-03, -4.66797426e-02, -3.05856083e-05, 2.11712886e-02,\n", + " 4.57162829e-03, -6.59482926e-02, 3.58506329e-02, -5.66818044e-02,\n", + " 1.09159976e-01, 9.46280360e-02, -1.45741776e-02, 8.89856145e-02,\n", + " 7.42360484e-03, -6.50147069e-03, 1.07539643e-03, -6.10144772e-02,\n", + " 7.97795728e-02, -5.07865921e-02, 2.79751550e-02, 6.34170175e-02,\n", + " 5.97889498e-02, 1.96064934e-02, 3.76590490e-02, -7.36658424e-02,\n", + " -4.59946170e-02, 2.09045149e-02, -2.89786384e-02, -1.59840344e-03,\n", + " -2.33985987e-02, 5.19292839e-02, -9.05067543e-04, -5.12565039e-02,\n", + " 3.13991345e-02, 5.35394717e-03, 3.69249582e-02, -1.92631911e-02,\n", + " 6.35778755e-02, -1.66081246e-02, 2.13495959e-02, 2.52991915e-02,\n", + " 1.18455421e-02, -7.56287109e-03, 6.56914860e-02, -2.22543441e-03,\n", + " 6.94764480e-02, 1.04600089e-02, -3.13320458e-02, 6.09784089e-02,\n", + " 5.89383114e-03, 3.87893105e-03, -1.26397675e-02, 5.52299693e-02,\n", + " -6.93048537e-02, -7.28922412e-02, -3.01046297e-02, -4.26823236e-02,\n", + " -1.41470730e-02, -5.82202375e-02, 3.11104674e-02, 1.90218333e-02,\n", + " 5.58962822e-02, 2.19248850e-02, 3.46248038e-02, 2.39384249e-02,\n", + " -1.16308266e-02, -7.35199451e-02, 4.75460105e-02, -5.92908524e-02,\n", + " 1.81670114e-02, 1.02708619e-02, -4.24497947e-03, 5.58442529e-03,\n", + " 6.15964569e-02, -4.49698754e-02, 4.40620296e-02, -6.07554168e-02,\n", + " -3.68078761e-02, 3.90757471e-02, -3.85164097e-02, -2.67252158e-02,\n", + " -3.61269042e-02, -1.73912849e-02, 5.61321713e-02, -7.58717656e-02,\n", + " 3.09000164e-02, 1.99048296e-02, -4.89114784e-02, 2.17666477e-02,\n", + " 3.10930051e-03, 2.16650665e-02, 4.35021408e-02, 4.08993401e-02,\n", + " -9.68422089e-03, -9.99179669e-03, 3.71321701e-02, 6.84710321e-05,\n", + " -6.39796304e-03, 3.88148986e-02, -4.66439920e-03, -5.37038669e-02,\n", + " -6.70375973e-02, 2.09471472e-02, 1.09973224e-02, -4.37962357e-03,\n", + " -3.15875225e-02, -2.87802238e-03, -2.30752137e-02, -7.11161643e-03,\n", + " 1.18402764e-04, -3.27782743e-02, 3.84083763e-02, 2.66488455e-02,\n", + " 3.42886932e-02, 4.61963005e-02, -8.17868337e-02, -5.93606681e-02,\n", + " -1.85964145e-02, -8.99645835e-02, 7.87325650e-02, -3.16157117e-02,\n", + " 5.77214314e-03, 5.56534305e-02, -2.50665209e-04, 4.11798023e-02,\n", + " 3.04766912e-02, -7.29565916e-04, -4.92484421e-02, -7.57742152e-02,\n", + " -6.13959096e-02, -6.03640527e-02, 2.23056823e-02, 2.30864584e-02,\n", + " -5.98571189e-02, -9.64401215e-02, -4.16774377e-02, -1.61851523e-03,\n", + " -4.11788514e-03, 4.80363481e-02, 3.30010019e-02, 4.80703264e-02,\n", + " 5.40196697e-06, -2.39382572e-02, 2.45661195e-02, 1.14923483e-02,\n", + " 3.17365527e-02, -2.51274072e-02, 2.70691961e-02, 8.00720137e-03,\n", + " 7.59266503e-03, 5.21597713e-02, -4.72620018e-02, 8.38174075e-02,\n", + " 1.19384481e-02, -5.76202236e-02, 2.73530409e-02, 3.80469300e-02,\n", + " -2.51086187e-02, 4.82048094e-02, -6.06601350e-02, 6.24365397e-02,\n", + " -1.71477031e-02, 1.98254064e-02, -3.12193530e-03, 1.05301522e-01,\n", + " -1.34950085e-02, -7.61624053e-02, -2.12372262e-02, 1.24399532e-02,\n", + " 8.53588339e-03, -8.29173550e-02, -1.20788841e-02, 3.10686510e-02,\n", + " 9.15688276e-03, 6.84849098e-02, -7.92127699e-02, 7.95277953e-02,\n", + " 1.60143748e-02, -1.10316910e-02, -5.70412800e-02, -6.57125860e-02,\n", + " -1.76433623e-02, 6.36939928e-02, 5.92441894e-02, 4.19832505e-02,\n", + " -3.15530300e-02, 2.43271478e-02, 5.33346198e-02, -4.92140539e-02,\n", + " -9.49010253e-03, 3.35166082e-02, 7.96948597e-02, 9.35266688e-02,\n", + " -4.21452597e-02, -5.03690504e-02, 4.95838672e-02, -7.24544451e-02,\n", + " 4.57415506e-02, 1.42656928e-02, 5.59303071e-03, 1.50836948e-02,\n", + " -1.77703751e-03, 1.00079803e-02, -2.58569662e-02, 5.30980574e-03,\n", + " 2.51242071e-02, -3.68951224e-02, 3.00628059e-02, -1.42559465e-02,\n", + " -2.35112458e-02, -9.47278291e-02, -4.53135557e-02, 2.40880251e-03,\n", + " 2.55071688e-02, -1.19579537e-02, -8.40864703e-02, -1.58306805e-03,\n", + " -6.34560734e-02, 3.60815786e-02, 2.66716946e-02, -3.81297246e-02,\n", + " 4.38378938e-02, 5.16037643e-02, -3.69294658e-02, -9.29967090e-02,\n", + " -3.08340956e-02, -1.02270888e-02, 4.18634452e-02, -1.11618266e-03,\n", + " -2.74340622e-02, -3.29741500e-02, 5.96650504e-02, -8.76902938e-02,\n", + " 7.31772333e-02, 2.98777800e-02, -5.76180592e-02, 5.00687063e-02,\n", + " -5.89269064e-02, -4.65865172e-02, 2.06374954e-02, -1.54341422e-02,\n", + " 4.64191586e-02, 4.95909201e-03, -7.48227015e-02, -3.97210196e-02,\n", + " -2.59292647e-02, -6.54241955e-03, -5.74038625e-02, 9.24727619e-02,\n", + " 4.26558331e-02, -6.28618449e-02, 4.24557067e-02, 5.47703058e-02,\n", + " 4.19440959e-03, 2.13806406e-02, 5.36382049e-02, -1.85115691e-02,\n", + " -2.43785027e-02, -2.73437425e-02, 1.80081073e-02, -6.24120049e-02,\n", + " -6.18575811e-02, 2.71372329e-02, -3.99724431e-02, 8.47561881e-02,\n", + " 4.09930618e-03, 5.22739813e-02, 3.64478528e-02, -1.22416150e-02,\n", + " 4.01362889e-02, -3.25129963e-02, -5.94159216e-02, -3.41437459e-02,\n", + " 4.01273966e-02, 1.35184750e-02, -1.03093870e-02, -1.81289595e-02,\n", + " -1.23641565e-02, -4.45375144e-02, 1.42934918e-02, 7.27783889e-03,\n", + " -7.73354294e-03, -4.47413512e-02, -8.41288790e-02, -3.38747608e-03,\n", + " -4.19294648e-02, 1.28766904e-02, -9.22035333e-03, 2.22486015e-02,\n", + " 1.35514943e-04, -3.00303362e-02, 5.59921116e-02, -4.55022305e-02,\n", + " -7.23699778e-02, -5.21848239e-02, 3.07216309e-02, -5.69595844e-02,\n", + " -1.75060984e-02, -3.47700925e-03, -5.66009469e-02, -7.95593951e-03,\n", + " -7.46362358e-02, -1.07114641e-02, 3.81466374e-02, -3.68379578e-02,\n", + " 7.73133859e-02, -3.80876996e-02, -5.92719913e-02, -7.04052597e-02],\n", + " dtype=float32), 'natural_language_instruction': b'sweep the green cloth to the left side of the table', 'robot_state': array([-3.0464277 , -1.5272526 , 2.3538566 , -2.5545247 , -1.5915655 ,\n", + " 3.390477 , 0.40496254, 0.14374796, -0.00899097, 0.75652575,\n", + " 0.64914 , 0.05783392, 0.05423374, 0. , 0. ],\n", + " dtype=float32)}, 'reward': 0.0}\n" + ] + } + ], + "source": [ + "number_of_samples = 3\n", + "rtx_loader = RTXLoader(os.path.expanduser(\"/home/lifelong/fog_x/datasets/0.1.0\"), split=f'train[:{number_of_samples}]')\n", + "hdf5_exporter = HDF5Exporter()\n", + "hdf5_exporter.export(rtx_loader,'please.hdf5')\n", + "hdf5_loader = HDF5Loader('/home/lifelong/fog_x/please.hdf5')\n", + "print(hdf5_loader.__len__())\n", + "for trajectory in hdf5_loader:\n", + " for step in trajectory:\n", + " print(step)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarking/misc/tf_to_hdf5.py b/benchmarking/misc/tf_to_hdf5.py new file mode 100644 index 0000000..b6755e8 --- /dev/null +++ b/benchmarking/misc/tf_to_hdf5.py @@ -0,0 +1,92 @@ +import os +import sys +import tensorflow as tf +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +# NOTE: this code assumes you have the paths: home/username/datasets/berkeley_autolab_ur5/TRAIN.tfrecord + +def parse_tfrecord(raw_record): + FLF = tf.io.FixedLenFeature + + features = {'steps': tf.io.VarLenFeature(tf.string)} + contexts = tf.io.parse_single_example(raw_record, features) + steps = tf.sparse.to_dense(contexts['steps']) + + steps_features = { + 'is_last': FLF([], tf.bool), + 'reward': FLF([], tf.float32), + 'is_first': FLF([], tf.bool), + 'action/gripper_closedness_action': FLF([], tf.float32), + 'action/rotation_delta': FLF([3], tf.float32), + 'action/terminate_episode': FLF([], tf.float32), + 'action/world_vector': FLF([3], tf.float32), + 'observation/natural_language_embedding': FLF([512], tf.float32), + 'observation/image': FLF([480, 640, 3], tf.uint8), + 'observation/hand_image': FLF([480, 640, 3], tf.uint8), + 'observation/natural_language_instruction': FLF([], tf.string), + 'observation/image_with_depth': FLF([480, 640, 1], tf.float32), + 'observation/robot_state': FLF([15], tf.float32), + 'is_terminal': FLF([], tf.bool), + } + return [tf.io.parse_single_example(s, steps_features) for s in steps ] + + +def tfrecord_to_df(tf_file): + dataset = tf.data.TFRecordDataset(tf_file) + all_records = [] + + for raw_record in dataset: + for step in parse_tfrecord(raw_record): + record = {} + for key, value in step.items(): + record[key] = value.numpy() + all_records.append(record) + + return pd.DataFrame(all_records) + + +def tf_to_parquet(in_dir, out_dir, prefix, suffix): + for i in range(1): + j = str(i).zfill(5) + + input_file = f"{in_dir}{prefix}{j}-of-00412" + output_file = f"{out_dir}{prefix}{j}{suffix}" + + print(f"Reading {input_file}") + + df = tfrecord_to_df(input_file) + table = pa.Table.from_pandas(df) + pq.write_table(table, output_file) + + print(f"Writing {output_file}") + + +DIR = os.path.expanduser("~") + "/datasets/" +in_dir = DIR + "berkeley_autolab_ur5/" +out_dir = DIR + "parquet/" +prefix = "berkeley_autolab_ur5-train.tfrecord-" +suffix = ".parquet" + +tf_to_parquet(in_dir, out_dir, prefix, suffix) + + +def disk_size(file): + return os.path.getsize(file) + +def tf_memory(tf_file): + dataset = tf.data.TFRecordDataset(tf_file) + return sys.getsizeof([record.numpy() for record in dataset]) + +def pq_memory(pq_file): + return pq.read_table(pq_file).nbytes + +MB = 1024 * 1024 +tf_file = in_dir + prefix + "00000-of-00412" +pq_file = out_dir + prefix + "00000.parquet" + +print(f"TFRecord size on disk: {disk_size(tf_file) / MB:.2f} MB") +print(f"Parquet size on disk: {disk_size(pq_file) / MB:.2f} MB") +print(f"TFRecord size in memory: {tf_memory(tf_file) / MB:.2f} MB") +print(f"Parquet size in memory: {pq_memory(pq_file) / MB:.2f} MB") \ No newline at end of file diff --git a/benchmarking/old_loaders.py b/benchmarking/old_loaders.py new file mode 100644 index 0000000..b96a5c0 --- /dev/null +++ b/benchmarking/old_loaders.py @@ -0,0 +1,322 @@ +import os +import av +import sys +import time +import pickle +import shutil +import numpy as np +from logging import getLogger + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = "2" +import tensorflow_datasets as tfds +import tensorflow as tf + +PATH = os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") +PATH += "/datasets/" + + +class BaseLoader(): + def __init__(self, path): + super(BaseLoader, self).__init__() + self.path = path + self.logger = getLogger(__name__) + + def __len__(self): + raise NotImplementedError + + def __iter___(self): + raise NotImplementedError + + +class BaseWriter(): + def __init__(self): + super(BaseWriter, self).__init__() + self.logger = getLogger(__name__) + + def export(self, loader, path): + raise NotImplementedError + + +class RTXLoader(BaseLoader): + def __init__(self, path, split): + super(RTXLoader, self).__init__(path) + + builder = tfds.builder_from_directory(path) + self.ds = builder.as_dataset(split) + + def __len__(self): + return len(self.ds) + + def __iter__(self): + return self.ds.__iter__() + + +class RTXWriter(BaseWriter): + def __init__(self): + super(RTXWriter, self).__init__() + self.wr = tf.io.TFRecordWriter + + def export(self, loader, path): + for i, data in enumerate(loader): + data_list = list(data["steps"].as_numpy_iterator()) + data_flat = flatten(data_list) + + with self.wr(f"{path}/output_{i}.tfrecord") as writer: + for dl in data_flat: + writer.write(dl) + + def export_temp(self, data_flat, path): + with self.wr(path + "/temp.tfrecord") as writer: + for dl in data_flat: + writer.write(dl) + + +N = 51 +MB = 1024 * 1024 + +def flatten(data): + if isinstance(data, dict): + data = list(data.values()) + if isinstance(data, list): + return [x for lst in data for x in flatten(lst)] + if isinstance(data, bytes): + return [data] + return [data.tobytes()] + + +def iterate_rtx(loader, writer, n): + read_time, write_time, data_size = 0, 0, 0 + + for i, data in enumerate(loader): + print(f"Measuring trajectory {i}") + + stop = time.time() + data_list = list(data["steps"].as_numpy_iterator()) + read_time += time.time() - stop + + name = "berkeley_autolab_ur5" + num = f"{str(i).zfill(5)}-of-00412" + data_size += os.path.getsize(f"{PATH}/{name}/{name}-train.tfrecord-{num}") + + data_flat = flatten(data_list) + stop = time.time() + writer.export_temp(data_flat, PATH) + write_time += time.time() - stop + + if i == n: break + return read_time, write_time, data_size / MB + + +rtx_loader = RTXLoader(PATH + "berkeley_autolab_ur5", f"train[:{N}]") +rtx_writer = RTXWriter() +rt, wt, mb = iterate_rtx(rtx_loader, rtx_writer, N) + +print(f"\nRTX Data: \nMem. size = {mb:.4f} MB; Num. traj = {N}") +print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") +print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") + + +class MVKWriter(BaseWriter): + def __init__(self): + super(MVKWriter, self).__init__() + + # Function to create a frame from numpy array + def create_frame(self, image, stream): + frame = av.VideoFrame.from_ndarray(np.array(image), format='rgb24') + frame.pict_type = 'NONE' + frame.time_base = stream.time_base + return frame + + # Function to create a frame from numpy array + def create_frame_depth(self, image, stream): + image_array = np.array(image) + # if float, convert to uint8 + if image_array.dtype == np.float32: + image_array = (image_array * 255).astype(np.uint8) + # if 3 dim, convert to 2 dim + if len(image_array.shape) == 3: + image_array = image_array[:,:,0] + frame = av.VideoFrame.from_ndarray(image_array, format='gray') + frame.pict_type = 'NONE' + frame.time_base = stream.time_base + return frame + + def export(self, loader, path): + # Create an output container + i = -1 + for traj_tensor in loader: + i += 1 + trajectory = dict(traj_tensor) + output = av.open(f'{path}/output_{i}.mkv', mode='w') + + # Define video streams (assuming images are 640x480 RGB) + video_stream_1 = output.add_stream('libx264', rate=1) + video_stream_1.width = 640 + video_stream_1.height = 480 + video_stream_1.pix_fmt = 'yuv420p' + + video_stream_2 = output.add_stream('libx264', rate=1) + video_stream_2.width = 640 + video_stream_2.height = 480 + video_stream_2.pix_fmt = 'yuv420p' + + # Define custom data stream for vectors + depth_stream = output.add_stream('libx264', rate=1) + data_stream = output.add_stream('rawvideo', rate=1) + + ts = 0 + # convert step data to stream + for step_tensor in trajectory["steps"]: + step = dict(step_tensor) + observation = step_tensor["observation"].copy() + observation.pop("image") + observation.pop("hand_image") + observation.pop("image_with_depth") + non_image_data_step = step.copy() + non_image_data_step["observation"] = observation + + non_image_data_bytes = pickle.dumps(non_image_data_step) + packet = av.Packet(non_image_data_bytes) + packet.stream = data_stream + packet.pts = ts + output.mux(packet) + + image = np.array(step["observation"]["image"]) + # Create a frame from the numpy array + frame = self.create_frame(image, video_stream_1) + frame.pts = ts + packet = video_stream_1.encode(frame) + output.mux(packet) + + hand_image =np.array(step["observation"]["hand_image"]) + # Create a frame from the numpy array + frame = self.create_frame(hand_image, video_stream_2) + frame.pts = ts + packet = video_stream_2.encode(frame) + output.mux(packet) + + # Create a frame from the numpy array + frame = self.create_frame_depth(step["observation"]["image_with_depth"], depth_stream) + # frame.pts = ts + # Encode the frame + packet = depth_stream.encode(frame) + # Write the packet to the output file + output.mux(packet) + ts += 1 + + # Flush the remaining frames + for packet in video_stream_1.encode(): + output.mux(packet) + + for packet in video_stream_2.encode(): + output.mux(packet) + + for packet in depth_stream.encode(): + output.mux(packet) + output.close() + + +class MKVLoader(BaseLoader): + def __init__(self, path): + super(MKVLoader, self).__init__(path) + self.files = [path + f for f in os.listdir(path) if f.endswith('.mkv')] + self.index = 0 + + def __len__(self): + return len(self.ds) + + def __getitem__(self, idx): + return idx + + def __iter__(self): + return self + + def __next__(self): + if self.index < len(self.files): + result = self.files[self.index] + self.index += 1 + return self._parse_mkv_file(result) + else: + raise StopIteration + + def _parse_mkv_file(self, filename): + print(filename) + input_container = av.open(filename) + + video_stream1 = input_container.streams.video[0] + video_stream1.thread_type = 'AUTO' + video_stream2 = input_container.streams.video[1] + video_stream2.thread_type = 'AUTO' + depth_stream = input_container.streams.video[2] + depth_stream.thread_type = 'AUTO' + data_stream = input_container.streams[3] + + decoded_stream_1 = [] + decoded_stream_2 = [] + decoded_stream_depth = [] + decoded_stream_data = [] + + pkt_counter = 0 + for packet in input_container.demux(video_stream1, video_stream2, depth_stream, data_stream): + pkt_counter += 1 + if packet.stream.index == video_stream1.index: + frame = packet.decode() + if frame: + for f in frame: + image = f.to_ndarray(format='rgb24') + decoded_stream_1.append(image) + elif packet.stream.index == video_stream2.index: + frame = packet.decode() + if frame: + for f in frame: + image = f.to_ndarray(format='rgb24') + decoded_stream_2.append(image) + elif packet.stream.index == depth_stream.index: + frame = packet.decode() + if frame: + for f in frame: + image = f.to_ndarray(format='gray') + decoded_stream_depth.append(image) + elif packet.stream.index == data_stream.index: + packet_in_bytes = bytes(packet) + if packet_in_bytes: + non_dict = pickle.loads(packet_in_bytes) + decoded_stream_data.append(non_dict) + else: + print("Unknown stream") + print(pkt_counter, len(decoded_stream_1), len(decoded_stream_2), len(decoded_stream_depth), len(decoded_stream_data)) + input_container.close() + + +def iterate_mkv(rtx_loader, mkv_writer, n): + read_time, write_time, data_size = 0, 0, 0 + path = PATH + "mkv_convert/" + + if len(os.listdir(path)) != 0: + raise SystemError(f"{path} must be empty.") + + stop = time.time() + mkv_writer.export(rtx_loader, path) + write_time += time.time() - stop + + mkv_loader = MKVLoader(path) + + stop = time.time() + for i, data in enumerate(mkv_loader): + print(f"Reading trajectory {i}") + if i == n: break + read_time += time.time() - stop + + for i in range(n): + data_size += os.path.getsize(f"{path}/output_{i}.mkv") + + return read_time, write_time, data_size / MB + + +mkv_writer = MVKWriter() +rt, wt, mb = iterate_mkv(rtx_loader, mkv_writer, N) + +print(f"\nMKV Data: \nMem. size = {mb:.4f} MB; Num. traj = {N}") +print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") +print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") \ No newline at end of file diff --git a/benchmarking/parquet.py b/benchmarking/parquet.py new file mode 100644 index 0000000..67045dc --- /dev/null +++ b/benchmarking/parquet.py @@ -0,0 +1,69 @@ +import os +import sys +import time +import polars as pl +import pandas as pd +import pyarrow.dataset as ds +import pyarrow.parquet as pq + +PATH = os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") +from fog_x.dataset import Dataset + + +PATH += "/datasets/" +NAME = "test_convert" +dataset = Dataset(name=NAME, path=PATH) + +N = 51 +MB = 1024 * 1024 +SDC = dataset.db_manager.step_data_connector + + +def measure_traj(read_func, write_func): + read_time, write_time, data_size = 0, 0, 0 + + for i in range(N): + print(f"Measuring trajectory {i}") + path = f"{SDC.path}/{NAME}_{i}-0.parquet" + + stop = time.time() + traj = read_func(path) + read_time += time.time() - stop + + data_size += os.path.getsize(path) + path = PATH + "/temp.parquet" + + stop = time.time() + write_func(path, traj) + write_time += time.time() - stop + + os.remove(path) + return read_time, write_time, data_size / MB + + +if __name__ == "__main__": + + fx_dict = {"name": "Fog-X", + "read_func": lambda path: SDC.tables[path.split("/")[-1].split("-")[0]].collect(), + "write_func": lambda path, data: data.write_parquet(path) + } + pl_dict = {"name": "Polars", + "read_func": lambda path: pl.read_parquet(path), + "write_func": lambda path, data: data.write_parquet(path) + } + pq_dict = {"name": "PyArrow", + "read_func": lambda path: pl.from_arrow(pq.read_table(path)), + "write_func": lambda path, data: pq.write_table(data.to_arrow(), path) + } + pd_dict = {"name": "Pandas", + "read_func": lambda path: pd.read_parquet(path), + "write_func": lambda path, data: data.to_parquet(path) + } + + for lib in [fx_dict, pl_dict, pq_dict, pd_dict]: + rt, wt, mb = measure_traj(lib["read_func"], lib["write_func"]) + + print(f"\n{lib['name']}: \nDisk size = {mb:.4f} MB; Num. traj = {N}") + print(f"Read: latency = {rt:.4f} s; throughput = {mb / rt :.4f} MB/s, {N / rt :.4f} traj/s") + print(f"Write: latency = {wt:.4f} s; throughput = {mb / wt :.4f} MB/s, {N / wt :.4f} traj/s") \ No newline at end of file diff --git a/benchmarking/plots/Disk size (MB).png b/benchmarking/plots/Disk size (MB).png new file mode 100644 index 0000000..d11752c Binary files /dev/null and b/benchmarking/plots/Disk size (MB).png differ diff --git a/benchmarking/plots/Read latency (s).png b/benchmarking/plots/Read latency (s).png new file mode 100644 index 0000000..c3f8627 Binary files /dev/null and b/benchmarking/plots/Read latency (s).png differ diff --git a/benchmarking/plots/Read throughput (traj_s).png b/benchmarking/plots/Read throughput (traj_s).png new file mode 100644 index 0000000..8db5c5a Binary files /dev/null and b/benchmarking/plots/Read throughput (traj_s).png differ diff --git a/benchmarking/plots/Write latency (s).png b/benchmarking/plots/Write latency (s).png new file mode 100644 index 0000000..88df18b Binary files /dev/null and b/benchmarking/plots/Write latency (s).png differ diff --git a/benchmarking/plots/Write throughput (traj_s).png b/benchmarking/plots/Write throughput (traj_s).png new file mode 100644 index 0000000..1440207 Binary files /dev/null and b/benchmarking/plots/Write throughput (traj_s).png differ diff --git a/benchmarking/plots/plot_notes.py b/benchmarking/plots/plot_notes.py new file mode 100644 index 0000000..059d5f9 --- /dev/null +++ b/benchmarking/plots/plot_notes.py @@ -0,0 +1,153 @@ +# TF_Record: +# Disk size (MB) = 8039.7179 MB ; Num. traj = 51 +# Read: latency = 15.5696 s; throughput = 516.3728 MB/s, 3.2756 (traj/s) +# Write: latency = 36.6353 s; throughput = 219.4528 MB/s, 1.3921 (traj/s) + +"""###""" + +# Fog-X: +# Disk size (MB) = 6707.3818 MB ; Num. traj = 51 +# Read: latency = 42.1416 s ; throughput = 159.1630 MB/s, 1.2102 (traj/s) +# Write: latency = 258.8635 s; throughput = 25.9109 MB/s, 0.1970 (traj/s) + +# Polars: +# Disk size (MB) = 6707.3818 MB ; Num. traj = 51 +# Read: latency = 22.9974 s ; throughput = 291.6581 MB/s, 2.2176 (traj/s) +# Write: latency = 241.4137 s; throughput = 27.7838 MB/s, 0.2113 (traj/s) + +# PyArrow: +# Disk size (MB) = 6707.3818 MB ; Num. traj = 51 +# Read: latency = 27.9808 s ; throughput = 239.7136 MB/s, 1.8227 (traj/s) +# Write: latency = 80.9986 s ; throughput = 82.8086 MB/s, 0.6296 (traj/s) + +# Pandas: +# Disk size (MB) = 6707.3818 MB ; Num. traj = 51 +# Read: latency = 49.1056 s ; throughput = 136.5909 MB/s, 1.0386 (traj/s) +# Write: latency = 103.6820 s; throughput = 64.6918 MB/s, 0.4919 (traj/s) + +"""###""" + +# Arrow_REG: +# Disk size (MB) = 14651.9697 MB ; Num. traj = 51 +# Read: latency = 39.5354 s; throughput = 370.6041 MB/s, 1.2900 (traj/s) +# Write: latency = 26.4765 s; throughput = 553.3952 MB/s, 1.9262 (traj/s) + +# Arrow_IPC: +# Disk size (MB) = 14651.9697 MB ; Num. traj = 51 +# Read: latency = 44.0609 s; throughput = 332.5391 MB/s, 1.1575 (traj/s) +# Write: latency = 25.1574 s; throughput = 582.4108 MB/s, 2.0272 (traj/s) + +# Feather_FTH: +# Disk size (MB) = 14651.9697 MB; Num. traj = 51 +# Read: latency = 47.5646 s; throughput = 308.0433 MB/s, 1.0722 (traj/s) +# Write: latency = 35.4595 s; throughput = 413.2030 MB/s, 1.4383 (traj/s) + +# Pandas_FTH: +# Disk size (MB) = 14651.9697 MB; Num. traj = 51 +# Read: latency = 26.6213 s; throughput = 550.3844 MB/s, 1.9158 (traj/s) +# Write: latency = 45.9015 s; throughput = 319.2046 MB/s, 1.1111 (traj/s) + +"""###""" + +# HDF5: +# Disk size (MB) = 44319.6689 MB ; Num. traj = 51 +# Read: latency = 191.5964 s; throughput = 231.3179 MB/s, 0.2662 (traj/s) +# Write: latency = 129.6112 s; throughput = 341.9432 MB/s, 0.3935 (traj/s) + +import os +import matplotlib.pyplot as plt +PATH = os.path.expanduser("~") + "/fog_x_fork/benchmarking/plots/" + +formats = { + "TFRecord": { + "Disk size (MB)": 8039.7179, + "Read latency (s)": 15.5696, + "Read throughput MB/s": 516.3728, + "Read throughput (traj/s)": 3.2756, + "Write latency (s)": 36.6353, + "Write throughput MB/s": 219.4528, + "Write throughput (traj/s)": 1.3921, + }, + "Parquet": { + "Disk size (MB)": 6707.3818, + "Read latency (s)": 27.9808, + "Read throughput MB/s": 239.7136, + "Read throughput (traj/s)": 1.8227, + "Write latency (s)": 80.9986, + "Write throughput MB/s": 82.8086, + "Write throughput (traj/s)": 0.6296, + }, + "Arrow": { + "Disk size (MB)": 14651.9697, + "Read latency (s)": 44.0609, + "Read throughput MB/s": 332.5391, + "Read throughput (traj/s)": 1.1575, + "Write latency (s)": 25.1574, + "Write throughput MB/s": 582.4108, + "Write throughput (traj/s)": 2.0272, + }, + "HDF5": { + "Disk size (MB)": 44319.6689, + "Read latency (s)": 191.5964, + "Read throughput MB/s": 231.3179, + "Read throughput (traj/s)": 0.2662, + "Write latency (s)": 129.6112, + "Write throughput MB/s": 341.9432, + "Write throughput (traj/s)": 0.3935, + }, + "MKV": { + "Disk size (MB)": 135.6294, + "Read latency (s)": 11.8973, + "Read throughput MB/s": 11.4, + "Read throughput (traj/s)": 4.2867, + "Write latency (s)": 205.9287, + "Write throughput MB/s": 0.6586, + "Write throughput (traj/s)": 0.2477, + }, +} +metrics = [ + "Disk size (MB)", + "Read latency (s)", + "Read throughput (traj/s)", + "Write latency (s)", + "Write throughput (traj/s)" +] +colors = { + "TFRecord": "cyan", + "Parquet": "pink", + "Arrow": "purple", + "HDF5": "green", + "MKV": "orange" +} +F_SIZE = 12 + +for metric in metrics: + plt.figure(figsize=(9, 6)) + labels = [] + values = [] + + for label, format in formats.items(): + labels.append(label) + values.append(format[metric]) + colist = [colors[l] for l in labels] + + bars = plt.bar(labels, values, color=colist) + + for bar in bars: + height = bar.get_height() + plt.text( + bar.get_x() + bar.get_width() / 2, + height, + f'{round(height, 2)} {metric.split(" ")[-1]}', + ha='center', + va='bottom', + fontsize=F_SIZE + ) + plt.title(metric) + plt.yticks(fontsize=F_SIZE) + plt.xticks(fontsize=F_SIZE) + plt.tight_layout() + + save_path = PATH + f"{metric.replace("/", "_")}.png" + plt.savefig(save_path) + plt.close() \ No newline at end of file diff --git a/benchmarking/tf_record.py b/benchmarking/tf_record.py new file mode 100644 index 0000000..70ae5de --- /dev/null +++ b/benchmarking/tf_record.py @@ -0,0 +1,53 @@ +import os +import sys +import time +import numpy as np + +PATH = "/shared/ryanhoque/datasets/datasets/" +os.path.expanduser("~") +sys.path.append(PATH + "/fog_x_fork") + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = "2" +import tensorflow as tf + +PATH += "/datasets/" +NAME = "berkeley_autolab_ur5" +MB = 1024 * 1024 +N = 51 + +def measure_traj(): + read_time, write_time, disk_size, memr_size = 0, 0, 0, 0 + + for i in range(N): + print(f"Measuring trajectory {i}") + path = f"{PATH}/{NAME}/{NAME}-train.tfrecord-{str(i).zfill(5)}-of-00412" + + TFRD = tf.data.TFRecordDataset + stop = time.time() + + for record in TFRD(path): + data = record.numpy() + memr_size += len(data) + read_time += time.time() - stop + + disk_size += os.path.getsize(path) + out_path = PATH + "/temp.tfrecord" + + for record in TFRD(path): + data = record.numpy() + stop = time.time() + + with tf.io.TFRecordWriter(out_path) as w: + w.write(data) + write_time += time.time() - stop + os.remove(out_path) + + return read_time, write_time, disk_size / MB, memr_size / MB + + +if __name__ == "__main__": + rt, wt, ds, ms = measure_traj() + + print(f"\nTF_Record: \nDisk size = {ds:.4f} MB; Mem. size = {ms:.4f} MB; Num. traj = {N}") + print(f"Read: latency = {rt:.4f} s; throughput = {ds / rt :.4f} MB/s, {N / rt :.4f} traj/s") + print(f"Write: latency = {wt:.4f} s; throughput = {ds / wt :.4f} MB/s, {N / wt :.4f} traj/s") \ No newline at end of file