add masks

.vscode/settings.json

@@ -11,6 +11,7 @@
     "*.dae": "xml",
     "*.pyx": "python",
     "*.pxd": "cpp",
+    "*.srdf": "xml",
     "*.sdf": "xml",
     "array": "cpp",
     "bitset": "cpp",
@@ -92,6 +93,13 @@
     "variant": "cpp",
     "mixinvector": "cpp",
     "fast_back_stack": "cpp",
-    "nonlinearoptimization": "cpp"
+    "nonlinearoptimization": "cpp",
+    "hash_map": "cpp",
+    "slist": "cpp",
+    "valarray": "cpp",
+    "bufferobject": "cpp",
+    "image": "cpp",
+    "texture": "cpp",
+    "buffered_value": "cpp"
   }
 }

CMakeLists.txt

@@ -4,7 +4,7 @@ project(BundleTrack)
 
 set(CMAKE_BUILD_TYPE Release)
 add_compile_options(-std=c++14 -fpermissive -fPIC -g)
-set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} -Xcompiler;-fPIC;-gencode arch=compute_30,code=sm_30;-gencode arch=compute_75,code=sm_75;-O3;-std=c++11;-use_fast_math;--default-stream per-thread)
+set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} -Xcompiler;-fPIC;-gencode arch=compute_30,code=sm_30;-gencode arch=compute_61,code=sm_61;-gencode arch=compute_75,code=sm_75;-O3;-std=c++11;-use_fast_math;--default-stream per-thread)
 
 if (${CMAKE_BUILD_TYPE} STREQUAL "Release")
   set(COMPILE_OPTIONS -fopenmp -march=native -msse -msse2 -msse3 -msse4 -msse4.2 -mavx2 -ftree-vectorize)
@@ -16,6 +16,8 @@ endif (${CMAKE_BUILD_TYPE} STREQUAL "Release")
 # add_definitions(-DPRINT_RESIDUALS_DENSE=1)
 # add_definitions(-DPRINT_RESIDUALS_SPARSE=1)
 
+add_definitions(-DNO_OPENCV_CUDA=0)
+
 find_package(Boost REQUIRED COMPONENTS system program_options)
 find_package(PCL 1.9 REQUIRED)
 find_package(OpenCV REQUIRED)

README.md

@@ -54,9 +54,8 @@ For convenience of benchmarking and making plots, results of pose outputs can be
 - For YCBInEOAT: https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/ycbineoat_ours_results.tar.gz
 
 
-# Getting Started
 
-## Setup
+# Setup
 For the environment setup, it's strongly recommended to use our provided docker environment (setting up from scratch is very complicated and not supported in this repo). For this, you don't have to know how docker works. Only some basic commands are needed and will be provided in the below steps.
 
 - Install docker (https://docs.docker.com/get-docker/).
@@ -67,10 +66,6 @@ For the environment setup, it's strongly recommended to use our provided docker
   docker pull wenbowen123/lf-net-release-env:latest
   ```
 
-- Download neural network weights
-  - Download https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/indoor.tar.gz, extract and put it under `lf-net-release/release/models`, so it'll be `lf-net-release/release/models/indoor/...`
-  - Download https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/pretrained.tar.gz, extract and put it under `./transductive-vos.pytorch`, so it'll be `./transductive-vos.pytorch/pretrained/...`
-
 - Edit the docker/run_container.sh, update the paths of `BUNDLETRACK_DIR`, `NOCS_DIR` and `YCBINEOAT_DIR`
 
 - Run `bash docker/run_container.sh`
@@ -80,26 +75,52 @@ For the environment setup, it's strongly recommended to use our provided docker
 - `rm -rf build && mkdir build && cd build && cmake .. && make`
 
 
+# Data
+Depending on what you want to run, download those data that are neccessary.
+- [Download weights of feature detection network](https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/indoor.tar.gz), extract and put it under `lf-net-release/release/models`, so it'll be `BundleTrack/lf-net-release/release/models/indoor`
+- [Download weights of video segmentation network](https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/pretrained.tar.gz), extract and put it under `./transductive-vos.pytorch`, so it'll be `BundleTrack/transductive-vos.pytorch/pretrained`
+- [Download our precomputed masks](https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/masks.tar.gz), extract and put in the repo so it becomes `BundleTrack/masks`
 
-## Run predictions on NOCS
-- Download NOCS Dataset from http://download.cs.stanford.edu/orion/nocs/real_test.zip, then put it under a folder named "NOCS". And download the addon from https://drive.google.com/file/d/1BknMsoRKRV-nhLDpozog2Lrz1OWTKqkM/view?usp=sharing. Finally unzip the files and make sure the path structure is like this:
+- [Download NOCS Dataset](https://github.com/hughw19/NOCS_CVPR2019), then put it under a folder named "NOCS". [Download the converted ground-truth text pose files](https://archive.cs.rutgers.edu/archive/a/2021/pracsys/2021_iros_bundletrack/real_test_text.tar.gz). And [download the addon](https://drive.google.com/file/d/1BknMsoRKRV-nhLDpozog2Lrz1OWTKqkM/view?usp=sharing). Finally unzip the files and make sure the path structure is like this:
 
   ```
   NOCS
   ├── NOCS-REAL275-additional
-  └── real_test
+  ├── real_test
+  ├── gts
+  |     └── real_test_text
+  └── obj_models
+  ```
+
+- [Download YCBInEOAT](https://archive.cs.rutgers.edu/archive/a/2020/pracsys/Bowen/iros2020/YCBInEOAT/) to make it like
   ```
+  YCBInEOAT
+      ├── bleach0
+      |       ├──annotated_poses
+      |       ├──depth
+      |       ├──depth_filled
+      |       ├──gt_mask
+      |       ├──masks
+      |       ├──masks_vis
+      |       ├──rgb
+      |       ├──cam_K.txt
+      |       ├──......
+      └── ....
+  ```
+
+- [Download YCB Objects](http://ycb-benchmarks.s3-website-us-east-1.amazonaws.com/)
 
+# Run predictions on NOCS
 - Open a separate terminal and run
 
   ```
   bash lf-net-release/docker/run_container.sh
   cd [PATH_TO_BUNDLETRACK]
-  python lf-net-release/run_server.py
+  cd lf-net-release && python run_server.py
   ```
 
 
-- Go back to the terminal where you launched the bundletrack docker in above
+- Go back to the terminal where you launched the bundletrack docker in above and run below. The output will be saved to `debug_dir` specified in config file. By default it's `/tmp/BundleTrack/`. For more detailed logs, change `LOG` to 2 or higher in `config_nocs.yml`.
 
   ```
   python scripts/run_nocs.py --nocs_dir [PATH_TO_NOCS] --scene_id 1 --port 5555 --model_name can_arizona_tea_norm
@@ -108,39 +129,39 @@ For the environment setup, it's strongly recommended to use our provided docker
 - Finally, the results will be saved in `/tmp/BundleTrack/`
 
 
-- For evaluating on the entire NOCS Dataset (by default this will add perturbation to the initial ground-truth pose), run
+- For evaluating on the entire NOCS Dataset, run (**NOTE that this will add noise to perturb the initial ground-truth pose for evaluation as explained in the paper. If you want to see how BundleTrack actually performs, run the above section**)
   ```
   python scripts/eval_nocs.py --nocs_dir [PAHT TO NOCS]  --results_dir [PATH TO THE RUNNING OUTPUTS]
   ```
 
 
-## Run predictions on YCBInEOAT
+# Run predictions on YCBInEOAT
 
-- Download YCBInEOAT from https://archive.cs.rutgers.edu/archive/a/2020/pracsys/Bowen/iros2020/YCBInEOAT/
+- Change the `model_name` and `model_dir` in `config_ycbineoat.yml` to the path to the .obj file (e.g. For folder `bleach0`, the model_name is `021_bleach_cleanser`, and model_dir is `[Your path to YCB Objects]/021_bleach_cleanser/textured_simple.obj`)
 
 - Open a separate terminal and run
 
   ```
   bash lf-net-release/docker/run_container.sh
   cd [PATH_TO_BUNDLETRACK]
-  python lf-net-release/run_server.py
+  cd lf-net-release && python run_server.py
   ```
 
-- Go back to the terminal where you launched the bundletrack docker in above, and run
+- Go back to the terminal where you launched the bundletrack docker in above, and run below. The output will be saved to `debug_dir` specified in config file. By default it's `/tmp/BundleTrack/`
 
   ```
   python scripts/run_ycbineoat.py --data_dir [PATH_TO_YCBInEOAT] --port 5555 --model_name [The YCB object's name, e.g. 021_bleach_cleanser]
   ```
 
-- Finally, the results will be saved in `/tmp/BundleTrack/`
+- Finally, the results will be saved in `/tmp/BundleTrack/`. For more detailed logs, change `LOG` to 2 or higher in `config_ycbineoat.yml`.
 
 - For evaluating on the entire YCBInEOAT Dataset, run
   ```
   python scripts/eval_nocs.py --ycbineoat_dir [PAHT TO YCBINEOAT] --ycb_model_dir [YCB MODELS FOLDER] --results_dir [PATH TO THE RUN OUTPUTS]
   ```
 
 
-## Run predictions on your own RGBD data
+# Run predictions on your own RGBD data
 
 - Download YCBInEOAT, if you haven't done so in above.
 
@@ -149,7 +170,7 @@ For the environment setup, it's strongly recommended to use our provided docker
   ```
   bash lf-net-release/docker/run_container.sh
   cd [PATH_TO_BUNDLETRACK]
-  python lf-net-release/run_server.py
+  cd lf-net-release && python run_server.py
   ```
 
 - Prepare segmentation masks. In YCBInEOAT Dataset, we computed masks from robotic arm forward kinematics. If your scene is not too complicated similar to NOCS Dataset, you can run the video segmentation network to get masks as below:
@@ -167,7 +188,7 @@ For the environment setup, it's strongly recommended to use our provided docker
     └── cam_K.txt
     ```
 
-- Go back to the terminal where you launched the bundletrack docker, run
+- Go back to the terminal where you launched the bundletrack docker, run below. The output will be saved to `debug_dir` specified in config file. By default it's `/tmp/BundleTrack/`
   ```
   python scripts/run_ycbineoat.py --data_dir [PATH TO YOUR FOLDER ABOVE] --port 5555
   ```

config_nocs.yml

@@ -1,5 +1,5 @@
-data_dir: /media/bowen/e25c9489-2f57-42dd-b076-021c59369fec/DATASET/NOCS/real_test/scene_4/
-mask_dir: /media/bowen/e25c9489-2f57-42dd-b076-021c59369fec/github/category_pick_place/BundleTrack/masks/transductive_vos.nocs.maskrcnn_realtrain/scene_2_model_bottle_red_stanford_norm  # Path to segmentation masks
+data_dir: /media/bowen/e25c9489-2f57-42dd-b076-021c59369fec/DATASET/NOCS/real_test/scene_2/
+mask_dir: /home/bowen/debug/BundleTrack/masks/transductive_vos.nocs.maskrcnn_realtrain/scene_2_model_bottle_red_stanford_norm  # Path to segmentation masks
 use_6pack_datalist: True
 
 model_name: bottle_red_stanford_norm

docker/run_container.sh

@@ -1,6 +1,6 @@
-BUNDLETRACK_DIR="/media/bowen/e25c9489-2f57-42dd-b076-021c59369fec/github/BundleTrack_release"
+BUNDLETRACK_DIR="/home/bowen/debug/BundleTrack"
 NOCS_DIR="/media/bowen/e25c9489-2f57-42dd-b076-021c59369fec/DATASET/NOCS"
-YCBINEOAT_DIR="/media/bowen/e25c9489-2f57-42dd-b076-021c59369fec/catkin_ws/src/iros20_dataset/video_rosbag/IROS_SELECTED/FINISHED_LABEL.iros_submission_version"
+YCBINEOAT_DIR="/iros_submission_version"
 echo "BUNDLETRACK_DIR $BUNDLETRACK_DIR"
 echo "NOCS_DIR $NOCS_DIR"
 echo "YCBINEOAT_DIR $YCBINEOAT_DIR"

src/Bundler.cpp

@@ -376,49 +376,37 @@ void Bundler::saveNewframeResult()
   ff<<std::setprecision(10)<<ob_in_cam<<std::endl;
   ff.close();
 
-  PointCloudRGBNormal::Ptr cloud_world(new PointCloudRGBNormal);
-  pcl::copyPointCloud(*_newframe->_cloud,*cloud_world);
-  Utils::passFilterPointCloud(cloud_world,cloud_world,"z",0.1,2.0);
-  pcl::io::savePLYFile(out_dir+"cloud_in_cam.ply",*cloud_world);
-  pcl::transformPointCloudWithNormals(*cloud_world,*cloud_world,cur_in_model);
-  pcl::io::savePLYFile(out_dir+"cloud_world.ply",*cloud_world);
-  pcl::transformPointCloudWithNormals(*_newframe->_cloud_down,*cloud_world,cur_in_model);
-  pcl::io::savePLYFile(out_dir+"cloud_world_downsample.ply",*cloud_world);
-  PointCloudRGBNormal::Ptr cloud_raw_depth(new PointCloudRGBNormal);
-  Utils::convert3dOrganizedRGB(_newframe->_depth_raw, _newframe->_color, _newframe->_K, cloud_raw_depth);
-  Utils::calNormalIntegralImage(cloud_raw_depth,-1,0.02,10,true);
-  Utils::passFilterPointCloud(cloud_raw_depth,cloud_raw_depth,"z",0.1,2.0);
-  pcl::transformPointCloudWithNormals(*cloud_raw_depth, *cloud_raw_depth, cur_in_model);
-  pcl::io::savePLYFile(out_dir+"cloud_raw_depth.ply",*cloud_raw_depth);
-
-  cv::Mat color_viz = _newframe->_vis.clone();
-  PointCloudRGBNormal::Ptr cur_model(new PointCloudRGBNormal);
-  pcl::transformPointCloudWithNormals(*(_data_loader->_real_model),*cur_model,ob_in_cam);
-  for (int h=0;h<_newframe->_H;h++)
+  if ((*yml)["LOG"].as<int>()>0)
   {
-    for (int w=0;w<_newframe->_W;w++)
+    cv::Mat color_viz = _newframe->_vis.clone();
+    PointCloudRGBNormal::Ptr cur_model(new PointCloudRGBNormal);
+    pcl::transformPointCloudWithNormals(*(_data_loader->_real_model),*cur_model,ob_in_cam);
+    for (int h=0;h<_newframe->_H;h++)
     {
-      auto &bgr = color_viz.at<cv::Vec3b>(h,w);
-      if (_newframe->_fg_mask.at<uchar>(h,w)==0)
+      for (int w=0;w<_newframe->_W;w++)
       {
-        for (int i=0;i<3;i++)
+        auto &bgr = color_viz.at<cv::Vec3b>(h,w);
+        if (_newframe->_fg_mask.at<uchar>(h,w)==0)
         {
-          bgr[i] = (uchar)bgr[i]*0.2;
+          for (int i=0;i<3;i++)
+          {
+            bgr[i] = (uchar)bgr[i]*0.2;
+          }
         }
       }
     }
+    Utils::drawProjectPoints(cur_model,_data_loader->_K,color_viz);
+    cv::putText(color_viz,_newframe->_id_str,{5,30},cv::FONT_HERSHEY_PLAIN,2,{255,0,0},1,8,false);
+    // cv::imshow("color_viz",color_viz);
+    // cv::waitKey(1);
+    cv::imwrite(debug_dir+"/color_viz/"+_newframe->_id_str+"_color_viz.jpg",color_viz,{CV_IMWRITE_JPEG_QUALITY, 80});
+    cv::imwrite(out_dir+"color_viz.jpg",color_viz,{CV_IMWRITE_JPEG_QUALITY, 80});
+
+    const std::string raw_dir = debug_dir+"/color_raw/";
+    if (!boost::filesystem::exists(raw_dir))
+    {
+      system(std::string("mkdir -p "+raw_dir).c_str());
+    }
+    cv::imwrite(raw_dir+_newframe->_id_str+"_color_raw.png",_newframe->_color);
   }
-  Utils::drawProjectPoints(cur_model,_data_loader->_K,color_viz);
-  cv::putText(color_viz,_newframe->_id_str,{5,30},cv::FONT_HERSHEY_PLAIN,2,{255,0,0},1,8,false);
-  cv::imshow("color_viz",color_viz);
-  cv::waitKey(1);
-  cv::imwrite(debug_dir+"/color_viz/"+_newframe->_id_str+"_color_viz.jpg",color_viz,{CV_IMWRITE_JPEG_QUALITY, 80});
-  cv::imwrite(out_dir+"color_viz.jpg",color_viz,{CV_IMWRITE_JPEG_QUALITY, 80});
-
-  const std::string raw_dir = debug_dir+"/color_raw/";
-  if (!boost::filesystem::exists(raw_dir))
-  {
-    system(std::string("mkdir -p "+raw_dir).c_str());
-  }
-  cv::imwrite(raw_dir+_newframe->_id_str+"_color_raw.png",_newframe->_color);
 }

src/DataLoader.cpp

@@ -70,8 +70,6 @@ DataLoaderNOCS::DataLoaderNOCS(std::shared_ptr<YAML::Node> yml1) : DataLoaderBas
     boost::replace_all(_model_dir, std::string(what[0]), "NOCS/obj_models/real_test/"+_model_name+".obj");
     _gt_dir = data_dir;
     boost::replace_all(_gt_dir, std::string(what[0]), "NOCS/gts/real_test_text/scene_" + std::to_string(_scene_id) + "/model_" + _model_name + "/");
-    _scale_dir = data_dir;
-    boost::replace_all(_scale_dir, std::string(what[0]), "NOCS/NOCS-REAL275-additional/model_scales/"+_model_name+".txt");
   }
 
   _K<<591.0125, 0, 322.525,
@@ -146,17 +144,6 @@ DataLoaderNOCS::DataLoaderNOCS(std::shared_ptr<YAML::Node> yml1) : DataLoaderBas
 
   }
 
-
-  Eigen::Matrix<float,8,3> vertices;
-  Utils::parseMatrixTxt<8,3>(_scale_dir,vertices);
-  vertices /= 1000.0;
-  float xmin = vertices.col(0).minCoeff();
-  float xmax = vertices.col(0).maxCoeff();
-  float ymin = vertices.col(1).minCoeff();
-  float ymax = vertices.col(1).maxCoeff();
-  float zmin = vertices.col(2).minCoeff();
-  float zmax = vertices.col(2).maxCoeff();
-
 }
 
 DataLoaderNOCS::~DataLoaderNOCS()

src/DataLoader.h

@@ -68,7 +68,6 @@ class DataLoaderNOCS : public DataLoaderBase
 {
 public:
   EIGEN_MAKE_ALIGNED_OPERATOR_NEW;
-  std::string _scale_dir;
 
 public:
   DataLoaderNOCS(std::shared_ptr<YAML::Node> yml1);