diff --git a/.gitignore b/.gitignore
index fbeb6b7..4f2135d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,7 @@
*.lua
*.csv
*.dat
-*.pt
*.caffemodel
+*.pt
*.pbtxt
*.prototxt
diff --git a/butia_image2world/scripts/butia_image2world/image2world/image2world.py b/butia_image2world/scripts/butia_image2world/image2world/image2world.py
index c194a2e..911edf8 100755
--- a/butia_image2world/scripts/butia_image2world/image2world/image2world.py
+++ b/butia_image2world/scripts/butia_image2world/image2world/image2world.py
@@ -123,7 +123,6 @@ def __mountLutTable(self, camera_info):
def __detectionDescriptionProcessing(self, data, description2d, header):
center_x, center_y = description2d.bbox.center.x, description2d.bbox.center.y
bbox_size_x, bbox_size_y = description2d.bbox.size_x, description2d.bbox.size_y
- print(bbox_size_x, bbox_size_y)
if bbox_size_x == 0 or bbox_size_y == 0:
rospy.logwarn('BBox with zero size.')
return None
@@ -280,7 +279,14 @@ def __detectionDescriptionProcessing(self, data, description2d, header):
box_points = box_points[np.logical_and(np.all(box_points > min_bound, axis=1),
np.all(box_points < max_bound, axis=1))]
- box = o3d.geometry.AxisAlignedBoundingBox().create_from_points(o3d.utility.Vector3dVector(box_points))
+ box_pcd = o3d.geometry.PointCloud()
+ box_pcd.points = o3d.utility.Vector3dVector(box_points)
+
+ box_pcd, _ = box_pcd.remove_statistical_outlier(20, 2.0)
+
+ # TODO: add a clustering algorithm and pick the closest cluster
+
+ box = o3d.geometry.AxisAlignedBoundingBox().create_from_points(box_pcd.points)
else:
box = o3d.geometry.AxisAlignedBoundingBox(min_bound, max_bound)
@@ -354,7 +360,6 @@ def __createDescription3D(self, source_data, description2d : Description2D, head
description3D : Description3D = self.DESCRIPTION_PROCESSING_ALGORITHMS[description2d.type](source_data, description2d, header)
description3D.bbox2D = description2d.bbox
description3D.class_num = description2d.class_num
- print(description3D.class_num)
return description3D
else:
return None
diff --git a/butia_recognition/config/yolov8_object_recognition.yaml b/butia_recognition/config/yolov8_object_recognition.yaml
index eb9a153..67ff2ab 100644
--- a/butia_recognition/config/yolov8_object_recognition.yaml
+++ b/butia_recognition/config/yolov8_object_recognition.yaml
@@ -10,90 +10,90 @@ classes_by_category:
Animals: ['bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'teddybear']
Household: ['backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'vase', 'scissors', 'book', 'toothbrush']
all_classes:
-- 'person'
-- 'bicycle'
-- 'car'
-- 'motorcycle'
-- 'airplane'
-- 'bus'
-- 'train'
-- 'truck'
-- 'boat'
-- 'traffic light'
-- 'fire hydrant'
-- 'stop sign'
-- 'parking meter'
-- 'bench'
-- 'bird'
-- 'cat'
-- 'dog'
-- 'horse'
-- 'sheep'
-- 'cow'
-- 'elephant'
-- 'bear'
-- 'zebra'
-- 'giraffe'
-- 'backpack'
-- 'umbrella'
-- 'handbag'
-- 'tie'
-- 'suitcase'
-- 'frisbee'
-- 'skis'
-- 'snowboard'
-- 'sports ball'
-- 'kite'
-- 'baseball bat'
-- 'baseball glove'
-- 'skateboard'
-- 'surfboard'
-- 'tennis racket'
-- 'bottle'
-- 'wine glass'
-- 'cup'
-- 'fork'
-- 'knife'
-- 'spoon'
-- 'bowl'
-- 'banana'
-- 'apple'
-- 'sandwich'
-- 'orange'
-- 'broccoli'
-- 'carrot'
-- 'hot dog'
-- 'pizza'
-- 'donut'
-- 'cake'
-- 'chair'
-- 'couch'
-- 'potted plant'
-- 'bed'
-- 'dining table'
-- 'toilet'
-- 'tv'
-- 'laptop'
-- 'mouse'
-- 'remote'
-- 'keyboard'
-- 'cell phone'
-- 'microwave'
-- 'oven'
-- 'toaster'
-- 'sink'
-- 'refrigerator'
-- 'book'
-- 'clock'
-- 'vase'
-- 'scissors'
-- 'teddy bear'
-- 'hair drier'
-- 'toothbrush'
+ - 'person'
+ - 'bicycle'
+ - 'car'
+ - 'motorcycle'
+ - 'airplane'
+ - 'bus'
+ - 'train'
+ - 'truck'
+ - 'boat'
+ - 'traffic light'
+ - 'fire hydrant'
+ - 'stop sign'
+ - 'parking meter'
+ - 'bench'
+ - 'bird'
+ - 'cat'
+ - 'dog'
+ - 'horse'
+ - 'sheep'
+ - 'cow'
+ - 'elephant'
+ - 'bear'
+ - 'zebra'
+ - 'giraffe'
+ - 'backpack'
+ - 'umbrella'
+ - 'handbag'
+ - 'tie'
+ - 'suitcase'
+ - 'frisbee'
+ - 'skis'
+ - 'snowboard'
+ - 'sports ball'
+ - 'kite'
+ - 'baseball bat'
+ - 'baseball glove'
+ - 'skateboard'
+ - 'surfboard'
+ - 'tennis racket'
+ - 'bottle'
+ - 'wine glass'
+ - 'cup'
+ - 'fork'
+ - 'knife'
+ - 'spoon'
+ - 'bowl'
+ - 'banana'
+ - 'apple'
+ - 'sandwich'
+ - 'orange'
+ - 'broccoli'
+ - 'carrot'
+ - 'hot dog'
+ - 'pizza'
+ - 'donut'
+ - 'cake'
+ - 'chair'
+ - 'couch'
+ - 'potted plant'
+ - 'bed'
+ - 'dining table'
+ - 'toilet'
+ - 'tv'
+ - 'laptop'
+ - 'mouse'
+ - 'remote'
+ - 'keyboard'
+ - 'cell phone'
+ - 'microwave'
+ - 'oven'
+ - 'toaster'
+ - 'sink'
+ - 'refrigerator'
+ - 'book'
+ - 'clock'
+ - 'vase'
+ - 'scissors'
+ - 'teddy bear'
+ - 'hair drier'
+ - 'toothbrush'
max_sizes:
- - [0.4, 2.5, 0.5]
+ - [0.05, 0.05, 0.05]
model_file: yolov8n.pt
diff --git a/butia_recognition/config/yolov8_object_recognition_custom.yaml b/butia_recognition/config/yolov8_object_recognition_custom.yaml
new file mode 100644
index 0000000..c6bfae3
--- /dev/null
+++ b/butia_recognition/config/yolov8_object_recognition_custom.yaml
@@ -0,0 +1,59 @@
+threshold: 0.3
+classes_by_category:
+ Drinks: ['nescau', 'kuat', 'cononut water', 'fanta']
+ Cleaning supplies: ['detergent', 'sponge', 'cloth']
+ Pantry items: ['gelatin', 'mustard', 'shoyo', 'sauce', 'tea']
+ Fruits: ['apple', 'pear', 'tangerine']
+ Snacks: ['treloso', 'chocolate', 'peanut']
+
+all_classes:
+ - 'apple'
+ - 'chocolate'
+ - 'cloth'
+ - 'cononut water'
+ - 'detergent'
+ - 'fanta'
+ - 'gelatin'
+ - 'kuat'
+ - 'mustard'
+ - 'nescau'
+ - 'peanut'
+ - 'pear'
+ - 'sauce'
+ - 'shoyo'
+ - 'sponge'
+ - 'tangerine'
+ - 'tea'
+ - 'treloso'
+
+max_sizes:
+ - [0.05, 0.05, 0.05]
+
+model_file: yolov8-larc2023.pt
+
+subscribers:
+
+ queue_size: 1
+ exact_time: false
+ slop: 0.2
+
+ image_rgb: /butia_vision/bvb/image_rgb
+ camera_info: /butia_vision/bvb/camera_info
+ image_depth: /butia_vision/bvb/image_depth
+
+publishers:
+
+ object_recognition:
+ topic: /butia_vision/br/object_recognition
+ queue_size: 1
+
+servers:
+
+ list_classes:
+ service: /butia_vision/br/object_recognition/list_classes
+
+ start:
+ service: /butia_vision/br/object_recognition/start
+
+ stop:
+ service: /butia_vision/br/object_recognition/stop
diff --git a/butia_recognition/launch/yolov8_object_recognition.launch b/butia_recognition/launch/yolov8_object_recognition.launch
index 45ddd50..fdc0f7f 100644
--- a/butia_recognition/launch/yolov8_object_recognition.launch
+++ b/butia_recognition/launch/yolov8_object_recognition.launch
@@ -3,7 +3,7 @@
-
+
diff --git a/butia_recognition/scripts/butia_recognition/yolo_tracker_recognition/yolo_tracker_recognition.py b/butia_recognition/scripts/butia_recognition/yolo_tracker_recognition/yolo_tracker_recognition.py
index 77816c9..73b4260 100755
--- a/butia_recognition/scripts/butia_recognition/yolo_tracker_recognition/yolo_tracker_recognition.py
+++ b/butia_recognition/scripts/butia_recognition/yolo_tracker_recognition/yolo_tracker_recognition.py
@@ -125,6 +125,7 @@ def callback(self, *args):
img = ros_numpy.numpify(img)
debug_img = deepcopy(img)
+ debug_img = cv.cvtColor(debug_img, cv.COLOR_BGR2RGB)
results = None
bboxs = None
diff --git a/butia_recognition/scripts/butia_recognition/yolov8_recognition/yolov8_recognition.py b/butia_recognition/scripts/butia_recognition/yolov8_recognition/yolov8_recognition.py
index 8af28e4..8198277 100755
--- a/butia_recognition/scripts/butia_recognition/yolov8_recognition/yolov8_recognition.py
+++ b/butia_recognition/scripts/butia_recognition/yolov8_recognition/yolov8_recognition.py
@@ -13,7 +13,7 @@
from geometry_msgs.msg import Vector3
from butia_vision_msgs.msg import Description2D, Recognitions2D
import torch
-
+import rospkg
class YoloV8Recognition(BaseRecognition):
def __init__(self, state=True):
@@ -41,7 +41,7 @@ def serverStop(self, req):
return super().serverStop(req)
def loadModel(self):
- self.model = YOLO("yolov8n.pt")
+ self.model = YOLO(self.model_file)
self.model.conf = self.threshold
print('Done loading model!')
@@ -74,8 +74,7 @@ def callback(self, *args):
description_header = img_rgb.header
description_header.seq = 0
- results = self.model.predict(cv_img)
- debug_img = cv_img
+ results = self.model.predict(cv_img[:, :, ::-1], conf=self.threshold, verbose=True)
boxes_ = results[0].boxes.cpu().numpy()
if len(results[0].boxes):
@@ -87,14 +86,14 @@ def callback(self, *args):
continue
label_class = self.all_classes[int(box.cls)]
-
+ max_size = self.max_sizes[label_class]
description = Description2D()
description.header = copy(description_header)
description.type = Description2D.DETECTION
description.id = description.header.seq
description.score = float(box.conf)
- description.max_size = Vector3(*[0.05, 0.05, 0.05])
+ description.max_size = Vector3(*max_size)
size = int(xyxy_box[2] - xyxy_box[0]), int(xyxy_box[3] - xyxy_box[1])
description.bbox.center.x = int(xyxy_box[0]) + int(size[0]/2)
description.bbox.center.y = int(xyxy_box[1]) + int(size[1]/2)
@@ -116,7 +115,7 @@ def callback(self, *args):
description.label = index + '/' + label_class if index is not None else label_class
objects_recognition.descriptions.append(description)
- debug_img = results[0].plot()
+ debug_img = results[0].plot()[:, :, ::-1]
description_header.seq += 1
self.debug_publisher.publish(ros_numpy.msgify(Image, debug_img, 'rgb8'))
@@ -127,7 +126,7 @@ def callback(self, *args):
if len(people_recognition.descriptions) > 0:
self.people_detection_publisher.publish(people_recognition)
else:
- debug_img = results[0].plot()
+ debug_img = results[0].plot()[:, :, ::-1]
self.debug_publisher.publish(ros_numpy.msgify(Image, debug_img, 'rgb8'))
def readParameters(self):
@@ -145,7 +144,21 @@ def readParameters(self):
self.all_classes = list(rospy.get_param("~all_classes", []))
self.classes_by_category = dict(rospy.get_param("~classes_by_category", {}))
- self.model_file = rospy.get_param("~model_file", "yolov8_lab_objects.pt")
+
+ r = rospkg.RosPack()
+ r.list()
+ self.model_file = r.get_path("butia_recognition") + "/weigths/" + rospy.get_param("~model_file", "yolov8n.pt")
+
+ max_sizes = rospy.get_param("~max_sizes", [[0.05, 0.05, 0.05]])
+
+ if len(max_sizes) == 1:
+ max_sizes = [max_sizes[0] for _ in self.all_classes]
+
+ if len(max_sizes) != len(self.all_classes):
+ rospy.logerr('Wrong definition of max_sizes in yaml of recognition node.')
+ assert False
+
+ self.max_sizes = dict([(self.all_classes[i], max_sizes[i]) for i in range(len(max_sizes))])
super().readParameters()
diff --git a/butia_recognition/weigths/tmpoxfg42bv b/butia_recognition/weigths/tmpoxfg42bv
deleted file mode 100644
index e816362..0000000
Binary files a/butia_recognition/weigths/tmpoxfg42bv and /dev/null differ
diff --git a/butia_vision_msgs b/butia_vision_msgs
index 8e82c40..632689a 160000
--- a/butia_vision_msgs
+++ b/butia_vision_msgs
@@ -1 +1 @@
-Subproject commit 8e82c409ef58364ee6d5bee71c0ea5f01f203faa
+Subproject commit 632689a56399ff2a2288fa3f16aca98ccd7a468f