diff --git a/sale_order_line_cancel/models/sale_order.py b/sale_order_line_cancel/models/sale_order.py
index 870d5136b38..560ab58e1b7 100644
--- a/sale_order_line_cancel/models/sale_order.py
+++ b/sale_order_line_cancel/models/sale_order.py
@@ -16,7 +16,5 @@ def action_draft(self):
 
     def _action_cancel(self):
         res = super()._action_cancel()
-        orders = self.filtered(lambda s: s.state == "cancel")
-        for line in orders.order_line:
-            line.product_qty_canceled = line.product_uom_qty - line.qty_delivered
+        self.order_line._update_qty_canceled()
         return res
diff --git a/sale_order_line_cancel/models/sale_order_line.py b/sale_order_line_cancel/models/sale_order_line.py
index 119d6ab8cd5..8d1a978c20f 100644
--- a/sale_order_line_cancel/models/sale_order_line.py
+++ b/sale_order_line_cancel/models/sale_order_line.py
@@ -49,7 +49,8 @@ def _compute_product_qty_remains_to_deliver(self):
             line.product_qty_remains_to_deliver = qty_remaining
 
     def _get_moves_to_cancel(self):
-        return self.move_ids.filtered(lambda m: m.state not in ("done", "cancel"))
+        lines = self.filtered(lambda l: l.qty_delivered_method == "stock_move")
+        return lines.move_ids.filtered(lambda m: m.state not in ("done", "cancel"))
 
     def _check_moves_to_cancel(self, moves):
         """Override this method to add checks before cancel"""
diff --git a/sale_order_line_cancel/models/stock_move.py b/sale_order_line_cancel/models/stock_move.py
index 461150927f3..ced76a9afa5 100644
--- a/sale_order_line_cancel/models/stock_move.py
+++ b/sale_order_line_cancel/models/stock_move.py
@@ -10,11 +10,8 @@ class StockMove(models.Model):
     _inherit = "stock.move"
 
     def _action_cancel(self):
-        sale_moves = self.filtered(
-            lambda m: m.sale_line_id and m.state not in ("done", "cancel")
-        )
         res = super()._action_cancel()
-        sale_lines = sale_moves.filtered(lambda m: m.state == "cancel").sale_line_id
+        sale_lines = self._get_sale_lines_to_update_qty_canceled()
         sale_lines._update_qty_canceled()
         return res
 
@@ -25,3 +22,21 @@ def _action_done(self, cancel_backorder=False):
             # _action_cancel will not be triggered. Call it now
             self.sale_line_id._update_qty_canceled()
         return moves_todo
+
+    def _get_sale_lines_to_update_qty_canceled(self):
+        sale_lines = self.env["sale.order.line"]
+        for move in self:
+            if (
+                move.sale_line_id
+                and move._is_move_to_take_into_account_for_qty_canceled()
+            ):
+                sale_lines |= move.sale_line_id
+        return sale_lines
+
+    def _is_move_to_take_into_account_for_qty_canceled(self):
+        self.ensure_one()
+        return (
+            self.state == "cancel"
+            and self.sale_line_id
+            and self.picking_type_id.code == "outgoing"
+        )