diff --git a/drivers/slimbus/core.c b/drivers/slimbus/core.c
index ae1e248a8fb8a5992d0c4a24e8469113eeda7d55..58b63ae0e75a631b268008fc12be8d252d71ba23 100644
--- a/drivers/slimbus/core.c
+++ b/drivers/slimbus/core.c
@@ -326,8 +326,8 @@ void slim_report_absent(struct slim_device *sbdev)
 	mutex_lock(&ctrl->lock);
 	sbdev->is_laddr_valid = false;
 	mutex_unlock(&ctrl->lock);
-
-	ida_simple_remove(&ctrl->laddr_ida, sbdev->laddr);
+	if (!ctrl->get_laddr)
+		ida_simple_remove(&ctrl->laddr_ida, sbdev->laddr);
 	slim_device_update_status(sbdev, SLIM_DEVICE_STATUS_DOWN);
 }
 EXPORT_SYMBOL_GPL(slim_report_absent);