diff --git a/pyvirtualdisplay/abstractdisplay.py b/pyvirtualdisplay/abstractdisplay.py
index 25fcf3d..bed35ed 100644
--- a/pyvirtualdisplay/abstractdisplay.py
+++ b/pyvirtualdisplay/abstractdisplay.py
@@ -99,6 +99,7 @@ def __init__(
         self._reset_global_env = False
         self._pipe_wfd = None
         self._retries_current = 0
+        self._display_to_rm = None
 
         helptext = get_helptext(program)
         self._has_displayfd = "-displayfd" in helptext
@@ -230,6 +231,7 @@ def _start1(self):
             self.new_display_var = ":%s" % int(self.display)
 
             _USED_DISPLAY_NR_LIST.append(self.display)
+            self._display_to_rm = self.display
 
         self._command = self._cmd() + self._extra_args
         log.debug("command: %s", self._command)
@@ -356,6 +358,11 @@ def stop(self):
 
         if self._use_xauth:
             self._clear_xauth()
+
+        if self._display_to_rm:
+            with _mutex:
+                _USED_DISPLAY_NR_LIST.remove(self._display_to_rm)
+                self._display_to_rm = None
         return self
 
     def _read_stdout_stderr(self):
diff --git a/tests/test_core.py b/tests/test_core.py
index 6d5d42d..fdd4aa1 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -4,7 +4,7 @@
 from tutil import has_xvnc, rfbport
 
 from pyvirtualdisplay import Display
-from pyvirtualdisplay.abstractdisplay import XStartError
+from pyvirtualdisplay.abstractdisplay import _USED_DISPLAY_NR_LIST, XStartError
 from pyvirtualdisplay.xephyr import XephyrDisplay
 from pyvirtualdisplay.xvfb import XvfbDisplay
 from pyvirtualdisplay.xvnc import XvncDisplay
@@ -229,3 +229,22 @@ def test_display():
     assert d.display is None
     d.start()
     assert d.display >= 0
+
+
+def test_USED_DISPLAY_NR_LIST():
+    n = len(_USED_DISPLAY_NR_LIST)
+    vd = Display()
+    vd._obj._has_displayfd = False
+    vd.start()
+    assert len(_USED_DISPLAY_NR_LIST) == n + 1
+
+    vd2 = Display()
+    vd2._obj._has_displayfd = False
+    vd2.start()
+    assert len(_USED_DISPLAY_NR_LIST) == n + 2
+
+    vd2.stop()
+    assert len(_USED_DISPLAY_NR_LIST) == n + 1
+
+    vd.stop()
+    assert len(_USED_DISPLAY_NR_LIST) == n + 0