diff --git a/ai2thor/server.py b/ai2thor/server.py
index 4091188abc..cb45d9fc51 100644
--- a/ai2thor/server.py
+++ b/ai2thor/server.py
@@ -99,9 +99,9 @@ def __init__(self, image_ids_data: bytes, metadata: dict):
         self.class_colors: Dict[str, List[List[int]]] = {}
         for c in metadata["colors"]:
             cls = c["name"]
+            self.instance_colors[c["name"]] = c["color"]
 
             if "|" in c["name"]:
-                self.instance_colors[c["name"]] = c["color"]
                 cls = c["name"].split("|")[0]
 
             if cls not in self.class_colors: