diff --git a/src/server/game/Entities/Object/Object.cpp b/src/server/game/Entities/Object/Object.cpp
index a183d1467..d39645bb4 100644
--- a/src/server/game/Entities/Object/Object.cpp
+++ b/src/server/game/Entities/Object/Object.cpp
@@ -1095,23 +1095,45 @@ void WorldObject::SetVisibilityDistanceOverride(VisibilityDistanceType type)
     if (type == GetVisibilityOverrideType())
         return;
 
-    if (IsPlayer())
+    if (!IsCreature() && !IsGameObject() && !IsDynamicObject())
         return;
 
-    if (IsVisibilityOverridden())
-    {
-        if (IsFarVisible())
-            GetMap()->RemoveWorldObjectFromFarVisibleMap(this);
-        else if (IsZoneWideVisible())
-            GetMap()->RemoveWorldObjectFromZoneWideVisibleMap(GetZoneId(), this);
-    }
-
-    if (type == VisibilityDistanceType::Large || type == VisibilityDistanceType::Gigantic)
-        GetMap()->AddWorldObjectToFarVisibleMap(this);
-    else if (type == VisibilityDistanceType::Infinite)
-        GetMap()->AddWorldObjectToZoneWideVisibleMap(GetZoneId(), this);
+    // Important to remove from old visibility override containers first
+    RemoveFromMapVisibilityOverrideContainers();
 
+    // Always update _visibilityDistanceOverrideType, even when not in world
     _visibilityDistanceOverrideType = type;
+
+    // Finally, add to new visibility override containers
+    AddToMapVisibilityOverrideContainers();
+}
+
+void WorldObject::RemoveFromMapVisibilityOverrideContainers()
+{
+    if (!IsVisibilityOverridden())
+        return;
+
+    if (!IsInWorld())
+        return;
+
+    if (IsFarVisible())
+        GetMap()->RemoveWorldObjectFromFarVisibleMap(this);
+    else if (IsZoneWideVisible())
+        GetMap()->RemoveWorldObjectFromZoneWideVisibleMap(_zoneId, this);
+}
+
+void WorldObject::AddToMapVisibilityOverrideContainers()
+{
+    if (!IsVisibilityOverridden())
+        return;
+
+    if (!IsInWorld())
+        return;
+
+    if (IsFarVisible())
+        GetMap()->AddWorldObjectToFarVisibleMap(this);
+    else if (IsZoneWideVisible())
+        GetMap()->AddWorldObjectToZoneWideVisibleMap(_zoneId, this);
 }
 
 void WorldObject::CleanupsBeforeDelete(bool /*finalCleanup*/)
@@ -1177,6 +1199,9 @@ void WorldObject::AddToWorld()
     Object::AddToWorld();
     GetMap()->GetZoneAndAreaId(GetPhaseMask(), _zoneId, _areaId, GetPositionX(), GetPositionY(), GetPositionZ());
     GetMap()->AddObjectToPendingUpdateList(this);
+
+    if (IsZoneWideVisible())
+        GetMap()->AddWorldObjectToZoneWideVisibleMap(_zoneId, this);
 }
 
 void WorldObject::RemoveFromWorld()
@@ -1184,8 +1209,7 @@ void WorldObject::RemoveFromWorld()
     if (!IsInWorld())
         return;
 
-    if (IsZoneWideVisible())
-        GetMap()->RemoveWorldObjectFromZoneWideVisibleMap(GetZoneId(), this);
+    RemoveFromMapVisibilityOverrideContainers();
 
     DestroyForVisiblePlayers();
 
diff --git a/src/server/game/Entities/Object/Object.h b/src/server/game/Entities/Object/Object.h
index 23784c9c6..141a7cf73 100644
--- a/src/server/game/Entities/Object/Object.h
+++ b/src/server/game/Entities/Object/Object.h
@@ -780,6 +780,9 @@ private:
     //bool CanDetectStealthOf(WorldObject const* obj) const;
     bool CanDetectStealthOf(WorldObject const* obj, bool checkAlert = false) const;
 
+    void RemoveFromMapVisibilityOverrideContainers();
+    void AddToMapVisibilityOverrideContainers();
+
     GuidUnorderedSet _allowedLooters;
 
     ObjectVisibilityContainer _objectVisibilityContainer;
diff --git a/src/server/game/Maps/Map.cpp b/src/server/game/Maps/Map.cpp
index 21652342a..906b3b966 100644
--- a/src/server/game/Maps/Map.cpp
+++ b/src/server/game/Maps/Map.cpp
@@ -727,8 +727,6 @@ void Map::RemoveFromMap(T* obj, bool remove)
     obj->RemoveFromWorld();
 
     obj->RemoveFromGrid();
-    if (obj->IsFarVisible())
-        RemoveWorldObjectFromFarVisibleMap(obj);
 
     obj->ResetMap();