The issue lies in the following file:
com.unity.xr.interaction.toolkit@1.0.0-pre.2\Runtime\Interaction\Interactors\XRDirectInteractor.cs

The culprit is the `OnTriggerEnter` and `OnTriggerExit` methods, which don't take into account that the colliding object might be made out of multiple colliders:
```
/// <summary>
/// See <see cref="MonoBehaviour"/>.
/// </summary>
/// <param name="other">The other <see cref="Collider"/> involved in this collision.</param>
protected void OnTriggerEnter(Collider other)
{
if (interactionManager == null)
return;

var interactable = interactionManager.TryGetInteractableForCollider(other);
if (interactable != null && !m_ValidTargets.Contains(interactable))
m_ValidTargets.Add(interactable);
}

/// <summary>
/// See <see cref="MonoBehaviour"/>.
/// </summary>
/// <param name="other">The other <see cref="Collider"/> involved in this collision.</param>
protected void OnTriggerExit(Collider other)
{
if (interactionManager == null)
return;

var interactable = interactionManager.TryGetInteractableForCollider(other);
if (interactable != null)
m_ValidTargets.Remove(interactable);
}
```

A workaround is to check if the collider belongs to the initial interactor that has entered the collider, and keep track of the colliders that are overlapping. For example:
```cs
public class PhysicsInteractor : XRDirectInteractor
{
#region Fields

private XRBaseInteractable currentInteractable;

// List of colliders from "currentInteractable" that are currently hovered.
private readonly List<Collider> colliders = new List<Collider>();

#endregion

#region Unity Lifecycle

protected override void Awake()
{
base.Awake();
if (interactionManager == null)
{
interactionManager = FindObjectOfType<XRInteractionManager>();
}
}

protected override void OnEnable()
{
base.OnEnable();
interactionManager.interactableUnregistered += OnInteractableUnregistered;
}

protected override void OnDisable()
{
base.OnDisable();
interactionManager.interactableUnregistered -= OnInteractableUnregistered;
ClearCurrentInteractable();
}

private new void OnTriggerEnter(Collider other)
{
var interactable = interactionManager.TryGetInteractableForCollider(other);
if (interactable == null)
{
return;
}

if (currentInteractable == null)
{
currentInteractable = interactable;
validTargets.Add(currentInteractable);
}

if (currentInteractable == interactable)
{
colliders.Add(other);
}
}

private new void OnTriggerExit(Collider other)
{
RemoveInteractableCollider(other);
}

#endregion

#region Methods

/// <summary>
/// Remove collider from the interactor. E.g. when the collider is detached or removed.
/// </summary>
public void RemoveInteractableCollider(Collider other)
{
colliders.Remove(other);
if (colliders.Count != 0)
{
return;
}

ClearCurrentInteractable();
}

protected override void OnSelectExited(SelectExitEventArgs args)
{
base.OnSelectExited(args);
ClearCurrentInteractable();
}

private void ClearCurrentInteractable()
{
currentInteractable = null;
validTargets.Clear();
colliders.Clear();
}

private void OnInteractableUnregistered(InteractableUnregisteredEventArgs obj)
{
foreach (var interactableCollider in obj.interactable.colliders)
{
RemoveInteractableCollider(interactableCollider);
}
}

#endregion
}
```